Add Starlink availability line to results

.github/workflows/.gitignore

@@ -0,0 +1,86 @@
+name: Update fiber data
+
+on:
+  workflow_dispatch:
+    inputs:
+      states:
+        description: "Comma separated state codes like AZ,NM, or ALL. Blank uses the STATES variable."
+        required: false
+        default: ""
+  schedule:
+    - cron: "17 9 5 * *"
+  push:
+    branches: [main]
+    paths:
+      - "data/contracts/**"
+
+permissions:
+  contents: write
+
+concurrency:
+  group: update-data
+  cancel-in-progress: false
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    timeout-minutes: 340
+    steps:
+      - uses: actions/checkout@v5
+
+      - uses: actions/setup-python@v6
+        with:
+          python-version: "3.12"
+
+      - name: Install
+        run: pip install -r scripts/requirements.txt
+
+      - name: Build
+        env:
+          FCC_USERNAME: ${{ secrets.FCC_USERNAME }}
+          FCC_API_TOKEN: ${{ secrets.FCC_API_TOKEN }}
+          STATES: ${{ github.event.inputs.states || vars.STATES || 'AZ' }}
+          EVENT: ${{ github.event_name }}
+        run: |
+          if [ "$EVENT" = "push" ]; then
+            python scripts/build_fcc.py --manifest-only
+          else
+            python scripts/build_fcc.py --states "$STATES"
+          fi
+
+      - name: Build BEAD project areas
+        if: github.event_name != 'push'
+        continue-on-error: true
+        env:
+          FCC_USERNAME: ${{ secrets.FCC_USERNAME }}
+          FCC_API_TOKEN: ${{ secrets.FCC_API_TOKEN }}
+          STATES: ${{ github.event.inputs.states || vars.STATES || 'AZ' }}
+        run: python scripts/build_bead.py --states "$STATES"
+
+      - name: Build Starlink availability
+        if: github.event_name != 'push'
+        continue-on-error: true
+        env:
+          FCC_USERNAME: ${{ secrets.FCC_USERNAME }}
+          FCC_API_TOKEN: ${{ secrets.FCC_API_TOKEN }}
+          STATES: ${{ github.event.inputs.states || vars.STATES || 'AZ' }}
+        run: python scripts/build_starlink.py --states "$STATES"
+
+      - name: Commit
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "41898341+github-actions[bot]@users.noreply.github.com"
+          git add -A data
+          if git diff --cached --quiet; then
+            echo "No changes"
+            exit 0
+          fi
+          git commit -m "Update fiber data"
+          for i in 1 2 3 4 5; do
+            if git pull --rebase -X theirs origin main && git push origin HEAD:main; then
+              exit 0
+            fi
+            git rebase --abort 2>/dev/null || true
+            sleep $((i * 10))
+          done
+          exit 1

README.md

@@ -45,6 +45,10 @@ python scripts/build_fcc.py --local path/to/folder --as-of "June 30, 2025"
 
 Please credit BroadbandExpanded if you reuse this data.
 
+## Starlink
+
+`scripts/build_starlink.py` reads the FCC low Earth orbit satellite availability files (technology code 61), keeps the rows reported by Starlink, and writes the max advertised speeds per H3 resolution 7 cell to `data/starlink/`. The page shows one line in the result saying whether Starlink reports service at the address.
+
 ## Adding contracts and bids
 
 Edit or add `data/contracts/XX.json` (XX is the state code). Pushing a change refreshes the manifest automatically.

js/app.js

@@ -21,7 +21,8 @@ const S = {
   contracts: new Map(),
   beadFine: new Map(),
   beadCoarse: new Map(),
-  beadProjects: new Map()
+  beadProjects: new Map(),
+  starlink: new Map()
 };
 
 const $ = id => document.getElementById(id);
@@ -258,6 +259,48 @@ function shard(cache, dir, id, root = "fiber") {
   return cache.get(key);
 }
 
+async function starlinkAt(lat, lng) {
+  const m = S.manifest;
+  const res = m.starlinkRes || 7;
+  const sr = m.starlinkShardRes || 4;
+  const read = async c => {
+    const id = h3.cellToParent(c, sr);
+    if (!S.starlink.has(id)) S.starlink.set(id, getJSON(`data/starlink/${id}.json`));
+    const d = await S.starlink.get(id);
+    const i = d && d.c[c];
+    return i == null ? null : d.t[i];
+  };
+  const cell = h3.latLngToCell(lat, lng, res);
+  const own = await read(cell);
+  if (own) return { here: true, down: own[0], up: own[1] };
+  const near = (await Promise.all(h3.gridDisk(cell, 1).filter(c => c !== cell).map(read))).filter(Boolean);
+  if (near.length) {
+    return { here: false, near: true, down: Math.max(...near.map(v => v[0])), up: Math.max(...near.map(v => v[1])) };
+  }
+  return { here: false, near: false };
+}
+
+function starlinkBlock(place, sl) {
+  const b = el("section", "block");
+  b.append(el("h3", null, "Starlink"));
+  const states = S.manifest.starlinkStates || [];
+  let text;
+  if (!states.includes(place.state) || !sl) text = "No Starlink data for this state yet.";
+  else if (sl.here) text = `Starlink reports service here, up to ${fmtSpeed(sl.down)} down and ${fmtSpeed(sl.up)} up.`;
+  else if (sl.near) text = `Starlink reports service close by, up to ${fmtSpeed(sl.down)} down and ${fmtSpeed(sl.up)} up.`;
+  else text = "Starlink does not report service here.";
+  b.append(el("p", null, text));
+  const n = el("p", "note");
+  n.append("Starlink can waitlist areas that are at capacity, so confirm on ");
+  const a = el("a", null, "starlink.com");
+  a.href = "https://www.starlink.com/map";
+  a.target = "_blank";
+  a.rel = "noopener";
+  n.append(a, " before ordering.");
+  b.append(n);
+  return b;
+}
+
 async function beadNear(lat, lng) {
   const m = S.manifest;
   const cell = h3.latLngToCell(lat, lng, m.fineRes);
@@ -629,7 +672,7 @@ function statewideList(items) {
   return wrap;
 }
 
-function render(place, covered, at, region, deals, bead) {
+function render(place, covered, at, region, deals, bead, sl) {
   const root = $("result");
   root.replaceChildren();
 
@@ -645,6 +688,8 @@ function render(place, covered, at, region, deals, bead) {
     root.append(b);
   }
 
+  if ((S.manifest.starlinkStates || []).length) root.append(starlinkBlock(place, sl));
+
   if (covered) {
     const b = el("section", "block");
     b.append(el("h3", null, "Fiber within 50 miles"));
@@ -733,11 +778,13 @@ async function check(q) {
     }
     const covered = S.manifest.states.includes(place.state);
     const hasBead = (S.manifest.beadStates || []).length > 0;
-    const [at, region, deals, bead] = await Promise.all([
+    const hasStarlink = (S.manifest.starlinkStates || []).includes(place.state);
+    const [at, region, deals, bead, sl] = await Promise.all([
       covered ? fiberAt(place.lat, place.lng) : null,
       covered ? fiberRegion(place.lat, place.lng) : null,
       contractsNear(place),
-      hasBead ? beadNear(place.lat, place.lng).catch(() => null) : null
+      hasBead ? beadNear(place.lat, place.lng).catch(() => null) : null,
+      hasStarlink ? starlinkAt(place.lat, place.lng).catch(() => null) : null
     ]);
     if (bead && (S.manifest.beadStates || []).includes(place.state)) {
       deals.items = deals.items.filter(it => it.program !== "BEAD");
@@ -747,7 +794,7 @@ async function check(q) {
     drawContracts(deals.items);
     drawFocus(place, at);
     S.map.fitBounds(L.latLng(place.lat, place.lng).toBounds(RADIUS_KM * 2000), { padding: [10, 10] });
-    render(place, covered, at, region, deals, bead);
+    render(place, covered, at, region, deals, bead, sl);
     const url = new URL(location.href);
     url.searchParams.set("q", q);
     history.replaceState(null, "", url);

scripts/build_starlink.py

@@ -0,0 +1,151 @@
+import argparse
+import csv
+import io
+import json
+import os
+import sys
+import zipfile
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+import h3
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from build_fcc import CACHE, DATA, FIPS, USPS, api_download, as_of_dates, dump, log, parse_states, reset_dir
+from build_bead import availability_files
+
+RES = 7
+SHARD = 4
+TECH = "61"
+
+
+def to_int(v):
+    try:
+        return int(float(v))
+    except (TypeError, ValueError):
+        return 0
+
+
+def is_starlink(brand, provider):
+    s = f"{brand} {provider}".lower()
+    return "starlink" in s or "space exploration" in s or "spacex" in s
+
+
+def rows(path):
+    openers = []
+    if path.suffix.lower() == ".zip":
+        z = zipfile.ZipFile(path)
+        for n in z.namelist():
+            if n.lower().endswith(".csv"):
+                openers.append(lambda n=n: io.TextIOWrapper(z.open(n), encoding="utf-8-sig", newline=""))
+    else:
+        openers.append(lambda: open(path, encoding="utf-8-sig", newline=""))
+    for o in openers:
+        with o() as fh:
+            r = csv.reader(fh)
+            head = next(r, None)
+            if not head:
+                continue
+            idx = {k: head.index(k) if k in head else None for k in
+                   ("brand_name", "provider_id", "technology", "h3_res8_id",
+                    "max_advertised_download_speed", "max_advertised_upload_speed")}
+            if idx["h3_res8_id"] is None:
+                continue
+            top = max(i for i in idx.values() if i is not None)
+            g = lambda row, k: row[idx[k]] if idx[k] is not None else ""
+            for row in r:
+                if len(row) <= top:
+                    continue
+                if idx["technology"] is not None and g(row, "technology").strip() != TECH:
+                    continue
+                yield g(row, "brand_name"), g(row, "provider_id"), g(row, "h3_res8_id"), g(row, "max_advertised_download_speed"), g(row, "max_advertised_upload_speed")
+
+
+def ingest(path, cells):
+    n = 0
+    for brand, provider, cell, down, up in rows(path):
+        if not is_starlink(brand, provider):
+            continue
+        cell = cell.strip().lower()
+        if not h3.is_valid_cell(cell):
+            continue
+        c7 = h3.cell_to_parent(cell, RES)
+        e = cells.get(c7)
+        d, u = to_int(down), to_int(up)
+        if e is None:
+            cells[c7] = [d, u]
+        else:
+            e[0] = max(e[0], d)
+            e[1] = max(e[1], u)
+        n += 1
+    return n
+
+
+def write(cells, states, as_of):
+    root = DATA / "starlink"
+    reset_dir(root)
+    shards = defaultdict(dict)
+    for c, v in cells.items():
+        shards[h3.cell_to_parent(c, SHARD)][c] = v
+    for sid, cs in shards.items():
+        tiers = sorted({tuple(v) for v in cs.values()})
+        idx = {t: i for i, t in enumerate(tiers)}
+        dump(root / f"{sid}.json", {"t": [list(t) for t in tiers], "c": {c: idx[tuple(v)] for c, v in cs.items()}})
+    path = DATA / "manifest.json"
+    m = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {}
+    m.update({
+        "starlinkStates": sorted(states),
+        "starlinkAsOf": as_of,
+        "starlinkRes": RES,
+        "starlinkShardRes": SHARD,
+        "starlinkGenerated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"),
+    })
+    path.write_text(json.dumps(m, indent=2) + "\n", encoding="utf-8")
+    log(f"Starlink: {len(cells)} cells in {len(shards)} shards, states {sorted(states)}")
+
+
+def main():
+    ap = argparse.ArgumentParser(description="Build Starlink availability tiles from FCC data")
+    ap.add_argument("--states", default=os.environ.get("STATES", "AZ"))
+    ap.add_argument("--local", help="Folder of FCC satellite availability CSV or ZIP files")
+    ap.add_argument("--as-of", default="local data")
+    a = ap.parse_args()
+    states = parse_states(a.states)
+    cells, done = {}, set()
+    if a.local:
+        for p in sorted(Path(a.local).iterdir()):
+            if p.suffix.lower() in (".zip", ".csv"):
+                log(f"{p.name}: {ingest(p, cells)} Starlink rows")
+        done = set(states)
+        as_of = a.as_of
+    else:
+        user = os.environ.get("FCC_USERNAME", "").strip()
+        token = os.environ.get("FCC_API_TOKEN", "").strip()
+        if not user or not token:
+            sys.exit("Set FCC_USERNAME and FCC_API_TOKEN.")
+        wanted = {FIPS[s] for s in states}
+        files, as_of = {}, None
+        for as_of in as_of_dates(user, token)[:4]:
+            files = availability_files(as_of, wanted, user, token)
+            if any(TECH in v for v in files.values()):
+                break
+        CACHE.mkdir(exist_ok=True)
+        for fips in sorted(files):
+            st = USPS[fips]
+            for f in files[fips].get(TECH, []):
+                fid = f.get("file_id")
+                dest = CACHE / f"{as_of}_{fid}.zip"
+                if not dest.exists():
+                    log(f"{st}: downloading {f.get('file_name', fid)}")
+                    api_download(f"/downloads/downloadFile/availability/{fid}", user, token, dest)
+                log(f"{st}: {ingest(dest, cells)} Starlink rows, {len(cells)} cells so far")
+                dest.unlink()
+                done.add(st)
+    if not cells:
+        sys.exit("No Starlink rows found. Nothing written.")
+    write(cells, done, as_of)
+
+
+if __name__ == "__main__":
+    main()