diff --git a/.github/workflows/.gitignore b/.github/workflows/.gitignore new file mode 100644 index 00000000..1a08bac8 --- /dev/null +++ b/.github/workflows/.gitignore @@ -0,0 +1,86 @@ +name: Update fiber data + +on: + workflow_dispatch: + inputs: + states: + description: "Comma separated state codes like AZ,NM, or ALL. Blank uses the STATES variable." + required: false + default: "" + schedule: + - cron: "17 9 5 * *" + push: + branches: [main] + paths: + - "data/contracts/**" + +permissions: + contents: write + +concurrency: + group: update-data + cancel-in-progress: false + +jobs: + build: + runs-on: ubuntu-latest + timeout-minutes: 340 + steps: + - uses: actions/checkout@v5 + + - uses: actions/setup-python@v6 + with: + python-version: "3.12" + + - name: Install + run: pip install -r scripts/requirements.txt + + - name: Build + env: + FCC_USERNAME: ${{ secrets.FCC_USERNAME }} + FCC_API_TOKEN: ${{ secrets.FCC_API_TOKEN }} + STATES: ${{ github.event.inputs.states || vars.STATES || 'AZ' }} + EVENT: ${{ github.event_name }} + run: | + if [ "$EVENT" = "push" ]; then + python scripts/build_fcc.py --manifest-only + else + python scripts/build_fcc.py --states "$STATES" + fi + + - name: Build BEAD project areas + if: github.event_name != 'push' + continue-on-error: true + env: + FCC_USERNAME: ${{ secrets.FCC_USERNAME }} + FCC_API_TOKEN: ${{ secrets.FCC_API_TOKEN }} + STATES: ${{ github.event.inputs.states || vars.STATES || 'AZ' }} + run: python scripts/build_bead.py --states "$STATES" + + - name: Build Starlink availability + if: github.event_name != 'push' + continue-on-error: true + env: + FCC_USERNAME: ${{ secrets.FCC_USERNAME }} + FCC_API_TOKEN: ${{ secrets.FCC_API_TOKEN }} + STATES: ${{ github.event.inputs.states || vars.STATES || 'AZ' }} + run: python scripts/build_starlink.py --states "$STATES" + + - name: Commit + run: | + git config user.name "github-actions[bot]" + git config user.email "41898341+github-actions[bot]@users.noreply.github.com" + git add -A data + if git diff --cached --quiet; then + echo "No changes" + exit 0 + fi + git commit -m "Update fiber data" + for i in 1 2 3 4 5; do + if git pull --rebase -X theirs origin main && git push origin HEAD:main; then + exit 0 + fi + git rebase --abort 2>/dev/null || true + sleep $((i * 10)) + done + exit 1 diff --git a/README.md b/README.md index bb1c2264..83f9d48f 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,10 @@ python scripts/build_fcc.py --local path/to/folder --as-of "June 30, 2025" Please credit BroadbandExpanded if you reuse this data. +## Starlink + +`scripts/build_starlink.py` reads the FCC low Earth orbit satellite availability files (technology code 61), keeps the rows reported by Starlink, and writes the max advertised speeds per H3 resolution 7 cell to `data/starlink/`. The page shows one line in the result saying whether Starlink reports service at the address. + ## Adding contracts and bids Edit or add `data/contracts/XX.json` (XX is the state code). Pushing a change refreshes the manifest automatically. diff --git a/js/app.js b/js/app.js index 4805fbdd..70ce52c0 100644 --- a/js/app.js +++ b/js/app.js @@ -21,7 +21,8 @@ const S = { contracts: new Map(), beadFine: new Map(), beadCoarse: new Map(), - beadProjects: new Map() + beadProjects: new Map(), + starlink: new Map() }; const $ = id => document.getElementById(id); @@ -258,6 +259,48 @@ function shard(cache, dir, id, root = "fiber") { return cache.get(key); } +async function starlinkAt(lat, lng) { + const m = S.manifest; + const res = m.starlinkRes || 7; + const sr = m.starlinkShardRes || 4; + const read = async c => { + const id = h3.cellToParent(c, sr); + if (!S.starlink.has(id)) S.starlink.set(id, getJSON(`data/starlink/${id}.json`)); + const d = await S.starlink.get(id); + const i = d && d.c[c]; + return i == null ? null : d.t[i]; + }; + const cell = h3.latLngToCell(lat, lng, res); + const own = await read(cell); + if (own) return { here: true, down: own[0], up: own[1] }; + const near = (await Promise.all(h3.gridDisk(cell, 1).filter(c => c !== cell).map(read))).filter(Boolean); + if (near.length) { + return { here: false, near: true, down: Math.max(...near.map(v => v[0])), up: Math.max(...near.map(v => v[1])) }; + } + return { here: false, near: false }; +} + +function starlinkBlock(place, sl) { + const b = el("section", "block"); + b.append(el("h3", null, "Starlink")); + const states = S.manifest.starlinkStates || []; + let text; + if (!states.includes(place.state) || !sl) text = "No Starlink data for this state yet."; + else if (sl.here) text = `Starlink reports service here, up to ${fmtSpeed(sl.down)} down and ${fmtSpeed(sl.up)} up.`; + else if (sl.near) text = `Starlink reports service close by, up to ${fmtSpeed(sl.down)} down and ${fmtSpeed(sl.up)} up.`; + else text = "Starlink does not report service here."; + b.append(el("p", null, text)); + const n = el("p", "note"); + n.append("Starlink can waitlist areas that are at capacity, so confirm on "); + const a = el("a", null, "starlink.com"); + a.href = "https://www.starlink.com/map"; + a.target = "_blank"; + a.rel = "noopener"; + n.append(a, " before ordering."); + b.append(n); + return b; +} + async function beadNear(lat, lng) { const m = S.manifest; const cell = h3.latLngToCell(lat, lng, m.fineRes); @@ -629,7 +672,7 @@ function statewideList(items) { return wrap; } -function render(place, covered, at, region, deals, bead) { +function render(place, covered, at, region, deals, bead, sl) { const root = $("result"); root.replaceChildren(); @@ -645,6 +688,8 @@ function render(place, covered, at, region, deals, bead) { root.append(b); } + if ((S.manifest.starlinkStates || []).length) root.append(starlinkBlock(place, sl)); + if (covered) { const b = el("section", "block"); b.append(el("h3", null, "Fiber within 50 miles")); @@ -733,11 +778,13 @@ async function check(q) { } const covered = S.manifest.states.includes(place.state); const hasBead = (S.manifest.beadStates || []).length > 0; - const [at, region, deals, bead] = await Promise.all([ + const hasStarlink = (S.manifest.starlinkStates || []).includes(place.state); + const [at, region, deals, bead, sl] = await Promise.all([ covered ? fiberAt(place.lat, place.lng) : null, covered ? fiberRegion(place.lat, place.lng) : null, contractsNear(place), - hasBead ? beadNear(place.lat, place.lng).catch(() => null) : null + hasBead ? beadNear(place.lat, place.lng).catch(() => null) : null, + hasStarlink ? starlinkAt(place.lat, place.lng).catch(() => null) : null ]); if (bead && (S.manifest.beadStates || []).includes(place.state)) { deals.items = deals.items.filter(it => it.program !== "BEAD"); @@ -747,7 +794,7 @@ async function check(q) { drawContracts(deals.items); drawFocus(place, at); S.map.fitBounds(L.latLng(place.lat, place.lng).toBounds(RADIUS_KM * 2000), { padding: [10, 10] }); - render(place, covered, at, region, deals, bead); + render(place, covered, at, region, deals, bead, sl); const url = new URL(location.href); url.searchParams.set("q", q); history.replaceState(null, "", url); diff --git a/scripts/build_starlink.py b/scripts/build_starlink.py new file mode 100644 index 00000000..8ba21aab --- /dev/null +++ b/scripts/build_starlink.py @@ -0,0 +1,151 @@ +import argparse +import csv +import io +import json +import os +import sys +import zipfile +from collections import defaultdict +from datetime import datetime, timezone +from pathlib import Path + +import h3 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from build_fcc import CACHE, DATA, FIPS, USPS, api_download, as_of_dates, dump, log, parse_states, reset_dir +from build_bead import availability_files + +RES = 7 +SHARD = 4 +TECH = "61" + + +def to_int(v): + try: + return int(float(v)) + except (TypeError, ValueError): + return 0 + + +def is_starlink(brand, provider): + s = f"{brand} {provider}".lower() + return "starlink" in s or "space exploration" in s or "spacex" in s + + +def rows(path): + openers = [] + if path.suffix.lower() == ".zip": + z = zipfile.ZipFile(path) + for n in z.namelist(): + if n.lower().endswith(".csv"): + openers.append(lambda n=n: io.TextIOWrapper(z.open(n), encoding="utf-8-sig", newline="")) + else: + openers.append(lambda: open(path, encoding="utf-8-sig", newline="")) + for o in openers: + with o() as fh: + r = csv.reader(fh) + head = next(r, None) + if not head: + continue + idx = {k: head.index(k) if k in head else None for k in + ("brand_name", "provider_id", "technology", "h3_res8_id", + "max_advertised_download_speed", "max_advertised_upload_speed")} + if idx["h3_res8_id"] is None: + continue + top = max(i for i in idx.values() if i is not None) + g = lambda row, k: row[idx[k]] if idx[k] is not None else "" + for row in r: + if len(row) <= top: + continue + if idx["technology"] is not None and g(row, "technology").strip() != TECH: + continue + yield g(row, "brand_name"), g(row, "provider_id"), g(row, "h3_res8_id"), g(row, "max_advertised_download_speed"), g(row, "max_advertised_upload_speed") + + +def ingest(path, cells): + n = 0 + for brand, provider, cell, down, up in rows(path): + if not is_starlink(brand, provider): + continue + cell = cell.strip().lower() + if not h3.is_valid_cell(cell): + continue + c7 = h3.cell_to_parent(cell, RES) + e = cells.get(c7) + d, u = to_int(down), to_int(up) + if e is None: + cells[c7] = [d, u] + else: + e[0] = max(e[0], d) + e[1] = max(e[1], u) + n += 1 + return n + + +def write(cells, states, as_of): + root = DATA / "starlink" + reset_dir(root) + shards = defaultdict(dict) + for c, v in cells.items(): + shards[h3.cell_to_parent(c, SHARD)][c] = v + for sid, cs in shards.items(): + tiers = sorted({tuple(v) for v in cs.values()}) + idx = {t: i for i, t in enumerate(tiers)} + dump(root / f"{sid}.json", {"t": [list(t) for t in tiers], "c": {c: idx[tuple(v)] for c, v in cs.items()}}) + path = DATA / "manifest.json" + m = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + m.update({ + "starlinkStates": sorted(states), + "starlinkAsOf": as_of, + "starlinkRes": RES, + "starlinkShardRes": SHARD, + "starlinkGenerated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + }) + path.write_text(json.dumps(m, indent=2) + "\n", encoding="utf-8") + log(f"Starlink: {len(cells)} cells in {len(shards)} shards, states {sorted(states)}") + + +def main(): + ap = argparse.ArgumentParser(description="Build Starlink availability tiles from FCC data") + ap.add_argument("--states", default=os.environ.get("STATES", "AZ")) + ap.add_argument("--local", help="Folder of FCC satellite availability CSV or ZIP files") + ap.add_argument("--as-of", default="local data") + a = ap.parse_args() + states = parse_states(a.states) + cells, done = {}, set() + if a.local: + for p in sorted(Path(a.local).iterdir()): + if p.suffix.lower() in (".zip", ".csv"): + log(f"{p.name}: {ingest(p, cells)} Starlink rows") + done = set(states) + as_of = a.as_of + else: + user = os.environ.get("FCC_USERNAME", "").strip() + token = os.environ.get("FCC_API_TOKEN", "").strip() + if not user or not token: + sys.exit("Set FCC_USERNAME and FCC_API_TOKEN.") + wanted = {FIPS[s] for s in states} + files, as_of = {}, None + for as_of in as_of_dates(user, token)[:4]: + files = availability_files(as_of, wanted, user, token) + if any(TECH in v for v in files.values()): + break + CACHE.mkdir(exist_ok=True) + for fips in sorted(files): + st = USPS[fips] + for f in files[fips].get(TECH, []): + fid = f.get("file_id") + dest = CACHE / f"{as_of}_{fid}.zip" + if not dest.exists(): + log(f"{st}: downloading {f.get('file_name', fid)}") + api_download(f"/downloads/downloadFile/availability/{fid}", user, token, dest) + log(f"{st}: {ingest(dest, cells)} Starlink rows, {len(cells)} cells so far") + dest.unlink() + done.add(st) + if not cells: + sys.exit("No Starlink rows found. Nothing written.") + write(cells, done, as_of) + + +if __name__ == "__main__": + main()