diff --git a/.github/workflows/update-data.yml b/.github/workflows/update-data.yml index dc4da236..f5dbbcb0 100644 --- a/.github/workflows/update-data.yml +++ b/.github/workflows/update-data.yml @@ -48,6 +48,15 @@ jobs: python scripts/build_fcc.py --states "$STATES" fi + - name: Build BEAD project areas + if: github.event_name != 'push' + continue-on-error: true + env: + FCC_USERNAME: ${{ secrets.FCC_USERNAME }} + FCC_API_TOKEN: ${{ secrets.FCC_API_TOKEN }} + STATES: ${{ github.event.inputs.states || vars.STATES || 'AZ' }} + run: python scripts/build_bead.py --states "$STATES" + - name: Commit run: | git config user.name "github-actions[bot]" diff --git a/README.md b/README.md index f38288d2..bb1c2264 100644 --- a/README.md +++ b/README.md @@ -39,6 +39,12 @@ pip install -r scripts/requirements.txt python scripts/build_fcc.py --local path/to/folder --as-of "June 30, 2025" ``` +## BEAD project areas + +`scripts/build_bead.py` downloads the BEAD Final Proposal data compiled by [BroadbandExpanded](https://broadbandexpanded.com/funding/beadfinalproposaldata) (every state's funded projects and the FCC location IDs in each), then maps those locations to H3 cells using FCC availability files. The page shades those cells purple and lists the projects within 50 miles. It runs as part of the Update fiber data workflow and writes to `data/bead/`. + +Please credit BroadbandExpanded if you reuse this data. + ## Adding contracts and bids Edit or add `data/contracts/XX.json` (XX is the state code). Pushing a change refreshes the manifest automatically. diff --git a/js/app.js b/js/app.js index f7698ed5..e8691230 100644 --- a/js/app.js +++ b/js/app.js @@ -18,6 +18,9 @@ const S = { fine: new Map(), coarse: new Map(), contracts: new Map(), + beadFine: new Map(), + beadCoarse: new Map(), + beadProjects: new Map(), search: 0 }; @@ -98,6 +101,7 @@ function initMap() { labels.style.zIndex = 450; labels.style.pointerEvents = "none"; L.tileLayer(esri + "World_Light_Gray_Reference/MapServer/tile/{z}/{y}/{x}", { maxZoom: 16, pane: "labels" }).addTo(map); + S.layers.bead = L.layerGroup().addTo(map); S.layers.fiber = L.layerGroup().addTo(map); S.layers.contracts = L.layerGroup().addTo(map); S.layers.focus = L.layerGroup().addTo(map); @@ -174,9 +178,72 @@ async function geocode(q) { return (await geocodeEsri(q)) || (await geocodeOsm(q)); } -function shard(cache, dir, id) { - if (!cache.has(id)) cache.set(id, getJSON(`data/fiber/${dir}/${id}.json`)); - return cache.get(id); +function shard(cache, dir, id, root = "fiber") { + const key = `${root}/${dir}/${id}`; + if (!cache.has(key)) cache.set(key, getJSON(`data/${key}.json`)); + return cache.get(key); +} + +async function beadNear(lat, lng) { + const m = S.manifest; + const cell = h3.latLngToCell(lat, lng, m.fineRes); + const ring = h3.gridDisk(cell, 1); + const ids = [...new Set(ring.map(c => h3.cellToParent(c, m.fineShardRes)))]; + const docs = new Map(await Promise.all(ids.map(async id => [id, await shard(S.beadFine, "r8", id, "bead")]))); + const own = new Set(); + const next = new Set(); + for (const c of ring) { + const d = docs.get(h3.cellToParent(c, m.fineShardRes)); + const v = d && d.c[c]; + if (!v) continue; + for (const [gpid] of v) (c === cell ? own : next).add(gpid); + } + const ids2 = h3.gridDisk(h3.latLngToCell(lat, lng, m.coarseShardRes), 2); + const coarse = await Promise.all(ids2.map(id => shard(S.beadCoarse, "r6", id, "bead"))); + const cells = []; + const projects = new Map(); + for (const d of coarse) { + if (!d) continue; + for (const [c, v] of Object.entries(d.c)) { + const km = haversineKm([lat, lng], h3.cellToLatLng(c)); + if (km > RADIUS_KM) continue; + cells.push({ cell: c, projects: v }); + for (const [gpid, n] of v) { + const e = projects.get(gpid) || { gpid, near: 0, km: Infinity }; + e.near += n; + e.km = Math.min(e.km, km); + projects.set(gpid, e); + } + } + } + for (const gpid of [...own, ...next]) if (!projects.has(gpid)) projects.set(gpid, { gpid, near: 0, km: 0 }); + const states = [...new Set([...projects.keys()].map(g => g.split(":")[0]))]; + const meta = new Map(await Promise.all(states.map(async st => { + if (!S.beadProjects.has(st)) S.beadProjects.set(st, getJSON(`data/bead/projects/${st}.json`)); + return [st, await S.beadProjects.get(st)]; + }))); + const list = []; + for (const e of projects.values()) { + const i = e.gpid.indexOf(":"); + const st = e.gpid.slice(0, i); + const pid = e.gpid.slice(i + 1); + const doc = meta.get(st); + const v = doc && doc.p[pid]; + if (!v) continue; + list.push({ + ...e, + state: st, + status: doc.status, + name: v[0], + awardee: v[1] || v[0] || "BEAD project", + amount: v[2], + technology: v[3], + locations: v[4], + covers: own.has(e.gpid) ? "own" : next.has(e.gpid) ? "next" : null + }); + } + list.sort((a, b) => (a.covers === "own" ? 0 : a.covers === "next" ? 1 : 2) - (b.covers === "own" ? 0 : b.covers === "next" ? 1 : 2) || a.km - b.km); + return { cells, list, byId: new Map(list.map(p => [p.gpid, p])) }; } async function fiberAt(lat, lng) { @@ -267,21 +334,43 @@ function drawFocus(place, at) { }).addTo(g); } -function drawRegion(region) { +function drawRegion(region, bead) { const g = S.layers.fiber; g.clearLayers(); - if (!region) return; - const max = region.cells.reduce((m, c) => Math.max(m, c.count), 1); - for (const c of region.cells) { + const b = S.layers.bead; + b.clearLayers(); + const fiberCells = new Map((region ? region.cells : []).map(c => [c.cell, c])); + const beadCells = new Map((bead ? bead.cells : []).map(c => [c.cell, c])); + const max = region ? region.cells.reduce((m, c) => Math.max(m, c.count), 1) : 1; + const beadText = c => { + const names = [...new Set(c.projects.map(([gpid]) => bead.byId.get(gpid)).filter(Boolean).map(p => p.awardee))]; + const n = c.projects.reduce((s, [, k]) => s + k, 0); + const shown = names.slice(0, 3).map(esc).join(", ") + (names.length > 3 ? `, +${names.length - 3} more` : ""); + return `${fmtInt(n)} BEAD locations${shown ? `
${shown}` : ""}`; + }; + for (const c of beadCells.values()) { + const poly = L.polygon(h3.cellToBoundary(c.cell), { + color: COLORS.contract, + weight: 1.5, + dashArray: "3 3", + fillColor: COLORS.contract, + fillOpacity: 0.16, + interactive: !fiberCells.has(c.cell) + }); + if (!fiberCells.has(c.cell)) poly.bindTooltip(beadText(c), { sticky: true }); + poly.addTo(b); + } + for (const c of fiberCells.values()) { const t = Math.log1p(c.count) / Math.log1p(max); const names = c.names.slice(0, 4).map(esc).join(", ") + (c.names.length > 4 ? `, +${c.names.length - 4} more` : ""); + const extra = beadCells.has(c.cell) ? `

${beadText(beadCells.get(c.cell))}` : ""; L.polygon(h3.cellToBoundary(c.cell), { color: COLORS.strand, weight: 0.5, opacity: 0.4, fillColor: COLORS.strand, fillOpacity: 0.12 + 0.5 * t - }).bindTooltip(`${fmtInt(c.count)} fiber locations
${names}`, { sticky: true }).addTo(g); + }).bindTooltip(`${fmtInt(c.count)} fiber locations
${names}${extra}`, { sticky: true }).addTo(g); } } @@ -385,6 +474,51 @@ function providerList(providers) { return ul; } +function beadSection(bead) { + const wrap = el("div", "bead"); + const covering = bead.list.filter(p => p.covers); + const others = bead.list.filter(p => !p.covers); + for (const p of covering) { + wrap.append(contractCard({ + awardee: p.awardee, + status: p.status, + where: p.covers === "own" ? "Covers this address" : "Next to this address", + program: "BEAD", + technology: p.technology, + amount: p.amount, + locations: p.locations, + notes: p.name && p.name !== p.awardee ? `Project: ${p.name}` : null + })); + } + if (others.length) { + const shown = 6; + wrap.append(el("h4", null, `BEAD projects within 50 miles (${others.length})`)); + const ul = el("ul", "list"); + others.forEach((p, i) => { + const li = el("li"); + if (i >= shown) li.hidden = true; + const parts = [`${Math.max(1, Math.round(p.km / KM_PER_MI))} mi`]; + if (p.amount) parts.push(fmtMoney(p.amount)); + if (p.technology) parts.push(p.technology); + li.append(el("span", null, p.awardee), el("span", "sub", parts.join(", "))); + ul.append(li); + }); + wrap.append(ul); + if (others.length > shown) { + const b = el("button", "more", `Show all ${others.length}`); + b.type = "button"; + b.addEventListener("click", () => { + ul.querySelectorAll("li[hidden]").forEach(li => { li.hidden = false; }); + b.remove(); + }); + wrap.append(b); + } + } + const proposed = bead.list.some(p => p.status === "proposed"); + wrap.append(el("p", "note", `BEAD project areas from state Final Proposals${proposed ? ", some still awaiting NTIA approval" : ""}. Shaded purple on the map.`)); + return wrap; +} + function contractCard(it) { const card = el("article", "contract" + (it.status === "open" ? " open" : "")); card.append(el("h4", null, it.awardee || it.title || "Open bid")); @@ -452,7 +586,7 @@ function statewideList(items) { return wrap; } -function render(place, covered, at, region, deals) { +function render(place, covered, at, region, deals, bead) { const root = $("result"); root.replaceChildren(); @@ -492,8 +626,9 @@ function render(place, covered, at, region, deals) { const local = deals.items.filter(it => it.where !== "Statewide"); const wide = deals.items.filter(it => it.where === "Statewide").sort((a, b) => (b.amount || 0) - (a.amount || 0)); local.forEach(it => cb.append(contractCard(it))); + if (bead && bead.list.length) cb.append(beadSection(bead)); if (wide.length) cb.append(statewideList(wide)); - if (!local.length && !wide.length) cb.append(el("p", "note", "No broadband contracts or open bids are listed near this address.")); + if (!local.length && !wide.length && !(bead && bead.list.length)) cb.append(el("p", "note", "No broadband contracts or open bids are listed near this address.")); for (const p of deals.programs) { const n = el("p", "note"); n.append(`${p.name}: ${p.summary} `); @@ -530,6 +665,16 @@ function renderMeta() { a.rel = "noopener"; p1.append(a); f.append(p1); + if (m.beadAsOf) { + const p2 = el("p"); + p2.append(`BEAD project areas: state Final Proposals compiled by `); + const b = el("a", null, "BroadbandExpanded"); + b.href = m.beadSource || "https://broadbandexpanded.com/funding/beadfinalproposaldata"; + b.target = "_blank"; + b.rel = "noopener"; + p2.append(b, `, as of ${m.beadAsOf}.`); + f.append(p2); + } f.append(el("p", null, "Address search by Esri, with OpenStreetMap Nominatim as backup.")); } @@ -545,12 +690,17 @@ async function check(q) { return; } const covered = S.manifest.states.includes(place.state); - const [at, region, deals] = await Promise.all([ + const hasBead = (S.manifest.beadStates || []).length > 0; + const [at, region, deals, bead] = await Promise.all([ covered ? fiberAt(place.lat, place.lng) : null, covered ? fiberRegion(place.lat, place.lng) : null, - contractsNear(place) + contractsNear(place), + hasBead ? beadNear(place.lat, place.lng).catch(() => null) : null ]); - drawRegion(region); + if (bead && (S.manifest.beadStates || []).includes(place.state)) { + deals.items = deals.items.filter(it => it.program !== "BEAD"); + } + drawRegion(region, bead); drawContracts(deals.items); const countyItems = deals.items.filter(it => !it.geometry && place.county && it.where === `${place.county} County`); if (countyItems.length) { @@ -560,7 +710,7 @@ async function check(q) { } drawFocus(place, at); S.map.fitBounds(L.latLng(place.lat, place.lng).toBounds(RADIUS_KM * 2000), { padding: [10, 10] }); - render(place, covered, at, region, deals); + render(place, covered, at, region, deals, bead); const url = new URL(location.href); url.searchParams.set("q", q); history.replaceState(null, "", url); diff --git a/scripts/build_bead.py b/scripts/build_bead.py new file mode 100644 index 00000000..fbc8fbae --- /dev/null +++ b/scripts/build_bead.py @@ -0,0 +1,326 @@ +import argparse +import csv +import io +import json +import os +import re +import shutil +import sys +import time +import urllib.request +import zipfile +from collections import Counter, defaultdict +from datetime import datetime, timezone +from pathlib import Path + +import h3 + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from build_fcc import (CACHE, COARSE, COARSE_SHARD, DATA, FINE_SHARD, FIPS, USPS, api_download, api_json, + as_of_dates, dump, log, parse_states, reset_dir) + +PAGE = "https://broadbandexpanded.com/funding/beadfinalproposaldata" +BASE = "https://broadbandexpanded.com/files/data/bead/" +TECH_ORDER = ["60", "61", "70", "71", "72", "10", "40", "0", "50"] +TECH_LABEL = { + "50": "Fiber", "61": "LEO satellite", "60": "Satellite", "70": "Unlicensed fixed wireless", + "71": "Licensed fixed wireless", "72": "Licensed fixed wireless", "40": "Cable", "10": "DSL", "0": "Other", +} +DRAFT = {"MO", "NC", "RI", "AS", "GU", "MP", "PR", "VI"} +SUBMITTED = {"GA", "IL", "PA"} + + +def norm(s): + return re.sub(r"[^a-z0-9]", "", (s or "").lower()) + + +def pick(header, *alts): + keys = {norm(h): h for h in header} + for a in alts: + if norm(a) in keys: + return keys[norm(a)] + for a in alts: + for k, h in keys.items(): + if norm(a) in k: + return h + return None + + +def clean_id(v): + v = str(v or "").strip() + return v[:-2] if v.endswith(".0") else v + + +def money(v): + try: + return round(float(re.sub(r"[^0-9.\-]", "", str(v)))) + except ValueError: + return None + + +def fetch(url, dest=None): + req = urllib.request.Request(url, headers={"User-Agent": "got-fiber-build/1.0"}) + for attempt in range(5): + try: + with urllib.request.urlopen(req, timeout=900) as r: + if dest is None: + return r.read() + with open(dest, "wb") as f: + shutil.copyfileobj(r, f, 1 << 20) + return dest + except Exception as e: + if attempt == 4: + raise + log(f" retry {url}: {e}") + time.sleep(10 * (attempt + 1)) + + +def latest_zip(): + html = fetch(PAGE).decode("utf-8", "replace") + stamps = sorted(set(re.findall(r"BEAD_FP_Data_BroadbandExpanded_(\d{8})\.zip", html))) + if not stamps: + sys.exit(f"Could not find the BEAD data zip on {PAGE}") + d = stamps[-1] + return f"{BASE}BEAD_FP_Data_BroadbandExpanded_{d}.zip", f"{d[:4]}-{d[4:6]}-{d[6:]}" + + +def table(z, prefix): + for n in z.namelist(): + base = n.rsplit("/", 1)[-1].upper() + if base.startswith(prefix) and base.endswith(".CSV"): + r = csv.DictReader(io.TextIOWrapper(z.open(n), encoding="utf-8-sig", newline="")) + log(f"{prefix} columns: {r.fieldnames}") + return r + sys.exit(f"No {prefix} table in the BEAD zip") + + +def load_bead(path, wanted): + z = zipfile.ZipFile(path) + subs = {} + r = table(z, "SUBGRANTEE") + c_st, c_uei, c_name = pick(r.fieldnames, "state"), pick(r.fieldnames, "uei"), pick(r.fieldnames, "uei_name", "name") + for row in r: + uei = (row.get(c_uei) or "").strip() + name = (row.get(c_name) or "").strip() + st = (row.get(c_st) or "").strip().upper() + if uei and name: + subs[(st, uei)] = name + subs.setdefault(("", uei), name) + + projects = {} + r = table(z, "PROJECT") + f = r.fieldnames + c_st, c_pid, c_name, c_uei = pick(f, "state"), pick(f, "project_id"), pick(f, "project_name"), pick(f, "uei") + c_amt = pick(f, "projected_bead_funding", "bead_funding", "bead_amount", "funding") + for row in r: + st = (row.get(c_st) or "").strip().upper() + if st not in wanted: + continue + uei = (row.get(c_uei) or "").strip() + projects[(st, clean_id(row.get(c_pid)))] = { + "name": (row.get(c_name) or "").strip(), + "awardee": subs.get((st, uei)) or subs.get(("", uei)) or uei, + "amount": money(row.get(c_amt)) if c_amt else None, + "tech": Counter(), + "locs": 0, + } + + locs = defaultdict(dict) + r = table(z, "LOCATION") + f = r.fieldnames + c_st, c_loc, c_pid = pick(f, "state"), pick(f, "location_id"), pick(f, "project_id") + c_tech = pick(f, "technology_code", "technology", "tech") + for row in r: + st = (row.get(c_st) or "").strip().upper() + if st not in wanted: + continue + lid, pid = clean_id(row.get(c_loc)), clean_id(row.get(c_pid)) + if not lid or not pid: + continue + locs[st][lid] = pid + p = projects.get((st, pid)) + if p is None: + p = projects[(st, pid)] = {"name": "", "awardee": "", "amount": None, "tech": Counter(), "locs": 0} + p["locs"] += 1 + p["tech"][clean_id(row.get(c_tech)) if c_tech else ""] += 1 + return projects, locs + + +def tech_label(counter): + total = sum(counter.values()) or 1 + out = [] + for code, n in counter.most_common(): + label = TECH_LABEL.get(code) + if label and n / total >= 0.1 and label not in out: + out.append(label) + return "/".join(out) + + +def fast_rows(path): + opener = [] + if path.suffix.lower() == ".zip": + z = zipfile.ZipFile(path) + for n in z.namelist(): + if n.lower().endswith(".csv"): + opener.append(lambda n=n: io.TextIOWrapper(z.open(n), encoding="utf-8-sig", newline="")) + else: + opener.append(lambda: open(path, encoding="utf-8-sig", newline="")) + for o in opener: + with o() as fh: + r = csv.reader(fh) + head = next(r, None) + if not head: + continue + try: + i_loc, i_h3 = head.index("location_id"), head.index("h3_res8_id") + except ValueError: + continue + for row in r: + if len(row) > max(i_loc, i_h3): + yield row[i_loc], row[i_h3] + + +def resolve(path, pending, found): + for lid, cell in fast_rows(path): + lid = clean_id(lid) + if lid in pending: + cell = cell.strip().lower() + if h3.is_valid_cell(cell): + found[lid] = cell + pending.discard(lid) + if not pending: + return + + +def availability_files(as_of, fips_wanted, user, token): + d = api_json(f"/downloads/listAvailabilityData/{as_of}?category=State", user, token) + out = defaultdict(lambda: defaultdict(list)) + for f in d.get("data", []): + fips = str(f.get("state_fips") or "").zfill(2) + if fips not in fips_wanted or (f.get("category") or "State") != "State": + continue + kind = f"{f.get('technology_type') or ''} {f.get('subcategory') or ''}".lower() + if "mobile" in kind or (f.get("file_type") or "csv").lower() != "csv": + continue + tech = str(f.get("technology_code") or "").strip() + if tech in TECH_ORDER: + out[fips][tech].append(f) + return out + + +def map_locations_api(locs, user, token): + fips_wanted = {FIPS[s] for s in locs if s in FIPS} + files, as_of = {}, None + for as_of in as_of_dates(user, token)[:4]: + files = availability_files(as_of, fips_wanted, user, token) + if files: + break + log(f"Mapping BEAD locations with FCC data {as_of}") + CACHE.mkdir(exist_ok=True) + found = {} + for st in sorted(locs): + pending = set(locs[st]) + total = len(pending) + by_tech = files.get(FIPS.get(st, ""), {}) + for tech in TECH_ORDER: + if not pending: + break + for f in by_tech.get(tech, []): + if not pending: + break + fid = f.get("file_id") + dest = CACHE / f"{as_of}_{fid}.zip" + log(f"{st}: tech {tech}, {len(pending)} locations left, downloading {f.get('file_name', fid)}") + api_download(f"/downloads/downloadFile/availability/{fid}", user, token, dest) + resolve(dest, pending, found) + dest.unlink() + log(f"{st}: mapped {total - len(pending)} of {total} BEAD locations") + return found + + +def map_locations_local(locs, folder): + pending = set().union(*locs.values()) if locs else set() + found = {} + for p in sorted(Path(folder).iterdir()): + if p.suffix.lower() in (".zip", ".csv") and pending: + resolve(p, pending, found) + log(f"Mapped {len(found)} BEAD locations from {folder}, {len(pending)} unmapped") + return found + + +def write(projects, locs, found, as_of, source): + root = DATA / "bead" + for sub in ("r8", "r6", "projects"): + reset_dir(root / sub) + fine = defaultdict(Counter) + mapped = Counter() + for st, ids in locs.items(): + for lid, pid in ids.items(): + cell = found.get(lid) + if cell: + fine[cell][f"{st}:{pid}"] += 1 + mapped[(st, pid)] += 1 + fine_shards = defaultdict(dict) + coarse = defaultdict(Counter) + for cell, c in fine.items(): + fine_shards[h3.cell_to_parent(cell, FINE_SHARD)][cell] = sorted(c.items()) + coarse[h3.cell_to_parent(cell, COARSE)].update(c) + for sid, cells in fine_shards.items(): + dump(root / "r8" / f"{sid}.json", {"c": {k: [list(x) for x in v] for k, v in cells.items()}}) + coarse_shards = defaultdict(dict) + for cell, c in coarse.items(): + coarse_shards[h3.cell_to_parent(cell, COARSE_SHARD)][cell] = [list(x) for x in sorted(c.items())] + for sid, cells in coarse_shards.items(): + dump(root / "r6" / f"{sid}.json", {"c": cells}) + states = sorted({st for st, _ in projects}) + for st in states: + status = "proposed" if st in DRAFT or st in SUBMITTED else "awarded" + p = {pid: [v["name"], v["awardee"], v["amount"], tech_label(v["tech"]), v["locs"], mapped[(s, pid)]] + for (s, pid), v in projects.items() if s == st} + dump(root / "projects" / f"{st}.json", {"status": status, "p": p}) + path = DATA / "manifest.json" + m = json.loads(path.read_text(encoding="utf-8")) if path.exists() else {} + m.update({ + "beadStates": states, + "beadAsOf": as_of, + "beadSource": source, + "beadGenerated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + }) + path.write_text(json.dumps(m, indent=2) + "\n", encoding="utf-8") + log(f"BEAD: {len(projects)} projects, {len(fine)} cells, {len(fine_shards)} fine and {len(coarse_shards)} coarse shards, states {states}") + + +def main(): + ap = argparse.ArgumentParser(description="Map BEAD Final Proposal projects to H3 cells") + ap.add_argument("--states", default=os.environ.get("STATES", "AZ")) + ap.add_argument("--bead-zip", help="Local copy of the BroadbandExpanded BEAD CSV zip") + ap.add_argument("--as-of", help="Date label for a local BEAD zip") + ap.add_argument("--fcc-local", help="Folder of FCC availability CSV or ZIP files instead of the API") + a = ap.parse_args() + wanted = set(parse_states(a.states)) + if a.bead_zip: + zpath, as_of = Path(a.bead_zip), a.as_of or "local data" + else: + url, as_of = latest_zip() + CACHE.mkdir(exist_ok=True) + zpath = CACHE / url.rsplit("/", 1)[-1] + if not zpath.exists(): + log(f"Downloading {url}") + fetch(url, zpath) + projects, locs = load_bead(zpath, wanted) + log(f"Loaded {len(projects)} projects and {sum(len(v) for v in locs.values())} locations") + if not projects: + sys.exit("No BEAD projects found for the requested states.") + if a.fcc_local: + found = map_locations_local(locs, a.fcc_local) + else: + user = os.environ.get("FCC_USERNAME", "").strip() + token = os.environ.get("FCC_API_TOKEN", "").strip() + if not user or not token: + sys.exit("Set FCC_USERNAME and FCC_API_TOKEN.") + found = map_locations_api(locs, user, token) + write(projects, locs, found, as_of, PAGE) + + +if __name__ == "__main__": + main() diff --git a/scripts/build_fcc.py b/scripts/build_fcc.py index 47d95acb..de1d30f7 100644 --- a/scripts/build_fcc.py +++ b/scripts/build_fcc.py @@ -224,7 +224,8 @@ def write_manifest(as_of=None, states=None): contract_files = sorted((DATA / "contracts").glob("*.json")) contract_states = [p.stem for p in contract_files] geo_states = [p.stem for p in contract_files if any(i.get("geometry") for i in json.loads(p.read_text(encoding="utf-8")).get("items", []))] - m = { + m = dict(old) + m.update({ "generated": datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), "asOf": as_of if as_of is not None else old.get("asOf"), "states": sorted(states) if states is not None else old.get("states", []), @@ -235,7 +236,7 @@ def write_manifest(as_of=None, states=None): "coarseRes": COARSE, "coarseShardRes": COARSE_SHARD, "source": "FCC National Broadband Map, fixed broadband availability, technology code 50 (fiber to the premises)", - } + }) path.write_text(json.dumps(m, indent=2) + "\n", encoding="utf-8") log(f"Manifest: states={m['states']} asOf={m['asOf']} contracts={contract_states}")