Fabio
1beeaab2db
The first edges in the repo. Every US county now links to the districts it sits in, weighted by area overlap, and every legislator links to the district they represent — county → district → representative, traversable. Data (PostGIS, no Atlas dependency — computed directly from geometry): - data/county_district_edges.jsonl — 16,328 weighted edges. area_weight = ST_Area(ST_Intersection(county, district))/ST_Area(county) over ST_MakeValid Census TIGER 2024 geometries, overlaps <0.5% dropped. Uses area-intersection, NOT centroid-in-polygon, so urban districts that carve through counties (e.g. GA CD-5 / Atlanta across Clayton+DeKalb+Fulton) are captured, not silently dropped. Per-county weights sum to ~1.0 per district type. build.py: - ~6,896 new district nodes: 435 CD (ACS demographics where available; sparse for the 8 states missing from acs_cd) + 4,927 state-house + 1,897 state-senate (geometry-only). GEOID/number-keyed, stable. - county index.md gains a `districts:` block (frontmatter) + a ## Districts section (body links, so link-integrity checks them). - legislators + US House members gain `represents:` pointing at their district node; edge stored once on the person, inverse left to the view. - schemas/jurisdiction: classification enum + chamber/districts fields. Honest gaps (logged, not hidden): ID + NH have no state-house geometry, so their house districts get nodes (represents resolves) but no county edges; CD nodes for acs_cd-missing states are demographically sparse. Deterministic (byte-identical rebuild); validate.py passes 26,617 records, link-integrity clean (every district/represents link resolves). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
864 lines
35 KiB
Python
864 lines
35 KiB
Python
#!/usr/bin/env python3
|
||
"""Build the OKF entity tree from the raw source exports in data/.
|
||
|
||
Inputs (committed raw, one JSON object per line):
|
||
officeholders-v3.jsonl Atlas — person->seat records (13,329)
|
||
bodies.jsonl congress-legislators — institutions (233)
|
||
leadership.jsonl current federal leadership roles (28)
|
||
committee_memberships.jsonl person->committee edges (3,879)
|
||
place_county_crosswalk.jsonl Census place -> county (PostGIS join) (32,041)
|
||
county_district_edges.jsonl county -> district area-overlap edges (16,328)
|
||
|
||
Outputs (fully regenerated each run):
|
||
data/jurisdictions/** Person files (federal ones enriched with
|
||
bioguide, leadership, committee seats)
|
||
data/jurisdictions/us/bodies/** Body files (chambers, committees,
|
||
subcommittees) with their leadership
|
||
|
||
Deterministic by construction: sorted iteration, fixed key order, one full
|
||
rebuild per run. Two consecutive runs produce a byte-identical tree.
|
||
"""
|
||
|
||
import json
|
||
import re
|
||
import shutil
|
||
import unicodedata
|
||
from pathlib import Path
|
||
|
||
REPO = Path(__file__).resolve().parent.parent
|
||
DATA = REPO / "data"
|
||
OUT = DATA / "jurisdictions"
|
||
BODIES_OUT = OUT / "us" / "bodies"
|
||
DATASET_DATE = "2026-07-04" # officeholders v3 export date
|
||
CONGRESS_DATE = "2026-07-03" # congress-legislators ingest date
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# helpers
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def slugify(text):
|
||
text = unicodedata.normalize("NFKD", text).encode("ascii", "ignore").decode()
|
||
text = re.sub(r"[^a-z0-9]+", "-", text.lower()).strip("-")
|
||
return re.sub(r"-{2,}", "-", text) or "unnamed"
|
||
|
||
|
||
def yval(v):
|
||
if isinstance(v, bool):
|
||
return "true" if v else "false"
|
||
if isinstance(v, (int, float)):
|
||
return str(v)
|
||
return json.dumps(v, ensure_ascii=False)
|
||
|
||
|
||
def norm_name(n):
|
||
"""Normalize a person name for cross-source reconciliation."""
|
||
n = re.sub(r'["\'][^"\']*["\']', "", n) # quoted nickname
|
||
n = re.sub(r",?\s+(Jr|Sr|II|III|IV)\.?(?=\s|$)", "", n) # suffix (any pos)
|
||
return re.sub(r"\s+", " ", n).strip().lower()
|
||
|
||
|
||
def norm_county(name):
|
||
return re.sub(r"\bSaint\b", "St", name)
|
||
|
||
|
||
def norm_place(s):
|
||
"""Normalize a Census place name for crosswalk lookup (bare, ascii, alnum)."""
|
||
s = unicodedata.normalize("NFKD", s or "").encode("ascii", "ignore").decode()
|
||
return re.sub(r"[^a-z0-9]+", " ", s.lower()).strip()
|
||
|
||
|
||
def acs_cslug(row):
|
||
"""County directory slug derived from an ACS county row (the authority for
|
||
county node names — municipals must nest under the same slug)."""
|
||
base = re.sub(r",\s*[A-Z]{2}$", "", row.get("county_name") or "")
|
||
return slugify(norm_county(re.sub(r"\s+County$", "", base)))
|
||
|
||
|
||
def place_resolver(acs_county, crosswalk):
|
||
"""(normalized place name, state) -> county dir slug, from the crosswalk.
|
||
|
||
county_geoid is the stable key; the slug is taken from the ACS county node
|
||
(the same source that names county dirs) so municipals nest correctly. Only
|
||
unique name matches whose county has an ACS node are included — ambiguous
|
||
names, label-less rows, and counties with no ACS node fall through.
|
||
Shared by build.py and build_viz.py so the tree and the Board agree.
|
||
"""
|
||
fips_to_cslug = {canonical_fips(r["county_fips"]): acs_cslug(r) for r in acs_county}
|
||
geoids = {}
|
||
for x in crosswalk:
|
||
geoids.setdefault(
|
||
(norm_place(x["place_name"]), x["state_code"]), set()).add(x["county_geoid"])
|
||
out = {}
|
||
for key, geos in geoids.items():
|
||
if len(geos) == 1:
|
||
cf = canonical_fips(next(iter(geos)))
|
||
if cf in fips_to_cslug:
|
||
out[key] = fips_to_cslug[cf]
|
||
return out
|
||
|
||
|
||
# committee_title (source) -> normalized role on the committee
|
||
ROLE_MAP = {
|
||
"member": "member", "Chair": "chair", "Chairman": "chair",
|
||
"Chairwoman": "chair", "Cochairman": "co-chair", "Vice Chair": "vice-chair",
|
||
"Vice Chairman": "vice-chair", "Vice Chairwoman": "vice-chair",
|
||
"Ranking Member": "ranking-member", "Ex Officio": "ex-officio",
|
||
}
|
||
LEADER_ROLES = {"chair", "co-chair", "vice-chair", "ranking-member"}
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# body paths / ids
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
INSTITUTIONS = {"house", "senate", "executive"}
|
||
|
||
|
||
def committee_codes(bodies):
|
||
return sorted((b["code"] for b in bodies if b["type"] == "committee"),
|
||
key=len, reverse=True)
|
||
|
||
|
||
def parent_code(sub_code, comm_codes):
|
||
for c in comm_codes:
|
||
if sub_code.startswith(c) and sub_code != c:
|
||
return c
|
||
return None
|
||
|
||
|
||
def body_id(code, bodies_by_code, comm_codes):
|
||
"""Stable path-based id (relative to data/jurisdictions) for a body."""
|
||
b = bodies_by_code[code]
|
||
if b["type"] in INSTITUTIONS:
|
||
return f"us/bodies/{slugify(b['name'])}"
|
||
if b["type"] == "committee":
|
||
return f"us/bodies/{b['chamber']}/committees/{slugify(short_name(b))}"
|
||
parent = parent_code(code, comm_codes)
|
||
pb = bodies_by_code[parent]
|
||
return (f"us/bodies/{pb['chamber']}/committees/{slugify(short_name(pb))}"
|
||
f"/subcommittees/{slugify(short_name(b))}")
|
||
|
||
|
||
def short_name(b):
|
||
"""Committee/subcommittee name minus redundant chamber/parent prefix."""
|
||
name = b["name"]
|
||
name = re.sub(r"^(House|Senate|Joint)\s+(Committee|Subcommittee)\s+on\s+",
|
||
"", name)
|
||
if " - " in name: # subcommittee: keep the tail
|
||
name = name.split(" - ", 1)[1]
|
||
return name
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# federal enrichment index (name -> bioguide, leadership, committee seats)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def build_enrichment(leadership, memberships, bodies_by_code, comm_codes):
|
||
idx = {}
|
||
|
||
def slot(name):
|
||
return idx.setdefault(norm_name(name),
|
||
{"bioguide": None, "leadership": [], "committees": []})
|
||
|
||
for l in leadership:
|
||
s = slot(l["person_name"])
|
||
s["bioguide"] = l["bioguide"]
|
||
s["leadership"].append({
|
||
"role": l["role_title"], "body": l["body_code"],
|
||
"since": l.get("start_date"),
|
||
})
|
||
for m in memberships:
|
||
s = slot(m["person_name"])
|
||
s["bioguide"] = m["bioguide"]
|
||
s["committees"].append({
|
||
"code": m["body_code"], "name": m["committee_name"],
|
||
"role": ROLE_MAP.get(m["committee_title"], "member"),
|
||
"rank": m.get("rank"),
|
||
"id": body_id(m["body_code"], bodies_by_code, comm_codes),
|
||
})
|
||
for s in idx.values():
|
||
s["leadership"].sort(key=lambda r: (r["role"], r["body"]))
|
||
s["committees"].sort(key=lambda c: (c["id"]))
|
||
return idx
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Person files
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def county_slug(rec, place_to_cslug=None):
|
||
label = norm_county(rec.get("jurisdiction_label") or "")
|
||
m = re.match(r"(.+?) County, FL$", label)
|
||
if m:
|
||
return slugify(m.group(1))
|
||
if rec["level"] == "municipal":
|
||
m = re.search(r"County:\s*([^;]+)", rec.get("description") or "")
|
||
if m:
|
||
return slugify(norm_county(m.group(1).strip()))
|
||
title = rec.get("title") or ""
|
||
m = re.search(r"\bof\s+(.+?)\s+County\b", title) or re.match(r"(.+?)\s+County\b", title)
|
||
if m:
|
||
return slugify(norm_county(m.group(1)))
|
||
# Census place -> county via the committed crosswalk (place_county_crosswalk).
|
||
# Only unique name matches whose county has an ACS node are resolved here;
|
||
# ambiguous names, label-less rows, and CT planning regions fall through.
|
||
if place_to_cslug and rec["level"] == "municipal":
|
||
jl = rec.get("jurisdiction_label") or ""
|
||
pm = re.match(r"(.+?),\s*[A-Z]{2}$", jl) # "Tampa, FL" -> "Tampa"
|
||
placename = pm.group(1) if pm else jl # "Elgin" -> "Elgin"
|
||
cslug = place_to_cslug.get((norm_place(placename), rec.get("state_abbr")))
|
||
if cslug:
|
||
return cslug
|
||
return "_unresolved"
|
||
|
||
|
||
def city_slug(rec):
|
||
jl = rec.get("jurisdiction_label") or ""
|
||
m = re.match(r"(.+?),\s*[A-Z]{2}$", jl) # "Elgin, TX" / "Naples, FL"
|
||
if m:
|
||
return slugify(m.group(1))
|
||
if jl: # bare "Elgin"
|
||
return slugify(jl)
|
||
m = re.match(r"Mayor of (.+)$", rec.get("title") or "")
|
||
if m:
|
||
return slugify(m.group(1))
|
||
return "_unresolved"
|
||
|
||
|
||
def person_dir(rec, place_to_cslug=None):
|
||
lvl, st = rec["level"], (rec.get("state_abbr") or "").lower()
|
||
if lvl == "federal":
|
||
return OUT / "us" / "people"
|
||
if lvl == "state":
|
||
return OUT / "us" / "states" / st / "people"
|
||
if lvl == "county":
|
||
return OUT / "us" / "states" / st / "counties" / county_slug(rec, place_to_cslug) / "people"
|
||
if lvl == "municipal":
|
||
return (OUT / "us" / "states" / st / "counties" / county_slug(rec, place_to_cslug)
|
||
/ "municipalities" / city_slug(rec) / "people")
|
||
return OUT / "_unresolved" / "people"
|
||
|
||
|
||
def display_name(rec):
|
||
return rec.get("full_name") or rec.get("title") or "Unknown"
|
||
|
||
|
||
def person_frontmatter(rec, enr):
|
||
lines = ["type: Person", f"title: {yval(display_name(rec))}"]
|
||
jl = rec.get("jurisdiction_label")
|
||
desc = rec.get("title") or ""
|
||
if jl and jl not in desc:
|
||
desc = f"{desc} — {jl}" if desc else jl
|
||
if desc:
|
||
lines.append(f"description: {yval(desc)}")
|
||
if rec.get("title"):
|
||
lines.append(f"role: {yval(rec['title'])}")
|
||
if rec.get("party"):
|
||
lines.append(f"party: {yval(rec['party'])}")
|
||
lines.append(f"level: {yval(rec['level'])}")
|
||
if rec.get("branch"):
|
||
lines.append(f"branch: {yval(rec['branch'])}")
|
||
if rec.get("state_abbr"):
|
||
lines.append(f"state: {yval(rec['state_abbr'])}")
|
||
if rec.get("jurisdiction_type") == "district" and jl:
|
||
lines.append(f"district: {yval(jl)}")
|
||
dnode = person_district_node(rec)
|
||
if dnode:
|
||
lines.append(f"represents: {yval(dnode)}")
|
||
if enr and enr["leadership"]:
|
||
lines.append("leadership:")
|
||
for r in enr["leadership"]:
|
||
lines.append(f" - role: {yval(r['role'])}")
|
||
lines.append(f" body: {yval(r['body'])}")
|
||
if r.get("since"):
|
||
lines.append(f" since: {yval(r['since'])}")
|
||
if enr and enr["committees"]:
|
||
lines.append("committees:")
|
||
for c in enr["committees"]:
|
||
lines.append(f" - name: {yval(c['name'])}")
|
||
lines.append(f" role: {yval(c['role'])}")
|
||
lines.append(f" body: {yval(c['id'])}")
|
||
contact = [(k, rec.get(k)) for k in ("email", "phone", "website") if rec.get(k)]
|
||
if contact:
|
||
lines.append("contact:")
|
||
for k, v in contact:
|
||
lines.append(f" {k}: {yval(v)}")
|
||
tenure = [(k2, rec.get(k1)) for k1, k2 in
|
||
(("start_date", "start"), ("end_date", "end"),
|
||
("is_current", "current"), ("tenure_notes", "notes"))
|
||
if rec.get(k1) is not None]
|
||
if tenure:
|
||
lines.append("tenure:")
|
||
for k, v in tenure:
|
||
lines.append(f" {k}: {yval(v)}")
|
||
election = [(k2, rec.get(k1)) for k1, k2 in
|
||
(("next_election_year", "next"), ("term_length", "term_length"),
|
||
("term_limit", "term_limit")) if rec.get(k1) is not None]
|
||
if election:
|
||
lines.append("election:")
|
||
for k, v in election:
|
||
lines.append(f" {k}: {yval(v)}")
|
||
lines.append("ids:")
|
||
for k1, k2 in (("person_id", "person"), ("office_id", "office"),
|
||
("tenure_id", "tenure"), ("jurisdiction_id", "jurisdiction")):
|
||
if rec.get(k1):
|
||
lines.append(f" {k2}: {yval(rec[k1])}")
|
||
if enr and enr["bioguide"]:
|
||
lines.append(f" bioguide: {yval(enr['bioguide'])}")
|
||
srcs = [(f, rec[k]) for f, k in (("office", "office_source"),
|
||
("tenure", "tenure_source"), ("jurisdiction", "jurisdiction_source"))
|
||
if rec.get(k)]
|
||
if enr and (enr["leadership"] or enr["committees"]):
|
||
srcs.append(("roles", "congress-legislators (unitedstates project)"))
|
||
if srcs:
|
||
lines.append("sources:")
|
||
for f, s in srcs:
|
||
lines.append(f" - field: {f}")
|
||
lines.append(f" source: {yval(s)}")
|
||
lines.append("confidence: official")
|
||
tags = ["officeholder", rec["level"]]
|
||
if rec.get("branch"):
|
||
tags.append(rec["branch"])
|
||
if rec.get("state_abbr"):
|
||
tags.append(rec["state_abbr"].lower())
|
||
lines.append("tags: [" + ", ".join(tags) + "]")
|
||
lines.append(f"timestamp: {yval(DATASET_DATE)}")
|
||
return lines
|
||
|
||
|
||
def person_body(rec, enr):
|
||
name = display_name(rec)
|
||
role = rec.get("title") or "officeholder"
|
||
jl = rec.get("jurisdiction_label")
|
||
where = f" ({jl})" if jl and jl not in role else ""
|
||
cur = "Current" if rec.get("is_current") else "Former"
|
||
out = [f"# {name}", "", f"{cur} {role}{where}.", ""]
|
||
dnode = person_district_node(rec)
|
||
if dnode:
|
||
out += [f"Represents [{rec.get('jurisdiction_label')}](/{dnode}.md).", ""]
|
||
if enr and enr["leadership"]:
|
||
out += ["## Leadership", ""]
|
||
for r in enr["leadership"]:
|
||
since = f" (since {r['since']})" if r.get("since") else ""
|
||
out.append(f"- {r['role']}{since}")
|
||
out.append("")
|
||
if enr and enr["committees"]:
|
||
out += ["## Committees", ""]
|
||
for c in enr["committees"]:
|
||
tag = "" if c["role"] == "member" else f" — **{c['role']}**"
|
||
out.append(f"- [{c['name']}](/{c['id']}.md){tag}")
|
||
out.append("")
|
||
out += ["## Sources", ""]
|
||
any_src = False
|
||
for f, k in (("office", "office_source"), ("tenure", "tenure_source"),
|
||
("jurisdiction", "jurisdiction_source")):
|
||
if rec.get(k):
|
||
out.append(f"- {f}: {rec[k]}")
|
||
any_src = True
|
||
if enr and (enr["leadership"] or enr["committees"]):
|
||
out.append("- roles: congress-legislators (unitedstates project)")
|
||
any_src = True
|
||
if not any_src:
|
||
out.append("- (no field-level source recorded)")
|
||
out += ["", f"Generated from the Atlas officeholders v3 export ({DATASET_DATE})."]
|
||
return out
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Body files
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
TYPE_LABEL = {"house": "Chamber", "senate": "Chamber", "executive": "Executive Office",
|
||
"committee": "Committee", "subcommittee": "Subcommittee"}
|
||
|
||
|
||
def body_file(b, bid, bodies_by_code, comm_codes, leaders_of, members_count):
|
||
lines = ["type: Body", f"title: {yval(b['name'])}",
|
||
f"classification: {yval(b['type'])}",
|
||
f"chamber: {yval(b['chamber'])}", f"code: {yval(b['code'])}"]
|
||
if b["type"] == "subcommittee":
|
||
pc = parent_code(b["code"], comm_codes)
|
||
lines.append(f"parent: {yval(body_id(pc, bodies_by_code, comm_codes))}")
|
||
lead = leaders_of.get(b["code"], [])
|
||
if lead:
|
||
lines.append("leadership:")
|
||
for person_name, role, pid in lead:
|
||
lines.append(f" - person: {yval(person_name)}")
|
||
lines.append(f" role: {yval(role)}")
|
||
lines.append("sources:")
|
||
lines.append(" - field: definition")
|
||
lines.append(" source: congress-legislators (unitedstates project)")
|
||
lines.append("confidence: official")
|
||
tags = ["body", b["type"], b["chamber"]]
|
||
lines.append("tags: [" + ", ".join(dict.fromkeys(tags)) + "]")
|
||
lines.append(f"timestamp: {yval(CONGRESS_DATE)}")
|
||
|
||
body = [f"# {b['name']}", "",
|
||
f"{TYPE_LABEL.get(b['type'], 'Body')} ({b['chamber']}).", ""]
|
||
if lead:
|
||
body += ["## Leadership", ""]
|
||
for person_name, role, pid in lead:
|
||
body.append(f"- {role}: {person_name}")
|
||
body.append("")
|
||
if members_count:
|
||
body += [f"{members_count} members "
|
||
"(see each member's file for their seat on this body).", ""]
|
||
body += ["## Source", "",
|
||
"- definition: congress-legislators (unitedstates project)"]
|
||
return "---\n" + "\n".join(lines) + "\n---\n\n" + "\n".join(body) + "\n"
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Candidate files (FEC)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
STANCE = {"I": "incumbent", "C": "challenger", "O": "open-seat"}
|
||
CAND_STATUS = {"C": "statutory candidate", "N": "not yet a candidate",
|
||
"P": "prior candidate", "F": "future candidate"}
|
||
PARTY_FULL = {"REP": "Republican", "DEM": "Democratic", "IND": "Independent",
|
||
"LIB": "Libertarian", "GRE": "Green",
|
||
"DFL": "Democratic-Farmer-Labor"}
|
||
|
||
|
||
def candidate_name(raw):
|
||
"""'CARL, JERRY LEE, JR' -> 'Jerry Lee Carl Jr'."""
|
||
parts = [p.strip() for p in raw.split(",")]
|
||
last = parts[0] if parts else raw
|
||
first = parts[1] if len(parts) > 1 else ""
|
||
suffix = parts[2] if len(parts) > 2 else ""
|
||
return " ".join(x for x in (first, last, suffix) if x).title()
|
||
|
||
|
||
def candidate_office(rec):
|
||
return "U.S. Senate" if rec.get("office") == "S" else "U.S. House"
|
||
|
||
|
||
def candidate_seat(rec):
|
||
st = rec.get("state") or ""
|
||
if rec.get("office") == "H" and rec.get("district"):
|
||
return f"{st}-{rec['district']}"
|
||
return st
|
||
|
||
|
||
def candidate_frontmatter(rec, disp):
|
||
office, seat, yr = candidate_office(rec), candidate_seat(rec), rec.get("election_year")
|
||
lines = ["type: Candidate", f"title: {yval(disp)}",
|
||
f"description: {yval(f'{office} candidate, {seat} ({yr})')}",
|
||
f"office: {yval(office)}"]
|
||
if rec.get("state"):
|
||
lines.append(f"state: {yval(rec['state'])}")
|
||
if rec.get("office") == "H" and rec.get("district"):
|
||
lines.append(f"district: {yval(seat)}")
|
||
if rec.get("party"):
|
||
lines.append(f"party: {yval(rec['party'])}")
|
||
if rec.get("incumbent_challenge") in STANCE:
|
||
lines.append(f"stance: {yval(STANCE[rec['incumbent_challenge']])}")
|
||
if yr:
|
||
lines.append(f"election_year: {yr}")
|
||
if rec.get("candidate_status") in CAND_STATUS:
|
||
lines.append(f"status: {yval(CAND_STATUS[rec['candidate_status']])}")
|
||
if rec.get("committee_name"):
|
||
lines.append("committee:")
|
||
lines.append(f" name: {yval(rec['committee_name'])}")
|
||
if rec.get("committee_id"):
|
||
lines.append(f" id: {yval(rec['committee_id'])}")
|
||
lines.append("ids:")
|
||
lines.append(f" fec: {yval(rec['fec_id'])}")
|
||
lines.append("sources:")
|
||
lines.append(" - field: filing")
|
||
lines.append(" source: FEC (Federal Election Commission)")
|
||
lines.append("confidence: official")
|
||
tags = ["candidate", "federal",
|
||
"senate" if rec.get("office") == "S" else "house"]
|
||
if rec.get("state"):
|
||
tags.append(rec["state"].lower())
|
||
lines.append("tags: [" + ", ".join(tags) + "]")
|
||
lines.append(f"timestamp: {yval(CONGRESS_DATE)}")
|
||
return lines
|
||
|
||
|
||
def candidate_body(rec, disp):
|
||
office, seat = candidate_office(rec), candidate_seat(rec)
|
||
stance = STANCE.get(rec.get("incumbent_challenge"), "candidate")
|
||
party = PARTY_FULL.get(rec.get("party"), rec.get("party") or "")
|
||
out = [f"# {disp}", "",
|
||
f"{party} {stance} for {office} ({seat}), {rec.get('election_year')}.", ""]
|
||
if rec.get("committee_name"):
|
||
cid = f" ({rec['committee_id']})" if rec.get("committee_id") else ""
|
||
out += ["## Campaign Committee", "", f"- {rec['committee_name']}{cid}", ""]
|
||
out += ["## Source", "", "- filing: FEC (Federal Election Commission)", "",
|
||
"Federal candidate filing; not yet linked to an officeholder record."]
|
||
return out
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# Jurisdiction demographic nodes (ACS)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def numify(v):
|
||
if v is None or v == "":
|
||
return None
|
||
if isinstance(v, (int, float)):
|
||
return v
|
||
try:
|
||
f = float(v)
|
||
return int(f) if "." not in str(v) and f == int(f) else round(f, 4)
|
||
except (ValueError, TypeError):
|
||
return v
|
||
|
||
|
||
def canonical_fips(fips):
|
||
return re.sub(r"\D", "", str(fips))[-5:]
|
||
|
||
|
||
DEMOG_LABELS = {
|
||
"population": "Population", "population_under_18": "Under 18",
|
||
"population_18_64": "18–64", "population_65_plus": "65+",
|
||
"median_household_income": "Median household income",
|
||
"poverty_rate": "Poverty rate", "homeownership_rate": "Homeownership rate",
|
||
"unemployment_rate": "Unemployment rate", "median_home_value": "Median home value",
|
||
"gini_index": "Gini index", "vacancy_rate": "Vacancy rate",
|
||
"race_white": "White", "race_black": "Black", "race_asian": "Asian",
|
||
"race_native": "Native", "hispanic": "Hispanic/Latino",
|
||
"bachelors_plus": "Bachelor's or higher",
|
||
}
|
||
|
||
|
||
def normalize_demog(row):
|
||
def pick(*cols):
|
||
for c in cols:
|
||
if row.get(c) not in (None, ""):
|
||
return numify(row.get(c))
|
||
return None
|
||
d = {
|
||
"population": pick("total_population"),
|
||
"population_under_18": pick("population_under_18"),
|
||
"population_18_64": pick("population_18_64"),
|
||
"population_65_plus": pick("population_65_plus"),
|
||
"median_household_income": pick("median_household_income"),
|
||
"poverty_rate": pick("poverty_rate"),
|
||
"homeownership_rate": pick("homeownership_rate"),
|
||
"unemployment_rate": pick("unemployment_rate"),
|
||
"median_home_value": pick("median_home_value"),
|
||
"gini_index": pick("gini_index"),
|
||
"vacancy_rate": pick("vacancy_rate"),
|
||
"race_white": pick("race_white"), "race_black": pick("race_black"),
|
||
"race_asian": pick("race_asian"), "race_native": pick("race_native"),
|
||
"hispanic": pick("hispanic_latino", "hispanic"),
|
||
}
|
||
bp = pick("edu_bachelors_plus")
|
||
if bp is None:
|
||
bp = (pick("edu_bachelors") or 0) + (pick("edu_graduate") or 0) or None
|
||
d["bachelors_plus"] = bp
|
||
return {k: v for k, v in d.items() if v is not None}
|
||
|
||
|
||
def demog_yaml(demog):
|
||
lines = ["demographics:"]
|
||
for k, v in demog.items():
|
||
lines.append(f" {k}: {yval(v) if isinstance(v, str) else v}")
|
||
return lines
|
||
|
||
|
||
def demog_table(demog):
|
||
out = ["## Demographics (ACS 2023)", "", "| Measure | Value |", "| --- | --- |"]
|
||
for k, v in demog.items():
|
||
out.append(f"| {DEMOG_LABELS.get(k, k)} | {v} |")
|
||
return out + [""]
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# District nodes + county->district edges (the geographic authority layer)
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
DTYPE_CLASS = {"CD": "congressional-district",
|
||
"SS": "state-senate-district", "SH": "state-house-district"}
|
||
DTYPE_RANK = {"CD": 0, "SS": 1, "SH": 2}
|
||
DTYPE_WORD = {"CD": "congressional", "SS": "state senate", "SH": "state house"}
|
||
DISTRICT_SOURCE = "PostGIS area-intersection over Census TIGER 2024 boundaries"
|
||
|
||
|
||
def cd_ident(label):
|
||
"""'Congressional District 5' -> '05'; at-large -> '00'."""
|
||
if re.search(r"at.?large", label, re.I):
|
||
return "00"
|
||
m = re.search(r"(\d+)", label)
|
||
return f"{int(m.group(1)):02d}" if m else "00"
|
||
|
||
|
||
def leg_ident(label):
|
||
"""State-leg identifier: text after 'District '. 'GA State Senate District 10A' -> '10A'."""
|
||
m = re.search(r"\bDistrict\s+(.+)$", label.strip())
|
||
return (m.group(1).strip() if m else label.strip())
|
||
|
||
|
||
def district_node_id(dtype, state, ident):
|
||
"""Concept id (path under data/jurisdictions) for a district node."""
|
||
st = (state or "").lower()
|
||
if dtype == "CD":
|
||
return f"us/states/{st}/districts/{ident}"
|
||
sub = "senate" if dtype == "SS" else "house"
|
||
return f"us/states/{st}/districts/{sub}/{slugify(ident)}"
|
||
|
||
|
||
def edge_node_id(e):
|
||
"""District node id for a county_district_edges row."""
|
||
dt = e["type"]
|
||
ident = cd_ident(e["district_label"]) if dt == "CD" else leg_ident(e["district_label"])
|
||
return district_node_id(dt, e["district_state"], ident)
|
||
|
||
|
||
def district_title(dtype, state, ident):
|
||
if dtype == "CD":
|
||
return f"{state}-{ident}"
|
||
return f"{state} {'Senate' if dtype == 'SS' else 'House'} District {ident}"
|
||
|
||
|
||
def person_district_parts(rec):
|
||
"""(dtype, state, ident) of the district a legislator / US House member holds, else None."""
|
||
jl = rec.get("jurisdiction_label") or ""
|
||
if rec["level"] == "federal" and re.search(r"congressional district", jl, re.I):
|
||
return ("CD", rec.get("state_abbr"), cd_ident(jl))
|
||
if rec["level"] == "state" and rec.get("branch") == "legislative":
|
||
m = re.match(r"^([A-Z]{2})\s", jl)
|
||
st = m.group(1) if m else rec.get("state_abbr")
|
||
dt = ("SS" if re.search(r"state senate", jl, re.I)
|
||
else "SH" if re.search(r"state house|house of rep", jl, re.I) else None)
|
||
if dt and st:
|
||
return (dt, st, leg_ident(jl))
|
||
return None
|
||
|
||
|
||
def person_district_node(rec):
|
||
"""District node id a legislator / US House member represents, else None."""
|
||
p = person_district_parts(rec)
|
||
return district_node_id(*p) if p else None
|
||
|
||
|
||
def jurisdiction_file(title, classification, extra_fm, demog, st, body_intro,
|
||
district_edges=None):
|
||
"""district_edges: sorted list of (node_id, dtype, title, area_weight)."""
|
||
lines = ["type: Jurisdiction", f"title: {yval(title)}",
|
||
f"classification: {classification}"]
|
||
lines += extra_fm
|
||
if demog:
|
||
lines += demog_yaml(demog)
|
||
if district_edges:
|
||
lines.append("districts:")
|
||
for nid, dt, dtitle, w in district_edges:
|
||
lines += [f" - to: {yval(nid)}", " rel: in-district",
|
||
f" area_weight: {w}"]
|
||
src = []
|
||
if demog:
|
||
src.append(("demographics", "Census ACS 2023"))
|
||
if district_edges:
|
||
src.append(("districts", DISTRICT_SOURCE))
|
||
if not src: # sparse district node
|
||
src.append(("boundary", "Census TIGER 2024"))
|
||
lines.append("sources:")
|
||
for f, s in src:
|
||
lines += [f" - field: {f}", f" source: {yval(s)}"]
|
||
lines += ["confidence: official",
|
||
f"tags: [jurisdiction, {classification}, {st}]",
|
||
f"timestamp: {yval(CONGRESS_DATE)}"]
|
||
body = [f"# {title}", "", body_intro, ""]
|
||
if demog:
|
||
body += demog_table(demog)
|
||
if district_edges:
|
||
body += ["## Districts", ""]
|
||
for nid, dt, dtitle, w in district_edges:
|
||
body.append(f"- [{dtitle}](/{nid}.md) — {round(w * 100)}% ({DTYPE_WORD[dt]})")
|
||
body.append("")
|
||
body += ["## Source", ""] + [f"- {f}: {s}" for f, s in src]
|
||
return "---\n" + "\n".join(lines) + "\n---\n\n" + "\n".join(body) + "\n"
|
||
|
||
|
||
# --------------------------------------------------------------------------- #
|
||
# main
|
||
# --------------------------------------------------------------------------- #
|
||
|
||
def load(name):
|
||
return [json.loads(l) for l in (DATA / name).open()]
|
||
|
||
|
||
def main():
|
||
officeholders = load("officeholders-v3.jsonl")
|
||
bodies = load("bodies.jsonl")
|
||
leadership = load("leadership.jsonl")
|
||
memberships = load("committee_memberships.jsonl")
|
||
candidates = load("fec_candidates.jsonl")
|
||
acs_county = load("acs_county.jsonl")
|
||
acs_cd = load("acs_cd.jsonl")
|
||
crosswalk = load("place_county_crosswalk.jsonl")
|
||
place_to_cslug = place_resolver(acs_county, crosswalk)
|
||
county_district_edges = load("county_district_edges.jsonl")
|
||
|
||
# --- geographic edge layer: county fips -> district edges, and the full
|
||
# district node set (edge targets + districts legislators represent) ---
|
||
district_nodes = {} # node_id -> (dtype, state, ident)
|
||
|
||
def reg_district(dt, state, ident):
|
||
district_nodes.setdefault(district_node_id(dt, state, ident), (dt, state, ident))
|
||
|
||
edges_by_county = {}
|
||
for e in county_district_edges:
|
||
dt = e["type"]
|
||
ident = cd_ident(e["district_label"]) if dt == "CD" else leg_ident(e["district_label"])
|
||
reg_district(dt, e["district_state"], ident)
|
||
edges_by_county.setdefault(canonical_fips(e["county_geoid"]), []).append(
|
||
(edge_node_id(e), dt, district_title(dt, e["district_state"], ident),
|
||
e["area_weight"]))
|
||
for lst in edges_by_county.values():
|
||
lst.sort(key=lambda x: (DTYPE_RANK[x[1]], -x[3], x[0]))
|
||
for row in acs_cd: # keep every existing ACS CD node
|
||
reg_district("CD", row["state_abbr"], row["district"])
|
||
for rec in officeholders: # ensure represents targets exist
|
||
p = person_district_parts(rec)
|
||
if p:
|
||
reg_district(*p)
|
||
|
||
bodies_by_code = {b["code"]: b for b in bodies}
|
||
comm_codes = committee_codes(bodies)
|
||
enrichment = build_enrichment(leadership, memberships, bodies_by_code, comm_codes)
|
||
|
||
# reverse index: body code -> its leaders, and -> member count
|
||
leaders_of, members_count = {}, {}
|
||
name_to_pid = {norm_name(r.get("full_name") or ""): r.get("person_id")
|
||
for r in officeholders if r.get("full_name")}
|
||
for m in memberships:
|
||
members_count[m["body_code"]] = members_count.get(m["body_code"], 0) + 1
|
||
role = ROLE_MAP.get(m["committee_title"], "member")
|
||
if role in LEADER_ROLES:
|
||
pid = name_to_pid.get(norm_name(m["person_name"]))
|
||
leaders_of.setdefault(m["body_code"], []).append(
|
||
(m["person_name"], role, pid))
|
||
for v in leaders_of.values():
|
||
v.sort(key=lambda t: (t[1], t[0]))
|
||
|
||
matched = sum(1 for r in officeholders if r["level"] == "federal"
|
||
and norm_name(r.get("full_name") or "") in enrichment)
|
||
|
||
# ---- plan person paths (disambiguate deterministically) ----
|
||
planned = {}
|
||
for rec in sorted(officeholders, key=lambda r: r["person_id"]):
|
||
planned.setdefault((person_dir(rec, place_to_cslug), slugify(display_name(rec))), []).append(rec)
|
||
person_files = {}
|
||
for (d, base), group in planned.items():
|
||
by_person = {}
|
||
for rec in group:
|
||
slug = base if len(group) == 1 else f"{base}-{rec['person_id'][:8]}"
|
||
by_person.setdefault(slug, []).append(rec)
|
||
for slug, sub in by_person.items():
|
||
for rec in sub:
|
||
final = slug if len(sub) == 1 else f"{slug}-{rec['office_id'][:8]}"
|
||
p = d / f"{final}.md"
|
||
assert p not in person_files, f"collision {p}"
|
||
person_files[p] = rec
|
||
|
||
# ---- write ----
|
||
if OUT.exists():
|
||
shutil.rmtree(OUT)
|
||
|
||
for path, rec in sorted(person_files.items()):
|
||
enr = enrichment.get(norm_name(rec.get("full_name") or "")) \
|
||
if rec["level"] == "federal" else None
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text("---\n" + "\n".join(person_frontmatter(rec, enr))
|
||
+ "\n---\n\n" + "\n".join(person_body(rec, enr)) + "\n",
|
||
encoding="utf-8")
|
||
|
||
for b in sorted(bodies, key=lambda x: x["code"]):
|
||
bid = body_id(b["code"], bodies_by_code, comm_codes)
|
||
path = OUT / (bid + ".md")
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(body_file(b, bid, bodies_by_code, comm_codes,
|
||
leaders_of, members_count.get(b["code"], 0)),
|
||
encoding="utf-8")
|
||
|
||
# ---- candidates (FEC) ----
|
||
cand_plan = {}
|
||
for rec in sorted(candidates, key=lambda r: r["fec_id"]):
|
||
d = OUT / "us" / "states" / (rec.get("state") or "xx").lower() / "candidates"
|
||
cand_plan.setdefault((d, slugify(candidate_name(rec["candidate_name"]))), []).append(rec)
|
||
n_cand = 0
|
||
for (d, base), group in sorted(cand_plan.items(), key=lambda kv: str(kv[0][0]) + kv[0][1]):
|
||
for rec in group:
|
||
slug = base if len(group) == 1 else f"{base}-{rec['fec_id'][-4:].lower()}"
|
||
disp = candidate_name(rec["candidate_name"])
|
||
path = d / f"{slug}.md"
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text("---\n" + "\n".join(candidate_frontmatter(rec, disp))
|
||
+ "\n---\n\n" + "\n".join(candidate_body(rec, disp)) + "\n",
|
||
encoding="utf-8")
|
||
n_cand += 1
|
||
|
||
# ---- county jurisdictions (ACS, deduped by canonical fips) ----
|
||
oh_count = {}
|
||
for rec in officeholders:
|
||
if rec["level"] in ("county", "municipal"):
|
||
key = ((rec.get("state_abbr") or "").lower(), county_slug(rec, place_to_cslug))
|
||
oh_count[key] = oh_count.get(key, 0) + 1
|
||
seen_fips, n_county = set(), 0
|
||
for row in sorted(acs_county, key=lambda r: canonical_fips(r["county_fips"])):
|
||
cf = canonical_fips(row["county_fips"])
|
||
if cf in seen_fips:
|
||
continue
|
||
seen_fips.add(cf)
|
||
st = row.get("state_abbr")
|
||
if not st:
|
||
continue
|
||
base = re.sub(r",\s*[A-Z]{2}$", "", row.get("county_name") or "")
|
||
cslug = acs_cslug(row)
|
||
title = f"{base}, {st}"
|
||
demog = normalize_demog(row)
|
||
ohc = oh_count.get((st.lower(), cslug), 0)
|
||
intro = (f"County jurisdiction — {ohc} officeholders mapped."
|
||
if ohc else "County jurisdiction.")
|
||
extra = [f"fips: {yval(cf)}", f"state: {yval(st)}"]
|
||
path = OUT / "us" / "states" / st.lower() / "counties" / cslug / "index.md"
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(jurisdiction_file(title, "county", extra, demog, st.lower(),
|
||
intro, district_edges=edges_by_county.get(cf)),
|
||
encoding="utf-8")
|
||
n_county += 1
|
||
|
||
# ---- district jurisdictions (CD demographics from ACS; state-leg + gap CDs sparse) ----
|
||
acs_cd_by_key = {(r["state_abbr"], r["district"]): r for r in acs_cd}
|
||
n_cd = n_sld = 0
|
||
for nid in sorted(district_nodes):
|
||
dt, st, ident = district_nodes[nid]
|
||
title = district_title(dt, st, ident)
|
||
cls = DTYPE_CLASS[dt]
|
||
if dt == "CD":
|
||
row = acs_cd_by_key.get((st, ident))
|
||
demog = normalize_demog(row) if row else None
|
||
extra = [f"state: {yval(st)}", f"district: {yval(title)}"]
|
||
intro = f"Congressional district {title}."
|
||
n_cd += 1
|
||
else:
|
||
demog = None
|
||
extra = [f"state: {yval(st)}",
|
||
f"chamber: {yval('senate' if dt == 'SS' else 'house')}",
|
||
f"district: {yval(ident)}"]
|
||
intro = f"{DTYPE_WORD[dt].title()} district {ident} ({st})."
|
||
n_sld += 1
|
||
path = OUT / (nid + ".md")
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(jurisdiction_file(title, cls, extra, demog, st.lower(), intro),
|
||
encoding="utf-8")
|
||
|
||
print(f"candidate files: {n_cand} county jurisdictions: {n_county} "
|
||
f"CD nodes: {n_cd} state-leg district nodes: {n_sld} "
|
||
f"county->district edges: {len(county_district_edges)}")
|
||
print(f"person files: {len(person_files)} "
|
||
f"(federal enriched: {matched}/{sum(1 for r in officeholders if r['level']=='federal')})")
|
||
print(f"body files: {len(bodies)} "
|
||
f"(chambers/exec {sum(1 for b in bodies if b['type'] in INSTITUTIONS)}, "
|
||
f"committees {sum(1 for b in bodies if b['type']=='committee')}, "
|
||
f"subcommittees {sum(1 for b in bodies if b['type']=='subcommittee')})")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|