Fabio
7764501bcf
The legislative-process corpus, parallel to the U.S. Code statute corpus. schemas/bill.schema.json + scripts/ingest_legislation.py (deterministic, stdlib-only, reads per-state JSONL from the depot); validate.py gains legislation/ as a third tree; Makefile 'legislation' target. Raw JSONL stays off-repo on the depot (gitignored); the per-state SHA-256 manifest is the committed provenance anchor. Vintage 2026-07-01. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
279 lines
11 KiB
Python
279 lines
11 KiB
Python
#!/usr/bin/env python3
|
||
"""Ingest OpenStates bills into Republic OS legislation files.
|
||
|
||
The legislative-process corpus, parallel to the LegalText statute corpus:
|
||
a bill is a proposed change to law. Each bill node records its version chain
|
||
(the diff), action timeline (the PR events), sponsors (authors), and votes
|
||
(the merge decisions). Source is the OpenStates/OpenCivicData bulk dump.
|
||
|
||
Mechanical: raw JSONL export (one bill per line) -> OKF Bill Markdown nodes.
|
||
No summaries or interpretation. Deterministic: same input -> byte-identical output.
|
||
|
||
scripts/ingest_legislation.py --state fl --input data/legislation/fl.jsonl
|
||
"""
|
||
|
||
import argparse
|
||
import hashlib
|
||
import json
|
||
import re
|
||
import shutil
|
||
from pathlib import Path
|
||
|
||
REPO = Path(__file__).resolve().parent.parent
|
||
OUT_ROOT = REPO / "legislation" / "us" / "states"
|
||
|
||
RETRIEVED_AT = "2026-07-06"
|
||
DEFAULT_VINTAGE = "2026-07-01"
|
||
|
||
|
||
def snapshot_url(vintage):
|
||
return f"https://data.openstates.org/daily/{vintage}/public.pgdump"
|
||
|
||
|
||
# Values that appear in the OpenStates `subject` field but are scraper artifacts,
|
||
# not real topic tags (e.g. FL stores the literal link text "Subject Index").
|
||
JUNK_SUBJECTS = {"subject index"}
|
||
|
||
DEPOT_SRC = "/Volumes/depot/exports/legislation"
|
||
|
||
# Official-name exceptions; the fallback is "{State} Legislature".
|
||
LEGISLATURE = {
|
||
"co": "Colorado General Assembly", "ct": "Connecticut General Assembly",
|
||
"de": "Delaware General Assembly", "ga": "Georgia General Assembly",
|
||
"il": "Illinois General Assembly", "in": "Indiana General Assembly",
|
||
"ia": "Iowa General Assembly", "ky": "Kentucky General Assembly",
|
||
"md": "Maryland General Assembly", "mo": "Missouri General Assembly",
|
||
"nc": "North Carolina General Assembly", "oh": "Ohio General Assembly",
|
||
"pa": "Pennsylvania General Assembly", "ri": "Rhode Island General Assembly",
|
||
"sc": "South Carolina General Assembly", "tn": "Tennessee General Assembly",
|
||
"vt": "Vermont General Assembly", "va": "Virginia General Assembly",
|
||
"ma": "Massachusetts General Court", "nh": "New Hampshire General Court",
|
||
"nd": "North Dakota Legislative Assembly", "or": "Oregon Legislative Assembly",
|
||
}
|
||
STATE_NAME = {
|
||
"al": "Alabama", "ak": "Alaska", "az": "Arizona", "ar": "Arkansas",
|
||
"ca": "California", "co": "Colorado", "ct": "Connecticut", "de": "Delaware",
|
||
"fl": "Florida", "ga": "Georgia", "hi": "Hawaii", "id": "Idaho",
|
||
"il": "Illinois", "in": "Indiana", "ia": "Iowa", "ks": "Kansas",
|
||
"ky": "Kentucky", "la": "Louisiana", "me": "Maine", "md": "Maryland",
|
||
"ma": "Massachusetts", "mi": "Michigan", "mn": "Minnesota", "ms": "Mississippi",
|
||
"mo": "Missouri", "mt": "Montana", "ne": "Nebraska", "nv": "Nevada",
|
||
"nh": "New Hampshire", "nj": "New Jersey", "nm": "New Mexico", "ny": "New York",
|
||
"nc": "North Carolina", "nd": "North Dakota", "oh": "Ohio", "ok": "Oklahoma",
|
||
"or": "Oregon", "pa": "Pennsylvania", "ri": "Rhode Island", "sc": "South Carolina",
|
||
"sd": "South Dakota", "tn": "Tennessee", "tx": "Texas", "ut": "Utah",
|
||
"vt": "Vermont", "va": "Virginia", "wa": "Washington", "wv": "West Virginia",
|
||
"wi": "Wisconsin", "wy": "Wyoming",
|
||
}
|
||
|
||
|
||
def yval(value):
|
||
if value is None:
|
||
return '""'
|
||
if isinstance(value, bool):
|
||
return "true" if value else "false"
|
||
if isinstance(value, int):
|
||
return str(value)
|
||
if isinstance(value, list):
|
||
return "[" + ", ".join(yval(v) for v in value) + "]"
|
||
return json.dumps(str(value), ensure_ascii=False)
|
||
|
||
|
||
def slugify(value):
|
||
value = (value or "").lower()
|
||
value = re.sub(r"[^a-z0-9]+", "-", value).strip("-")
|
||
return re.sub(r"-{2,}", "-", value) or "unnamed"
|
||
|
||
|
||
def sha256_text(text):
|
||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||
|
||
|
||
def version_rank(note):
|
||
"""Order a bill's versions into a commit chain from its OpenStates note."""
|
||
tok = (note or "").split()[-1].lower() if note else ""
|
||
if tok in ("filed", "introduced", "original"):
|
||
return (0, 0)
|
||
m = re.fullmatch(r"c(\d+)", tok)
|
||
if m:
|
||
return (1, int(m.group(1)))
|
||
m = re.fullmatch(r"e(\d+)", tok)
|
||
if m:
|
||
return (2, int(m.group(1)))
|
||
if tok in ("er", "enrolled"):
|
||
return (3, 0)
|
||
return (1, 50) # unknown mid-stage: sort stably after filed
|
||
|
||
|
||
STAGE_LABEL = {0: "filed", 1: "committee substitute", 2: "engrossed", 3: "enrolled"}
|
||
|
||
|
||
def clean(s):
|
||
return re.sub(r"\s+", " ", (s or "").strip())
|
||
|
||
|
||
def primary_person_sponsors(sponsors):
|
||
return [clean(s["name"]) for s in sponsors
|
||
if s.get("primary") and s.get("entity_type") == "person"]
|
||
|
||
|
||
def chapter_law(actions):
|
||
for a in actions:
|
||
m = re.search(r"Chapter No\.\s*([0-9][0-9-]*)", a.get("description") or "")
|
||
if m:
|
||
return m.group(1)
|
||
return None
|
||
|
||
|
||
def render_bill(rec, state, vintage):
|
||
ident = clean(rec["identifier"])
|
||
session = clean(rec["session"])
|
||
title = clean(rec.get("title")) or ident
|
||
sponsors = rec.get("sponsors") or []
|
||
actions = rec.get("actions") or []
|
||
versions = sorted(rec.get("versions") or [], key=lambda v: (version_rank(v.get("note")), v.get("note") or ""))
|
||
votes = rec.get("votes") or []
|
||
subjects = [clean(x) for x in (rec.get("subjects") or [])
|
||
if clean(x) and clean(x).lower() not in JUNK_SUBJECTS]
|
||
prim = primary_person_sponsors(sponsors)
|
||
chap = chapter_law(actions)
|
||
dates = [a.get("date") for a in actions if a.get("date")]
|
||
citation = f"{STATE_NAME.get(state, state.upper())} {ident} ({session})"
|
||
|
||
fm = {
|
||
"type": "Bill",
|
||
"title": title,
|
||
"description": clean(rec.get("abstract")) or title,
|
||
"jurisdiction": f"us/states/{state}",
|
||
"legislature": LEGISLATURE.get(state, f"{STATE_NAME.get(state, state.upper())} Legislature"),
|
||
"session": session,
|
||
"identifier": ident,
|
||
"citation": citation,
|
||
"classification": rec.get("classification") or [],
|
||
"subjects": subjects,
|
||
"status": "enacted",
|
||
"chapter_law": chap,
|
||
"primary_sponsors": prim,
|
||
"version_count": len(versions),
|
||
"action_count": len(actions),
|
||
"vote_count": len(votes),
|
||
"first_action": min(dates) if dates else None,
|
||
"last_action": max(dates) if dates else None,
|
||
"source": "openstates",
|
||
"source_identifier": rec["bill_id"],
|
||
"source_url": (rec.get("sources") or [None])[0],
|
||
"source_hash": sha256_text(json.dumps(rec, sort_keys=True, ensure_ascii=False)),
|
||
"vintage": vintage,
|
||
"source_snapshot": snapshot_url(vintage),
|
||
"retrieved_at": RETRIEVED_AT,
|
||
"confidence": "reported",
|
||
"tags": ["legislation", "bill", f"us-{state}"],
|
||
}
|
||
# drop null-valued optional keys for clean, deterministic frontmatter
|
||
fm = {k: v for k, v in fm.items() if v is not None}
|
||
|
||
out = ["---"]
|
||
for k, v in fm.items():
|
||
out.append(f"{k}: {yval(v)}")
|
||
out.append("---")
|
||
out.append("")
|
||
out.append(f"# {citation} — {title}")
|
||
out.append("")
|
||
if fm["description"] != title:
|
||
out.append(fm["description"])
|
||
out.append("")
|
||
if chap:
|
||
out.append(f"**Enacted** — Laws of {STATE_NAME.get(state, state.upper())}, Chapter {chap}.")
|
||
out.append("")
|
||
|
||
out.append("## Version chain")
|
||
out.append("")
|
||
out.append("The bill's text revisions, in order — the diff chain from filing to enrollment.")
|
||
out.append("")
|
||
for i, v in enumerate(versions):
|
||
stage = STAGE_LABEL[version_rank(v.get("note"))[0]]
|
||
note = clean(v.get("note")) or stage
|
||
link = f" — [source]({v['url']})" if v.get("url") else ""
|
||
out.append(f"{i+1}. **{note}** ({stage}){link}")
|
||
out.append("")
|
||
|
||
if votes:
|
||
out.append("## Votes")
|
||
out.append("")
|
||
for v in votes:
|
||
y, n = v.get("yes"), v.get("no")
|
||
tally = f"{y}–{n}" if y is not None and n is not None else (v.get("result") or "")
|
||
ch = clean(v.get("chamber")) or ""
|
||
motion = clean(v.get("motion")) or "vote"
|
||
out.append(f"- {motion} — **{tally}** ({v.get('result')}) {('· ' + ch) if ch else ''}".rstrip())
|
||
out.append("")
|
||
|
||
if sponsors:
|
||
out.append("## Sponsors")
|
||
out.append("")
|
||
for s in sponsors:
|
||
role = "primary" if s.get("primary") else (clean(s.get("classification")) or "cosponsor")
|
||
out.append(f"- {clean(s['name'])} — {role} ({clean(s.get('entity_type'))})")
|
||
out.append("")
|
||
|
||
out.append("## Timeline")
|
||
out.append("")
|
||
out.append("The legislative action history — every referral, reading, and vote.")
|
||
out.append("")
|
||
for a in actions:
|
||
cls = ", ".join(a.get("classification") or [])
|
||
tag = f" `{cls}`" if cls else ""
|
||
out.append(f"- **{a.get('date') or '—'}** {clean(a.get('description'))}{tag}")
|
||
out.append("")
|
||
|
||
out.append("## Source")
|
||
out.append("")
|
||
out.append(f"OpenStates / OpenCivicData bulk snapshot [{vintage}]({snapshot_url(vintage)}); "
|
||
f"origin `{rec['bill_id']}`. Confidence: reported (aggregated from official {STATE_NAME.get(state, state.upper())} legislature records).")
|
||
out.append("")
|
||
|
||
path = OUT_ROOT / state / slugify(session) / f"{slugify(ident)}.md"
|
||
return path, "\n".join(out)
|
||
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument("--state", required=True)
|
||
ap.add_argument("--input", default=None,
|
||
help="raw JSONL; default: depot export for --vintage")
|
||
ap.add_argument("--vintage", default=DEFAULT_VINTAGE,
|
||
help="OpenStates bulk-snapshot date, e.g. 2026-07-01")
|
||
args = ap.parse_args()
|
||
state = args.state.lower()
|
||
src = args.input or f"{DEPOT_SRC}/{args.vintage}/enacted_{state}_{args.vintage}.jsonl"
|
||
|
||
dest = OUT_ROOT / state
|
||
if dest.exists():
|
||
shutil.rmtree(dest)
|
||
|
||
n = 0
|
||
used = {}
|
||
with open(src, encoding="utf-8") as fh:
|
||
for line in fh:
|
||
line = line.strip()
|
||
if not line:
|
||
continue
|
||
rec = json.loads(line)
|
||
# Record ALL enacted legislation honestly — bills AND joint/concurrent
|
||
# resolutions. The classification frontmatter says which; filtering
|
||
# (e.g. dropping TN's ceremonial resolutions) is a view's job, not the
|
||
# mirror's.
|
||
path, content = render_bill(rec, state, args.vintage)
|
||
occ = used.get(path, 0) + 1
|
||
used[path] = occ
|
||
if occ > 1:
|
||
path = path.with_name(f"{path.stem}-{occ}.md")
|
||
path.parent.mkdir(parents=True, exist_ok=True)
|
||
path.write_text(content + "\n", encoding="utf-8")
|
||
n += 1
|
||
print(f"{state}: wrote {n} enacted-legislation nodes to {dest.relative_to(REPO)}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|