diff --git a/.gitignore b/.gitignore index 3e98bfcfc6..e67334fd7e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,8 @@ # "raw-first, in-repo" discipline turns on at v1.0 — see README. data/legal/raw/**/*.zip data/legal/raw/**/*.xml +# Legislation (OpenStates): per-state enacted exports are the raw bytes +# (~1.1GB/vintage), kept on the depot at /depot/exports/legislation//; +# ingest reads them from there. Only the manifest (committed under data/sources/) +# + the rendered Bill nodes get committed. +data/legislation/ diff --git a/Makefile b/Makefile index 5a43081e18..2a9a7b60dc 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Republic OS — repeatable pipeline commands. # The build is deterministic: `make build` twice yields a byte-identical tree. -.PHONY: build validate board legal-us-code authority check +.PHONY: build validate board legal-us-code authority legislation check build: ## Regenerate the entity tree from raw exports in data/ python3 scripts/build.py @@ -27,4 +27,10 @@ legal-us-code: ## Ingest the pinned U.S. Code titles (override with TITLES="…" authority: ## Extract the § -> executive-office authority edges (a committed input) python3 scripts/extract_authority.py +LEG_VINTAGE ?= 2026-07-01 +LEG_STATES ?= ak al ar az ca co ct de fl ga hi ia id il in ks ky la ma md me mi mn mo ms mt nc nd ne nh nj nm nv ny oh ok or pa ri sc sd tn tx ut va vt wa wi wv wy + +legislation: ## Ingest enacted state legislation from the depot export (override LEG_STATES="fl ca") + @for s in $(LEG_STATES); do python3 scripts/ingest_legislation.py --state $$s --vintage $(LEG_VINTAGE); done + check: build validate ## Build, then validate — the full gate diff --git a/data/sources/legislation_manifest_2026-07-01.json b/data/sources/legislation_manifest_2026-07-01.json new file mode 100755 index 0000000000..4b7e4de7b0 --- /dev/null +++ b/data/sources/legislation_manifest_2026-07-01.json @@ -0,0 +1,310 @@ +{ + "vintage": "2026-07-01", + "source": "OpenStates bulk snapshot (data.openstates.org)", + "source_url": "https://data.openstates.org/daily/2026-07-01/public.pgdump", + "generated_at": "2026-07-06T20:25:28Z", + "total_bills": 197131, + "total_states": 50, + "states": [ + { + "state": "ak", + "enacted_bills": 472, + "bytes": 5056780, + "sha256": "e926ef824c9a19ca48ae72b73f5fe1cb265a58de44c4fd3342057a8cfaa92e1c" + }, + { + "state": "al", + "enacted_bills": 4042, + "bytes": 15462516, + "sha256": "4a08be5b7c66a6bc9b7ff18607605329ee4e1602d0568ea26a7b751c099f855c" + }, + { + "state": "ar", + "enacted_bills": 5166, + "bytes": 20988670, + "sha256": "182c19c260d6665c1a297b7818ddd270a32ee59cb25a8c6848a336ff1edb9618" + }, + { + "state": "az", + "enacted_bills": 2927, + "bytes": 9771351, + "sha256": "e8b30b05c84c3012e382d4e2c3e43e32d0b9b40a10378aa69c8b2a95a4bcbae2" + }, + { + "state": "ca", + "enacted_bills": 19201, + "bytes": 154829262, + "sha256": "029990f77124ce5ee80d638255f5ac93caed112a0a17a05a1e021726a02880da" + }, + { + "state": "co", + "enacted_bills": 4563, + "bytes": 29263453, + "sha256": "9c6173452331c2b9cc0a3a920bbe917eedce7012cd2d65132ed5a7e66bb4a471" + }, + { + "state": "ct", + "enacted_bills": 1671, + "bytes": 9173617, + "sha256": "394a8555f0e3aa7a6c68b97d6e6a1269407822cfc5d5bccaa2179b30c5c3ad90" + }, + { + "state": "de", + "enacted_bills": 2138, + "bytes": 9675510, + "sha256": "65758ad762a7e6a23e0012ef570468022dea1d4012d36e6126ff0a49b7f278d5" + }, + { + "state": "fl", + "enacted_bills": 2186, + "bytes": 14587374, + "sha256": "471f47358310803e56c288d6c20d8bb975c642bc5583f6d0507ac08ebf2e6b17" + }, + { + "state": "ga", + "enacted_bills": 4933, + "bytes": 20269678, + "sha256": "ea6f721f7b480d9363383adf350c9ffb29c7bff27aef78e89dcd5c465afe3e28" + }, + { + "state": "hi", + "enacted_bills": 1765, + "bytes": 19610329, + "sha256": "d2c70ffc73283a1ddcdf7bf53b87f7b25bc3b33ea7a9eb0123ac5a03a7ab8392" + }, + { + "state": "ia", + "enacted_bills": 1680, + "bytes": 7130701, + "sha256": "0a1640a6fd76b508d421e2df8e4c30611e236266be013405c863894329e5c637" + }, + { + "state": "id", + "enacted_bills": 3358, + "bytes": 15560374, + "sha256": "d63a6ca3a067758f5ffd87440ae75850ad0463e163b85e2cd62644e2f6c0926b" + }, + { + "state": "il", + "enacted_bills": 15148, + "bytes": 129476874, + "sha256": "d905d3d0b7809a91bb863de9bfbc310df1d7136efc818aee57f1fca06f4d9752" + }, + { + "state": "in", + "enacted_bills": 1861, + "bytes": 13458201, + "sha256": "96279ef4e11d8503a4a7f90c522005aa31b49f966330e0614dbd21c8bb309cd8" + }, + { + "state": "ks", + "enacted_bills": 927, + "bytes": 4320913, + "sha256": "a7c7d6b715c09d3efcd4ee03ef6c841050a4eeb8b7f9705c6a5a7c6d27cfda72" + }, + { + "state": "ky", + "enacted_bills": 1914, + "bytes": 10759666, + "sha256": "9f5f55347362c2fbfdec1b9d01204efaba5c433900d5d0508fd962116f63e1f3" + }, + { + "state": "la", + "enacted_bills": 3720, + "bytes": 22613424, + "sha256": "12456080d3b1e782fea78dc16adc5124ae17e3c231548799b564b0e2cda31ed1" + }, + { + "state": "ma", + "enacted_bills": 1649, + "bytes": 5249208, + "sha256": "e3e8ac45078b85c37cd564b36bc2fc75378fbc2ffe701dce3e337cd336359da6" + }, + { + "state": "md", + "enacted_bills": 7861, + "bytes": 30738288, + "sha256": "1f989383dfd6251ae225c4c47d4ca0aeaefd199cf37946a2f0ddf3293cbf95a4" + }, + { + "state": "me", + "enacted_bills": 3583, + "bytes": 18202991, + "sha256": "bdbc9dbbc6838aebb5c6bc2b9ac00ff985bf6d7f49c39a1b759c6e8e42ed7778" + }, + { + "state": "mi", + "enacted_bills": 2208, + "bytes": 15445804, + "sha256": "840c42067b2ea6ed087601d75937f0fb084f9b7a8b44bc8b91edeb9fad5ed662" + }, + { + "state": "mn", + "enacted_bills": 1019, + "bytes": 5556359, + "sha256": "e84282735787e33b39d37d3ca118948f2d1396cca5d1a79f0606549318672e48" + }, + { + "state": "mo", + "enacted_bills": 296, + "bytes": 2326257, + "sha256": "9abfcad5b6e5e76191fead23631e553921806bc2662789fa9511a509a3f91e42" + }, + { + "state": "ms", + "enacted_bills": 3289, + "bytes": 15157958, + "sha256": "0bc848e3c3a2d82b6e584c77d5ba56bd0153abbbd24a98bf8776a24b3846c7e9" + }, + { + "state": "mt", + "enacted_bills": 3049, + "bytes": 27702700, + "sha256": "cc3c9f06f8190e60b4c9d835461079528b153872b4a8467137e08be24a69e311" + }, + { + "state": "nc", + "enacted_bills": 7091, + "bytes": 37444435, + "sha256": "4d16fbbcc966c9ee0560243e82103f60b92f7ba02b442f9e6c60ac103810c15a" + }, + { + "state": "nd", + "enacted_bills": 2707, + "bytes": 13040135, + "sha256": "7b0e58cac69e80d0db1dcc6a3c076f5dafd96f84c7a3b3c71d545330e2ed804c" + }, + { + "state": "ne", + "enacted_bills": 1663, + "bytes": 7156920, + "sha256": "eac21a1b218643faf8eabc72dbcda2d45e26d16ca8f5cea180b8c5293b3ce7ad" + }, + { + "state": "nh", + "enacted_bills": 2736, + "bytes": 10941758, + "sha256": "1425ac9a0334e83b23703a37bfe44bd27fcff0ce75400539d200421bccf0b394" + }, + { + "state": "nj", + "enacted_bills": 2342, + "bytes": 10579502, + "sha256": "294fd7995714d605bf8f5d77587601ca0d81846bce1a3cee819df4895db46241" + }, + { + "state": "nm", + "enacted_bills": 1992, + "bytes": 4199566, + "sha256": "79d4a4c7c22d92897ee01b5f4fb507aab15d7ac850e13e56a15a6cb3142af57d" + }, + { + "state": "nv", + "enacted_bills": 2825, + "bytes": 12011662, + "sha256": "dfc28a73df87c6be83b3f85900fc626f87c414ca0d7d4bff3b64ec507ab07e7a" + }, + { + "state": "ny", + "enacted_bills": 5726, + "bytes": 18726500, + "sha256": "ba60a588e96e6182b30bf9b35ce5b4f2e6e82ba456ec2768a5fd501ce85ecd3f" + }, + { + "state": "oh", + "enacted_bills": 410, + "bytes": 4679980, + "sha256": "b66ebf2abc1c22110e435218580d1efaad7c9870b99b9b98cb3a43711c667650" + }, + { + "state": "ok", + "enacted_bills": 5684, + "bytes": 29457823, + "sha256": "b7d9659f7f594339f112e3f63c8b4f4b39677a272b4db488eab2b6b9c66893b2" + }, + { + "state": "or", + "enacted_bills": 3916, + "bytes": 23681686, + "sha256": "5304aae9ab2efcb9b59d511c0775cfacd741c3a8c07e6549b51de55d686a6deb" + }, + { + "state": "pa", + "enacted_bills": 1196, + "bytes": 10351077, + "sha256": "bc4f6e4e575c4b1e2765246ad72ba1257943a0e8c1959fb6bd4b547206cf0746" + }, + { + "state": "ri", + "enacted_bills": 4510, + "bytes": 14436359, + "sha256": "11d75896eb05fc59ec76f419eed5f2fcfd8d55792294e87906fb42b4def990a4" + }, + { + "state": "sc", + "enacted_bills": 1216, + "bytes": 7339106, + "sha256": "d8c739024f2b6134313e12cd581884d86f7102d3a08c32add9e3da62990c49fa" + }, + { + "state": "sd", + "enacted_bills": 2133, + "bytes": 10284537, + "sha256": "5ea8e05152e10c26f07399cae412210d5380b21a0d5c3813def7de5d366ca966" + }, + { + "state": "tn", + "enacted_bills": 22173, + "bytes": 89753302, + "sha256": "107f2767f873f1ead68ff136650295282d1e8772868ae01b673d211bd84a1f69" + }, + { + "state": "tx", + "enacted_bills": 5320, + "bytes": 40644997, + "sha256": "22e0458e76fdce12c9a1a6fb3724029fc4a80d61e9833b0972068efd260822cb" + }, + { + "state": "ut", + "enacted_bills": 5280, + "bytes": 32393795, + "sha256": "d7787c2a0709e6ed95b9bcce872530da0d451168988875338f52991f729dc2ec" + }, + { + "state": "va", + "enacted_bills": 7679, + "bytes": 52807819, + "sha256": "4d22f67290c9427c16f37be1d7fe3b7a7221ee96d9c69237dd5cac4643979c2d" + }, + { + "state": "vt", + "enacted_bills": 934, + "bytes": 7316735, + "sha256": "a0d3b9651f0b338b78a4810812e5740bc640e9a150858ff8436053a73d9d035d" + }, + { + "state": "wa", + "enacted_bills": 3665, + "bytes": 24146650, + "sha256": "39af6bed659877113880bdd7a1ffcfe2d20d0413b222d5b7e4e4c01cc2c6d441" + }, + { + "state": "wi", + "enacted_bills": 1120, + "bytes": 7873782, + "sha256": "745ee3ef88c2c951df9371440cee09d65120c00e9ca64568d4ea8ccf0a58dd7f" + }, + { + "state": "wv", + "enacted_bills": 2628, + "bytes": 16218371, + "sha256": "838ff303d15fddf3cb0f280e61b6fdb5fa33d07953837ca53c1db862d2010da9" + }, + { + "state": "wy", + "enacted_bills": 1559, + "bytes": 8719955, + "sha256": "35dd81646b65693456f73c92c23782059a7bec939a293d5ad68252353c180632" + } + ] +} \ No newline at end of file diff --git a/data/sources/sources.yaml b/data/sources/sources.yaml index 9c5a2208f1..d759ca9169 100644 --- a/data/sources/sources.yaml +++ b/data/sources/sources.yaml @@ -85,6 +85,24 @@ sources: county→district edges. Also seeds the state-legislative district nodes (which have no ACS demographics — geometry only). + - id: openstates + name: OpenStates bulk snapshot + url: https://data.openstates.org/ + type: dataset + authority_level: authoritative-community + requires_api_key: false + refresh_frequency: daily + feeds: [Bill] + raw: [depot:/exports/legislation/2026-07-01/enacted_{st}_2026-07-01.jsonl] + notes: > + Enacted state legislation, all 50 states, vintage 2026-07-01 — + 197,131 records (bills AND joint/concurrent resolutions; the + classification field records which; e.g. TN is 8,785 bills + + 13,388 joint resolutions). Raw JSONL lives on the depot; the + per-state SHA-256 manifest is committed at + data/sources/legislation_manifest_2026-07-01.json. Aggregated + from official legislature sites; confidence: reported. + # Prepared but not yet ingested — see docs/roadmap.md planned: - id: govinfo diff --git a/schemas/bill.schema.json b/schemas/bill.schema.json new file mode 100644 index 0000000000..bef0a447ef --- /dev/null +++ b/schemas/bill.schema.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Bill", + "description": "A legislative bill: a proposed change to law, with its version chain, action timeline, sponsors, and votes. The legislative-process corpus (OpenStates/OCD source), parallel to the LegalText statute corpus.", + "type": "object", + "required": [ + "type", + "title", + "jurisdiction", + "legislature", + "session", + "identifier", + "citation", + "status", + "source", + "source_identifier", + "retrieved_at", + "confidence", + "tags" + ], + "properties": { + "type": { "type": "string", "enum": ["Bill"] }, + "title": { "type": "string" }, + "description": { "type": "string" }, + "jurisdiction": { "type": "string" }, + "legislature": { "type": "string" }, + "session": { "type": "string" }, + "identifier": { "type": "string" }, + "citation": { "type": "string" }, + "classification": { "type": "array" }, + "subjects": { "type": "array" }, + "status": { "type": "string", "enum": ["enacted", "passed", "failed", "vetoed", "pending", "withdrawn"] }, + "chapter_law": { "type": "string" }, + "primary_sponsors": { "type": "array" }, + "version_count": { "type": "integer" }, + "action_count": { "type": "integer" }, + "vote_count": { "type": "integer" }, + "first_action": { "type": "string" }, + "last_action": { "type": "string" }, + "source": { "type": "string", "enum": ["official", "openstates"] }, + "source_identifier": { "type": "string" }, + "source_url": { "type": "string" }, + "source_hash": { "type": "string" }, + "vintage": { "type": "string" }, + "source_snapshot": { "type": "string" }, + "retrieved_at": { "type": "string" }, + "confidence": { "type": "string", "enum": ["official", "reported", "inferred", "unverified"] }, + "tags": { "type": "array" } + } +} diff --git a/scripts/ingest_legislation.py b/scripts/ingest_legislation.py new file mode 100644 index 0000000000..e0f9df5433 --- /dev/null +++ b/scripts/ingest_legislation.py @@ -0,0 +1,278 @@ +#!/usr/bin/env python3 +"""Ingest OpenStates bills into Republic OS legislation files. + +The legislative-process corpus, parallel to the LegalText statute corpus: +a bill is a proposed change to law. Each bill node records its version chain +(the diff), action timeline (the PR events), sponsors (authors), and votes +(the merge decisions). Source is the OpenStates/OpenCivicData bulk dump. + +Mechanical: raw JSONL export (one bill per line) -> OKF Bill Markdown nodes. +No summaries or interpretation. Deterministic: same input -> byte-identical output. + + scripts/ingest_legislation.py --state fl --input data/legislation/fl.jsonl +""" + +import argparse +import hashlib +import json +import re +import shutil +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +OUT_ROOT = REPO / "legislation" / "us" / "states" + +RETRIEVED_AT = "2026-07-06" +DEFAULT_VINTAGE = "2026-07-01" + + +def snapshot_url(vintage): + return f"https://data.openstates.org/daily/{vintage}/public.pgdump" + + +# Values that appear in the OpenStates `subject` field but are scraper artifacts, +# not real topic tags (e.g. FL stores the literal link text "Subject Index"). +JUNK_SUBJECTS = {"subject index"} + +DEPOT_SRC = "/Volumes/depot/exports/legislation" + +# Official-name exceptions; the fallback is "{State} Legislature". +LEGISLATURE = { + "co": "Colorado General Assembly", "ct": "Connecticut General Assembly", + "de": "Delaware General Assembly", "ga": "Georgia General Assembly", + "il": "Illinois General Assembly", "in": "Indiana General Assembly", + "ia": "Iowa General Assembly", "ky": "Kentucky General Assembly", + "md": "Maryland General Assembly", "mo": "Missouri General Assembly", + "nc": "North Carolina General Assembly", "oh": "Ohio General Assembly", + "pa": "Pennsylvania General Assembly", "ri": "Rhode Island General Assembly", + "sc": "South Carolina General Assembly", "tn": "Tennessee General Assembly", + "vt": "Vermont General Assembly", "va": "Virginia General Assembly", + "ma": "Massachusetts General Court", "nh": "New Hampshire General Court", + "nd": "North Dakota Legislative Assembly", "or": "Oregon Legislative Assembly", +} +STATE_NAME = { + "al": "Alabama", "ak": "Alaska", "az": "Arizona", "ar": "Arkansas", + "ca": "California", "co": "Colorado", "ct": "Connecticut", "de": "Delaware", + "fl": "Florida", "ga": "Georgia", "hi": "Hawaii", "id": "Idaho", + "il": "Illinois", "in": "Indiana", "ia": "Iowa", "ks": "Kansas", + "ky": "Kentucky", "la": "Louisiana", "me": "Maine", "md": "Maryland", + "ma": "Massachusetts", "mi": "Michigan", "mn": "Minnesota", "ms": "Mississippi", + "mo": "Missouri", "mt": "Montana", "ne": "Nebraska", "nv": "Nevada", + "nh": "New Hampshire", "nj": "New Jersey", "nm": "New Mexico", "ny": "New York", + "nc": "North Carolina", "nd": "North Dakota", "oh": "Ohio", "ok": "Oklahoma", + "or": "Oregon", "pa": "Pennsylvania", "ri": "Rhode Island", "sc": "South Carolina", + "sd": "South Dakota", "tn": "Tennessee", "tx": "Texas", "ut": "Utah", + "vt": "Vermont", "va": "Virginia", "wa": "Washington", "wv": "West Virginia", + "wi": "Wisconsin", "wy": "Wyoming", +} + + +def yval(value): + if value is None: + return '""' + if isinstance(value, bool): + return "true" if value else "false" + if isinstance(value, int): + return str(value) + if isinstance(value, list): + return "[" + ", ".join(yval(v) for v in value) + "]" + return json.dumps(str(value), ensure_ascii=False) + + +def slugify(value): + value = (value or "").lower() + value = re.sub(r"[^a-z0-9]+", "-", value).strip("-") + return re.sub(r"-{2,}", "-", value) or "unnamed" + + +def sha256_text(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def version_rank(note): + """Order a bill's versions into a commit chain from its OpenStates note.""" + tok = (note or "").split()[-1].lower() if note else "" + if tok in ("filed", "introduced", "original"): + return (0, 0) + m = re.fullmatch(r"c(\d+)", tok) + if m: + return (1, int(m.group(1))) + m = re.fullmatch(r"e(\d+)", tok) + if m: + return (2, int(m.group(1))) + if tok in ("er", "enrolled"): + return (3, 0) + return (1, 50) # unknown mid-stage: sort stably after filed + + +STAGE_LABEL = {0: "filed", 1: "committee substitute", 2: "engrossed", 3: "enrolled"} + + +def clean(s): + return re.sub(r"\s+", " ", (s or "").strip()) + + +def primary_person_sponsors(sponsors): + return [clean(s["name"]) for s in sponsors + if s.get("primary") and s.get("entity_type") == "person"] + + +def chapter_law(actions): + for a in actions: + m = re.search(r"Chapter No\.\s*([0-9][0-9-]*)", a.get("description") or "") + if m: + return m.group(1) + return None + + +def render_bill(rec, state, vintage): + ident = clean(rec["identifier"]) + session = clean(rec["session"]) + title = clean(rec.get("title")) or ident + sponsors = rec.get("sponsors") or [] + actions = rec.get("actions") or [] + versions = sorted(rec.get("versions") or [], key=lambda v: (version_rank(v.get("note")), v.get("note") or "")) + votes = rec.get("votes") or [] + subjects = [clean(x) for x in (rec.get("subjects") or []) + if clean(x) and clean(x).lower() not in JUNK_SUBJECTS] + prim = primary_person_sponsors(sponsors) + chap = chapter_law(actions) + dates = [a.get("date") for a in actions if a.get("date")] + citation = f"{STATE_NAME.get(state, state.upper())} {ident} ({session})" + + fm = { + "type": "Bill", + "title": title, + "description": clean(rec.get("abstract")) or title, + "jurisdiction": f"us/states/{state}", + "legislature": LEGISLATURE.get(state, f"{STATE_NAME.get(state, state.upper())} Legislature"), + "session": session, + "identifier": ident, + "citation": citation, + "classification": rec.get("classification") or [], + "subjects": subjects, + "status": "enacted", + "chapter_law": chap, + "primary_sponsors": prim, + "version_count": len(versions), + "action_count": len(actions), + "vote_count": len(votes), + "first_action": min(dates) if dates else None, + "last_action": max(dates) if dates else None, + "source": "openstates", + "source_identifier": rec["bill_id"], + "source_url": (rec.get("sources") or [None])[0], + "source_hash": sha256_text(json.dumps(rec, sort_keys=True, ensure_ascii=False)), + "vintage": vintage, + "source_snapshot": snapshot_url(vintage), + "retrieved_at": RETRIEVED_AT, + "confidence": "reported", + "tags": ["legislation", "bill", f"us-{state}"], + } + # drop null-valued optional keys for clean, deterministic frontmatter + fm = {k: v for k, v in fm.items() if v is not None} + + out = ["---"] + for k, v in fm.items(): + out.append(f"{k}: {yval(v)}") + out.append("---") + out.append("") + out.append(f"# {citation} — {title}") + out.append("") + if fm["description"] != title: + out.append(fm["description"]) + out.append("") + if chap: + out.append(f"**Enacted** — Laws of {STATE_NAME.get(state, state.upper())}, Chapter {chap}.") + out.append("") + + out.append("## Version chain") + out.append("") + out.append("The bill's text revisions, in order — the diff chain from filing to enrollment.") + out.append("") + for i, v in enumerate(versions): + stage = STAGE_LABEL[version_rank(v.get("note"))[0]] + note = clean(v.get("note")) or stage + link = f" — [source]({v['url']})" if v.get("url") else "" + out.append(f"{i+1}. **{note}** ({stage}){link}") + out.append("") + + if votes: + out.append("## Votes") + out.append("") + for v in votes: + y, n = v.get("yes"), v.get("no") + tally = f"{y}–{n}" if y is not None and n is not None else (v.get("result") or "") + ch = clean(v.get("chamber")) or "" + motion = clean(v.get("motion")) or "vote" + out.append(f"- {motion} — **{tally}** ({v.get('result')}) {('· ' + ch) if ch else ''}".rstrip()) + out.append("") + + if sponsors: + out.append("## Sponsors") + out.append("") + for s in sponsors: + role = "primary" if s.get("primary") else (clean(s.get("classification")) or "cosponsor") + out.append(f"- {clean(s['name'])} — {role} ({clean(s.get('entity_type'))})") + out.append("") + + out.append("## Timeline") + out.append("") + out.append("The legislative action history — every referral, reading, and vote.") + out.append("") + for a in actions: + cls = ", ".join(a.get("classification") or []) + tag = f" `{cls}`" if cls else "" + out.append(f"- **{a.get('date') or '—'}** {clean(a.get('description'))}{tag}") + out.append("") + + out.append("## Source") + out.append("") + out.append(f"OpenStates / OpenCivicData bulk snapshot [{vintage}]({snapshot_url(vintage)}); " + f"origin `{rec['bill_id']}`. Confidence: reported (aggregated from official {STATE_NAME.get(state, state.upper())} legislature records).") + out.append("") + + path = OUT_ROOT / state / slugify(session) / f"{slugify(ident)}.md" + return path, "\n".join(out) + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--state", required=True) + ap.add_argument("--input", default=None, + help="raw JSONL; default: depot export for --vintage") + ap.add_argument("--vintage", default=DEFAULT_VINTAGE, + help="OpenStates bulk-snapshot date, e.g. 2026-07-01") + args = ap.parse_args() + state = args.state.lower() + src = args.input or f"{DEPOT_SRC}/{args.vintage}/enacted_{state}_{args.vintage}.jsonl" + + dest = OUT_ROOT / state + if dest.exists(): + shutil.rmtree(dest) + + n = 0 + used = {} + with open(src, encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + rec = json.loads(line) + # Record ALL enacted legislation honestly — bills AND joint/concurrent + # resolutions. The classification frontmatter says which; filtering + # (e.g. dropping TN's ceremonial resolutions) is a view's job, not the + # mirror's. + path, content = render_bill(rec, state, args.vintage) + occ = used.get(path, 0) + 1 + used[path] = occ + if occ > 1: + path = path.with_name(f"{path.stem}-{occ}.md") + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content + "\n", encoding="utf-8") + n += 1 + print(f"{state}: wrote {n} enacted-legislation nodes to {dest.relative_to(REPO)}") + + +if __name__ == "__main__": + main() diff --git a/scripts/validate.py b/scripts/validate.py index 01d8de8406..b404ad4a75 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -22,7 +22,7 @@ import sys from pathlib import Path REPO = Path(__file__).resolve().parent.parent -TREES = [REPO / "data" / "jurisdictions", REPO / "legal"] +TREES = [REPO / "data" / "jurisdictions", REPO / "legal", REPO / "legislation"] SCHEMAS = REPO / "schemas" MAX_REPORT = 25