Legislation axis: Bill schema, OpenStates ingest, validator + provenance
The legislative-process corpus, parallel to the U.S. Code statute corpus. schemas/bill.schema.json + scripts/ingest_legislation.py (deterministic, stdlib-only, reads per-state JSONL from the depot); validate.py gains legislation/ as a third tree; Makefile 'legislation' target. Raw JSONL stays off-repo on the depot (gitignored); the per-state SHA-256 manifest is the committed provenance anchor. Vintage 2026-07-01. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Fabio
@@ -7,3 +7,8 @@
|
||||
# "raw-first, in-repo" discipline turns on at v1.0 — see README.
|
||||
data/legal/raw/**/*.zip
|
||||
data/legal/raw/**/*.xml
|
||||
# Legislation (OpenStates): per-state enacted exports are the raw bytes
|
||||
# (~1.1GB/vintage), kept on the depot at /depot/exports/legislation/<vintage>/;
|
||||
# ingest reads them from there. Only the manifest (committed under data/sources/)
|
||||
# + the rendered Bill nodes get committed.
|
||||
data/legislation/
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Republic OS — repeatable pipeline commands.
|
||||
# The build is deterministic: `make build` twice yields a byte-identical tree.
|
||||
|
||||
.PHONY: build validate board legal-us-code authority check
|
||||
.PHONY: build validate board legal-us-code authority legislation check
|
||||
|
||||
build: ## Regenerate the entity tree from raw exports in data/
|
||||
python3 scripts/build.py
|
||||
@@ -27,4 +27,10 @@ legal-us-code: ## Ingest the pinned U.S. Code titles (override with TITLES="…"
|
||||
authority: ## Extract the § -> executive-office authority edges (a committed input)
|
||||
python3 scripts/extract_authority.py
|
||||
|
||||
LEG_VINTAGE ?= 2026-07-01
|
||||
LEG_STATES ?= ak al ar az ca co ct de fl ga hi ia id il in ks ky la ma md me mi mn mo ms mt nc nd ne nh nj nm nv ny oh ok or pa ri sc sd tn tx ut va vt wa wi wv wy
|
||||
|
||||
legislation: ## Ingest enacted state legislation from the depot export (override LEG_STATES="fl ca")
|
||||
@for s in $(LEG_STATES); do python3 scripts/ingest_legislation.py --state $$s --vintage $(LEG_VINTAGE); done
|
||||
|
||||
check: build validate ## Build, then validate — the full gate
|
||||
|
||||
+310
@@ -0,0 +1,310 @@
|
||||
{
|
||||
"vintage": "2026-07-01",
|
||||
"source": "OpenStates bulk snapshot (data.openstates.org)",
|
||||
"source_url": "https://data.openstates.org/daily/2026-07-01/public.pgdump",
|
||||
"generated_at": "2026-07-06T20:25:28Z",
|
||||
"total_bills": 197131,
|
||||
"total_states": 50,
|
||||
"states": [
|
||||
{
|
||||
"state": "ak",
|
||||
"enacted_bills": 472,
|
||||
"bytes": 5056780,
|
||||
"sha256": "e926ef824c9a19ca48ae72b73f5fe1cb265a58de44c4fd3342057a8cfaa92e1c"
|
||||
},
|
||||
{
|
||||
"state": "al",
|
||||
"enacted_bills": 4042,
|
||||
"bytes": 15462516,
|
||||
"sha256": "4a08be5b7c66a6bc9b7ff18607605329ee4e1602d0568ea26a7b751c099f855c"
|
||||
},
|
||||
{
|
||||
"state": "ar",
|
||||
"enacted_bills": 5166,
|
||||
"bytes": 20988670,
|
||||
"sha256": "182c19c260d6665c1a297b7818ddd270a32ee59cb25a8c6848a336ff1edb9618"
|
||||
},
|
||||
{
|
||||
"state": "az",
|
||||
"enacted_bills": 2927,
|
||||
"bytes": 9771351,
|
||||
"sha256": "e8b30b05c84c3012e382d4e2c3e43e32d0b9b40a10378aa69c8b2a95a4bcbae2"
|
||||
},
|
||||
{
|
||||
"state": "ca",
|
||||
"enacted_bills": 19201,
|
||||
"bytes": 154829262,
|
||||
"sha256": "029990f77124ce5ee80d638255f5ac93caed112a0a17a05a1e021726a02880da"
|
||||
},
|
||||
{
|
||||
"state": "co",
|
||||
"enacted_bills": 4563,
|
||||
"bytes": 29263453,
|
||||
"sha256": "9c6173452331c2b9cc0a3a920bbe917eedce7012cd2d65132ed5a7e66bb4a471"
|
||||
},
|
||||
{
|
||||
"state": "ct",
|
||||
"enacted_bills": 1671,
|
||||
"bytes": 9173617,
|
||||
"sha256": "394a8555f0e3aa7a6c68b97d6e6a1269407822cfc5d5bccaa2179b30c5c3ad90"
|
||||
},
|
||||
{
|
||||
"state": "de",
|
||||
"enacted_bills": 2138,
|
||||
"bytes": 9675510,
|
||||
"sha256": "65758ad762a7e6a23e0012ef570468022dea1d4012d36e6126ff0a49b7f278d5"
|
||||
},
|
||||
{
|
||||
"state": "fl",
|
||||
"enacted_bills": 2186,
|
||||
"bytes": 14587374,
|
||||
"sha256": "471f47358310803e56c288d6c20d8bb975c642bc5583f6d0507ac08ebf2e6b17"
|
||||
},
|
||||
{
|
||||
"state": "ga",
|
||||
"enacted_bills": 4933,
|
||||
"bytes": 20269678,
|
||||
"sha256": "ea6f721f7b480d9363383adf350c9ffb29c7bff27aef78e89dcd5c465afe3e28"
|
||||
},
|
||||
{
|
||||
"state": "hi",
|
||||
"enacted_bills": 1765,
|
||||
"bytes": 19610329,
|
||||
"sha256": "d2c70ffc73283a1ddcdf7bf53b87f7b25bc3b33ea7a9eb0123ac5a03a7ab8392"
|
||||
},
|
||||
{
|
||||
"state": "ia",
|
||||
"enacted_bills": 1680,
|
||||
"bytes": 7130701,
|
||||
"sha256": "0a1640a6fd76b508d421e2df8e4c30611e236266be013405c863894329e5c637"
|
||||
},
|
||||
{
|
||||
"state": "id",
|
||||
"enacted_bills": 3358,
|
||||
"bytes": 15560374,
|
||||
"sha256": "d63a6ca3a067758f5ffd87440ae75850ad0463e163b85e2cd62644e2f6c0926b"
|
||||
},
|
||||
{
|
||||
"state": "il",
|
||||
"enacted_bills": 15148,
|
||||
"bytes": 129476874,
|
||||
"sha256": "d905d3d0b7809a91bb863de9bfbc310df1d7136efc818aee57f1fca06f4d9752"
|
||||
},
|
||||
{
|
||||
"state": "in",
|
||||
"enacted_bills": 1861,
|
||||
"bytes": 13458201,
|
||||
"sha256": "96279ef4e11d8503a4a7f90c522005aa31b49f966330e0614dbd21c8bb309cd8"
|
||||
},
|
||||
{
|
||||
"state": "ks",
|
||||
"enacted_bills": 927,
|
||||
"bytes": 4320913,
|
||||
"sha256": "a7c7d6b715c09d3efcd4ee03ef6c841050a4eeb8b7f9705c6a5a7c6d27cfda72"
|
||||
},
|
||||
{
|
||||
"state": "ky",
|
||||
"enacted_bills": 1914,
|
||||
"bytes": 10759666,
|
||||
"sha256": "9f5f55347362c2fbfdec1b9d01204efaba5c433900d5d0508fd962116f63e1f3"
|
||||
},
|
||||
{
|
||||
"state": "la",
|
||||
"enacted_bills": 3720,
|
||||
"bytes": 22613424,
|
||||
"sha256": "12456080d3b1e782fea78dc16adc5124ae17e3c231548799b564b0e2cda31ed1"
|
||||
},
|
||||
{
|
||||
"state": "ma",
|
||||
"enacted_bills": 1649,
|
||||
"bytes": 5249208,
|
||||
"sha256": "e3e8ac45078b85c37cd564b36bc2fc75378fbc2ffe701dce3e337cd336359da6"
|
||||
},
|
||||
{
|
||||
"state": "md",
|
||||
"enacted_bills": 7861,
|
||||
"bytes": 30738288,
|
||||
"sha256": "1f989383dfd6251ae225c4c47d4ca0aeaefd199cf37946a2f0ddf3293cbf95a4"
|
||||
},
|
||||
{
|
||||
"state": "me",
|
||||
"enacted_bills": 3583,
|
||||
"bytes": 18202991,
|
||||
"sha256": "bdbc9dbbc6838aebb5c6bc2b9ac00ff985bf6d7f49c39a1b759c6e8e42ed7778"
|
||||
},
|
||||
{
|
||||
"state": "mi",
|
||||
"enacted_bills": 2208,
|
||||
"bytes": 15445804,
|
||||
"sha256": "840c42067b2ea6ed087601d75937f0fb084f9b7a8b44bc8b91edeb9fad5ed662"
|
||||
},
|
||||
{
|
||||
"state": "mn",
|
||||
"enacted_bills": 1019,
|
||||
"bytes": 5556359,
|
||||
"sha256": "e84282735787e33b39d37d3ca118948f2d1396cca5d1a79f0606549318672e48"
|
||||
},
|
||||
{
|
||||
"state": "mo",
|
||||
"enacted_bills": 296,
|
||||
"bytes": 2326257,
|
||||
"sha256": "9abfcad5b6e5e76191fead23631e553921806bc2662789fa9511a509a3f91e42"
|
||||
},
|
||||
{
|
||||
"state": "ms",
|
||||
"enacted_bills": 3289,
|
||||
"bytes": 15157958,
|
||||
"sha256": "0bc848e3c3a2d82b6e584c77d5ba56bd0153abbbd24a98bf8776a24b3846c7e9"
|
||||
},
|
||||
{
|
||||
"state": "mt",
|
||||
"enacted_bills": 3049,
|
||||
"bytes": 27702700,
|
||||
"sha256": "cc3c9f06f8190e60b4c9d835461079528b153872b4a8467137e08be24a69e311"
|
||||
},
|
||||
{
|
||||
"state": "nc",
|
||||
"enacted_bills": 7091,
|
||||
"bytes": 37444435,
|
||||
"sha256": "4d16fbbcc966c9ee0560243e82103f60b92f7ba02b442f9e6c60ac103810c15a"
|
||||
},
|
||||
{
|
||||
"state": "nd",
|
||||
"enacted_bills": 2707,
|
||||
"bytes": 13040135,
|
||||
"sha256": "7b0e58cac69e80d0db1dcc6a3c076f5dafd96f84c7a3b3c71d545330e2ed804c"
|
||||
},
|
||||
{
|
||||
"state": "ne",
|
||||
"enacted_bills": 1663,
|
||||
"bytes": 7156920,
|
||||
"sha256": "eac21a1b218643faf8eabc72dbcda2d45e26d16ca8f5cea180b8c5293b3ce7ad"
|
||||
},
|
||||
{
|
||||
"state": "nh",
|
||||
"enacted_bills": 2736,
|
||||
"bytes": 10941758,
|
||||
"sha256": "1425ac9a0334e83b23703a37bfe44bd27fcff0ce75400539d200421bccf0b394"
|
||||
},
|
||||
{
|
||||
"state": "nj",
|
||||
"enacted_bills": 2342,
|
||||
"bytes": 10579502,
|
||||
"sha256": "294fd7995714d605bf8f5d77587601ca0d81846bce1a3cee819df4895db46241"
|
||||
},
|
||||
{
|
||||
"state": "nm",
|
||||
"enacted_bills": 1992,
|
||||
"bytes": 4199566,
|
||||
"sha256": "79d4a4c7c22d92897ee01b5f4fb507aab15d7ac850e13e56a15a6cb3142af57d"
|
||||
},
|
||||
{
|
||||
"state": "nv",
|
||||
"enacted_bills": 2825,
|
||||
"bytes": 12011662,
|
||||
"sha256": "dfc28a73df87c6be83b3f85900fc626f87c414ca0d7d4bff3b64ec507ab07e7a"
|
||||
},
|
||||
{
|
||||
"state": "ny",
|
||||
"enacted_bills": 5726,
|
||||
"bytes": 18726500,
|
||||
"sha256": "ba60a588e96e6182b30bf9b35ce5b4f2e6e82ba456ec2768a5fd501ce85ecd3f"
|
||||
},
|
||||
{
|
||||
"state": "oh",
|
||||
"enacted_bills": 410,
|
||||
"bytes": 4679980,
|
||||
"sha256": "b66ebf2abc1c22110e435218580d1efaad7c9870b99b9b98cb3a43711c667650"
|
||||
},
|
||||
{
|
||||
"state": "ok",
|
||||
"enacted_bills": 5684,
|
||||
"bytes": 29457823,
|
||||
"sha256": "b7d9659f7f594339f112e3f63c8b4f4b39677a272b4db488eab2b6b9c66893b2"
|
||||
},
|
||||
{
|
||||
"state": "or",
|
||||
"enacted_bills": 3916,
|
||||
"bytes": 23681686,
|
||||
"sha256": "5304aae9ab2efcb9b59d511c0775cfacd741c3a8c07e6549b51de55d686a6deb"
|
||||
},
|
||||
{
|
||||
"state": "pa",
|
||||
"enacted_bills": 1196,
|
||||
"bytes": 10351077,
|
||||
"sha256": "bc4f6e4e575c4b1e2765246ad72ba1257943a0e8c1959fb6bd4b547206cf0746"
|
||||
},
|
||||
{
|
||||
"state": "ri",
|
||||
"enacted_bills": 4510,
|
||||
"bytes": 14436359,
|
||||
"sha256": "11d75896eb05fc59ec76f419eed5f2fcfd8d55792294e87906fb42b4def990a4"
|
||||
},
|
||||
{
|
||||
"state": "sc",
|
||||
"enacted_bills": 1216,
|
||||
"bytes": 7339106,
|
||||
"sha256": "d8c739024f2b6134313e12cd581884d86f7102d3a08c32add9e3da62990c49fa"
|
||||
},
|
||||
{
|
||||
"state": "sd",
|
||||
"enacted_bills": 2133,
|
||||
"bytes": 10284537,
|
||||
"sha256": "5ea8e05152e10c26f07399cae412210d5380b21a0d5c3813def7de5d366ca966"
|
||||
},
|
||||
{
|
||||
"state": "tn",
|
||||
"enacted_bills": 22173,
|
||||
"bytes": 89753302,
|
||||
"sha256": "107f2767f873f1ead68ff136650295282d1e8772868ae01b673d211bd84a1f69"
|
||||
},
|
||||
{
|
||||
"state": "tx",
|
||||
"enacted_bills": 5320,
|
||||
"bytes": 40644997,
|
||||
"sha256": "22e0458e76fdce12c9a1a6fb3724029fc4a80d61e9833b0972068efd260822cb"
|
||||
},
|
||||
{
|
||||
"state": "ut",
|
||||
"enacted_bills": 5280,
|
||||
"bytes": 32393795,
|
||||
"sha256": "d7787c2a0709e6ed95b9bcce872530da0d451168988875338f52991f729dc2ec"
|
||||
},
|
||||
{
|
||||
"state": "va",
|
||||
"enacted_bills": 7679,
|
||||
"bytes": 52807819,
|
||||
"sha256": "4d22f67290c9427c16f37be1d7fe3b7a7221ee96d9c69237dd5cac4643979c2d"
|
||||
},
|
||||
{
|
||||
"state": "vt",
|
||||
"enacted_bills": 934,
|
||||
"bytes": 7316735,
|
||||
"sha256": "a0d3b9651f0b338b78a4810812e5740bc640e9a150858ff8436053a73d9d035d"
|
||||
},
|
||||
{
|
||||
"state": "wa",
|
||||
"enacted_bills": 3665,
|
||||
"bytes": 24146650,
|
||||
"sha256": "39af6bed659877113880bdd7a1ffcfe2d20d0413b222d5b7e4e4c01cc2c6d441"
|
||||
},
|
||||
{
|
||||
"state": "wi",
|
||||
"enacted_bills": 1120,
|
||||
"bytes": 7873782,
|
||||
"sha256": "745ee3ef88c2c951df9371440cee09d65120c00e9ca64568d4ea8ccf0a58dd7f"
|
||||
},
|
||||
{
|
||||
"state": "wv",
|
||||
"enacted_bills": 2628,
|
||||
"bytes": 16218371,
|
||||
"sha256": "838ff303d15fddf3cb0f280e61b6fdb5fa33d07953837ca53c1db862d2010da9"
|
||||
},
|
||||
{
|
||||
"state": "wy",
|
||||
"enacted_bills": 1559,
|
||||
"bytes": 8719955,
|
||||
"sha256": "35dd81646b65693456f73c92c23782059a7bec939a293d5ad68252353c180632"
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -85,6 +85,24 @@ sources:
|
||||
county→district edges. Also seeds the state-legislative district nodes
|
||||
(which have no ACS demographics — geometry only).
|
||||
|
||||
- id: openstates
|
||||
name: OpenStates bulk snapshot
|
||||
url: https://data.openstates.org/
|
||||
type: dataset
|
||||
authority_level: authoritative-community
|
||||
requires_api_key: false
|
||||
refresh_frequency: daily
|
||||
feeds: [Bill]
|
||||
raw: [depot:/exports/legislation/2026-07-01/enacted_{st}_2026-07-01.jsonl]
|
||||
notes: >
|
||||
Enacted state legislation, all 50 states, vintage 2026-07-01 —
|
||||
197,131 records (bills AND joint/concurrent resolutions; the
|
||||
classification field records which; e.g. TN is 8,785 bills +
|
||||
13,388 joint resolutions). Raw JSONL lives on the depot; the
|
||||
per-state SHA-256 manifest is committed at
|
||||
data/sources/legislation_manifest_2026-07-01.json. Aggregated
|
||||
from official legislature sites; confidence: reported.
|
||||
|
||||
# Prepared but not yet ingested — see docs/roadmap.md
|
||||
planned:
|
||||
- id: govinfo
|
||||
|
||||
@@ -0,0 +1,50 @@
|
||||
{
|
||||
"$schema": "http://json-schema.org/draft-07/schema#",
|
||||
"title": "Bill",
|
||||
"description": "A legislative bill: a proposed change to law, with its version chain, action timeline, sponsors, and votes. The legislative-process corpus (OpenStates/OCD source), parallel to the LegalText statute corpus.",
|
||||
"type": "object",
|
||||
"required": [
|
||||
"type",
|
||||
"title",
|
||||
"jurisdiction",
|
||||
"legislature",
|
||||
"session",
|
||||
"identifier",
|
||||
"citation",
|
||||
"status",
|
||||
"source",
|
||||
"source_identifier",
|
||||
"retrieved_at",
|
||||
"confidence",
|
||||
"tags"
|
||||
],
|
||||
"properties": {
|
||||
"type": { "type": "string", "enum": ["Bill"] },
|
||||
"title": { "type": "string" },
|
||||
"description": { "type": "string" },
|
||||
"jurisdiction": { "type": "string" },
|
||||
"legislature": { "type": "string" },
|
||||
"session": { "type": "string" },
|
||||
"identifier": { "type": "string" },
|
||||
"citation": { "type": "string" },
|
||||
"classification": { "type": "array" },
|
||||
"subjects": { "type": "array" },
|
||||
"status": { "type": "string", "enum": ["enacted", "passed", "failed", "vetoed", "pending", "withdrawn"] },
|
||||
"chapter_law": { "type": "string" },
|
||||
"primary_sponsors": { "type": "array" },
|
||||
"version_count": { "type": "integer" },
|
||||
"action_count": { "type": "integer" },
|
||||
"vote_count": { "type": "integer" },
|
||||
"first_action": { "type": "string" },
|
||||
"last_action": { "type": "string" },
|
||||
"source": { "type": "string", "enum": ["official", "openstates"] },
|
||||
"source_identifier": { "type": "string" },
|
||||
"source_url": { "type": "string" },
|
||||
"source_hash": { "type": "string" },
|
||||
"vintage": { "type": "string" },
|
||||
"source_snapshot": { "type": "string" },
|
||||
"retrieved_at": { "type": "string" },
|
||||
"confidence": { "type": "string", "enum": ["official", "reported", "inferred", "unverified"] },
|
||||
"tags": { "type": "array" }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,278 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Ingest OpenStates bills into Republic OS legislation files.
|
||||
|
||||
The legislative-process corpus, parallel to the LegalText statute corpus:
|
||||
a bill is a proposed change to law. Each bill node records its version chain
|
||||
(the diff), action timeline (the PR events), sponsors (authors), and votes
|
||||
(the merge decisions). Source is the OpenStates/OpenCivicData bulk dump.
|
||||
|
||||
Mechanical: raw JSONL export (one bill per line) -> OKF Bill Markdown nodes.
|
||||
No summaries or interpretation. Deterministic: same input -> byte-identical output.
|
||||
|
||||
scripts/ingest_legislation.py --state fl --input data/legislation/fl.jsonl
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
import re
|
||||
import shutil
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
OUT_ROOT = REPO / "legislation" / "us" / "states"
|
||||
|
||||
RETRIEVED_AT = "2026-07-06"
|
||||
DEFAULT_VINTAGE = "2026-07-01"
|
||||
|
||||
|
||||
def snapshot_url(vintage):
|
||||
return f"https://data.openstates.org/daily/{vintage}/public.pgdump"
|
||||
|
||||
|
||||
# Values that appear in the OpenStates `subject` field but are scraper artifacts,
|
||||
# not real topic tags (e.g. FL stores the literal link text "Subject Index").
|
||||
JUNK_SUBJECTS = {"subject index"}
|
||||
|
||||
DEPOT_SRC = "/Volumes/depot/exports/legislation"
|
||||
|
||||
# Official-name exceptions; the fallback is "{State} Legislature".
|
||||
LEGISLATURE = {
|
||||
"co": "Colorado General Assembly", "ct": "Connecticut General Assembly",
|
||||
"de": "Delaware General Assembly", "ga": "Georgia General Assembly",
|
||||
"il": "Illinois General Assembly", "in": "Indiana General Assembly",
|
||||
"ia": "Iowa General Assembly", "ky": "Kentucky General Assembly",
|
||||
"md": "Maryland General Assembly", "mo": "Missouri General Assembly",
|
||||
"nc": "North Carolina General Assembly", "oh": "Ohio General Assembly",
|
||||
"pa": "Pennsylvania General Assembly", "ri": "Rhode Island General Assembly",
|
||||
"sc": "South Carolina General Assembly", "tn": "Tennessee General Assembly",
|
||||
"vt": "Vermont General Assembly", "va": "Virginia General Assembly",
|
||||
"ma": "Massachusetts General Court", "nh": "New Hampshire General Court",
|
||||
"nd": "North Dakota Legislative Assembly", "or": "Oregon Legislative Assembly",
|
||||
}
|
||||
STATE_NAME = {
|
||||
"al": "Alabama", "ak": "Alaska", "az": "Arizona", "ar": "Arkansas",
|
||||
"ca": "California", "co": "Colorado", "ct": "Connecticut", "de": "Delaware",
|
||||
"fl": "Florida", "ga": "Georgia", "hi": "Hawaii", "id": "Idaho",
|
||||
"il": "Illinois", "in": "Indiana", "ia": "Iowa", "ks": "Kansas",
|
||||
"ky": "Kentucky", "la": "Louisiana", "me": "Maine", "md": "Maryland",
|
||||
"ma": "Massachusetts", "mi": "Michigan", "mn": "Minnesota", "ms": "Mississippi",
|
||||
"mo": "Missouri", "mt": "Montana", "ne": "Nebraska", "nv": "Nevada",
|
||||
"nh": "New Hampshire", "nj": "New Jersey", "nm": "New Mexico", "ny": "New York",
|
||||
"nc": "North Carolina", "nd": "North Dakota", "oh": "Ohio", "ok": "Oklahoma",
|
||||
"or": "Oregon", "pa": "Pennsylvania", "ri": "Rhode Island", "sc": "South Carolina",
|
||||
"sd": "South Dakota", "tn": "Tennessee", "tx": "Texas", "ut": "Utah",
|
||||
"vt": "Vermont", "va": "Virginia", "wa": "Washington", "wv": "West Virginia",
|
||||
"wi": "Wisconsin", "wy": "Wyoming",
|
||||
}
|
||||
|
||||
|
||||
def yval(value):
|
||||
if value is None:
|
||||
return '""'
|
||||
if isinstance(value, bool):
|
||||
return "true" if value else "false"
|
||||
if isinstance(value, int):
|
||||
return str(value)
|
||||
if isinstance(value, list):
|
||||
return "[" + ", ".join(yval(v) for v in value) + "]"
|
||||
return json.dumps(str(value), ensure_ascii=False)
|
||||
|
||||
|
||||
def slugify(value):
|
||||
value = (value or "").lower()
|
||||
value = re.sub(r"[^a-z0-9]+", "-", value).strip("-")
|
||||
return re.sub(r"-{2,}", "-", value) or "unnamed"
|
||||
|
||||
|
||||
def sha256_text(text):
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def version_rank(note):
|
||||
"""Order a bill's versions into a commit chain from its OpenStates note."""
|
||||
tok = (note or "").split()[-1].lower() if note else ""
|
||||
if tok in ("filed", "introduced", "original"):
|
||||
return (0, 0)
|
||||
m = re.fullmatch(r"c(\d+)", tok)
|
||||
if m:
|
||||
return (1, int(m.group(1)))
|
||||
m = re.fullmatch(r"e(\d+)", tok)
|
||||
if m:
|
||||
return (2, int(m.group(1)))
|
||||
if tok in ("er", "enrolled"):
|
||||
return (3, 0)
|
||||
return (1, 50) # unknown mid-stage: sort stably after filed
|
||||
|
||||
|
||||
STAGE_LABEL = {0: "filed", 1: "committee substitute", 2: "engrossed", 3: "enrolled"}
|
||||
|
||||
|
||||
def clean(s):
|
||||
return re.sub(r"\s+", " ", (s or "").strip())
|
||||
|
||||
|
||||
def primary_person_sponsors(sponsors):
|
||||
return [clean(s["name"]) for s in sponsors
|
||||
if s.get("primary") and s.get("entity_type") == "person"]
|
||||
|
||||
|
||||
def chapter_law(actions):
|
||||
for a in actions:
|
||||
m = re.search(r"Chapter No\.\s*([0-9][0-9-]*)", a.get("description") or "")
|
||||
if m:
|
||||
return m.group(1)
|
||||
return None
|
||||
|
||||
|
||||
def render_bill(rec, state, vintage):
|
||||
ident = clean(rec["identifier"])
|
||||
session = clean(rec["session"])
|
||||
title = clean(rec.get("title")) or ident
|
||||
sponsors = rec.get("sponsors") or []
|
||||
actions = rec.get("actions") or []
|
||||
versions = sorted(rec.get("versions") or [], key=lambda v: (version_rank(v.get("note")), v.get("note") or ""))
|
||||
votes = rec.get("votes") or []
|
||||
subjects = [clean(x) for x in (rec.get("subjects") or [])
|
||||
if clean(x) and clean(x).lower() not in JUNK_SUBJECTS]
|
||||
prim = primary_person_sponsors(sponsors)
|
||||
chap = chapter_law(actions)
|
||||
dates = [a.get("date") for a in actions if a.get("date")]
|
||||
citation = f"{STATE_NAME.get(state, state.upper())} {ident} ({session})"
|
||||
|
||||
fm = {
|
||||
"type": "Bill",
|
||||
"title": title,
|
||||
"description": clean(rec.get("abstract")) or title,
|
||||
"jurisdiction": f"us/states/{state}",
|
||||
"legislature": LEGISLATURE.get(state, f"{STATE_NAME.get(state, state.upper())} Legislature"),
|
||||
"session": session,
|
||||
"identifier": ident,
|
||||
"citation": citation,
|
||||
"classification": rec.get("classification") or [],
|
||||
"subjects": subjects,
|
||||
"status": "enacted",
|
||||
"chapter_law": chap,
|
||||
"primary_sponsors": prim,
|
||||
"version_count": len(versions),
|
||||
"action_count": len(actions),
|
||||
"vote_count": len(votes),
|
||||
"first_action": min(dates) if dates else None,
|
||||
"last_action": max(dates) if dates else None,
|
||||
"source": "openstates",
|
||||
"source_identifier": rec["bill_id"],
|
||||
"source_url": (rec.get("sources") or [None])[0],
|
||||
"source_hash": sha256_text(json.dumps(rec, sort_keys=True, ensure_ascii=False)),
|
||||
"vintage": vintage,
|
||||
"source_snapshot": snapshot_url(vintage),
|
||||
"retrieved_at": RETRIEVED_AT,
|
||||
"confidence": "reported",
|
||||
"tags": ["legislation", "bill", f"us-{state}"],
|
||||
}
|
||||
# drop null-valued optional keys for clean, deterministic frontmatter
|
||||
fm = {k: v for k, v in fm.items() if v is not None}
|
||||
|
||||
out = ["---"]
|
||||
for k, v in fm.items():
|
||||
out.append(f"{k}: {yval(v)}")
|
||||
out.append("---")
|
||||
out.append("")
|
||||
out.append(f"# {citation} — {title}")
|
||||
out.append("")
|
||||
if fm["description"] != title:
|
||||
out.append(fm["description"])
|
||||
out.append("")
|
||||
if chap:
|
||||
out.append(f"**Enacted** — Laws of {STATE_NAME.get(state, state.upper())}, Chapter {chap}.")
|
||||
out.append("")
|
||||
|
||||
out.append("## Version chain")
|
||||
out.append("")
|
||||
out.append("The bill's text revisions, in order — the diff chain from filing to enrollment.")
|
||||
out.append("")
|
||||
for i, v in enumerate(versions):
|
||||
stage = STAGE_LABEL[version_rank(v.get("note"))[0]]
|
||||
note = clean(v.get("note")) or stage
|
||||
link = f" — [source]({v['url']})" if v.get("url") else ""
|
||||
out.append(f"{i+1}. **{note}** ({stage}){link}")
|
||||
out.append("")
|
||||
|
||||
if votes:
|
||||
out.append("## Votes")
|
||||
out.append("")
|
||||
for v in votes:
|
||||
y, n = v.get("yes"), v.get("no")
|
||||
tally = f"{y}–{n}" if y is not None and n is not None else (v.get("result") or "")
|
||||
ch = clean(v.get("chamber")) or ""
|
||||
motion = clean(v.get("motion")) or "vote"
|
||||
out.append(f"- {motion} — **{tally}** ({v.get('result')}) {('· ' + ch) if ch else ''}".rstrip())
|
||||
out.append("")
|
||||
|
||||
if sponsors:
|
||||
out.append("## Sponsors")
|
||||
out.append("")
|
||||
for s in sponsors:
|
||||
role = "primary" if s.get("primary") else (clean(s.get("classification")) or "cosponsor")
|
||||
out.append(f"- {clean(s['name'])} — {role} ({clean(s.get('entity_type'))})")
|
||||
out.append("")
|
||||
|
||||
out.append("## Timeline")
|
||||
out.append("")
|
||||
out.append("The legislative action history — every referral, reading, and vote.")
|
||||
out.append("")
|
||||
for a in actions:
|
||||
cls = ", ".join(a.get("classification") or [])
|
||||
tag = f" `{cls}`" if cls else ""
|
||||
out.append(f"- **{a.get('date') or '—'}** {clean(a.get('description'))}{tag}")
|
||||
out.append("")
|
||||
|
||||
out.append("## Source")
|
||||
out.append("")
|
||||
out.append(f"OpenStates / OpenCivicData bulk snapshot [{vintage}]({snapshot_url(vintage)}); "
|
||||
f"origin `{rec['bill_id']}`. Confidence: reported (aggregated from official {STATE_NAME.get(state, state.upper())} legislature records).")
|
||||
out.append("")
|
||||
|
||||
path = OUT_ROOT / state / slugify(session) / f"{slugify(ident)}.md"
|
||||
return path, "\n".join(out)
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--state", required=True)
|
||||
ap.add_argument("--input", default=None,
|
||||
help="raw JSONL; default: depot export for --vintage")
|
||||
ap.add_argument("--vintage", default=DEFAULT_VINTAGE,
|
||||
help="OpenStates bulk-snapshot date, e.g. 2026-07-01")
|
||||
args = ap.parse_args()
|
||||
state = args.state.lower()
|
||||
src = args.input or f"{DEPOT_SRC}/{args.vintage}/enacted_{state}_{args.vintage}.jsonl"
|
||||
|
||||
dest = OUT_ROOT / state
|
||||
if dest.exists():
|
||||
shutil.rmtree(dest)
|
||||
|
||||
n = 0
|
||||
used = {}
|
||||
with open(src, encoding="utf-8") as fh:
|
||||
for line in fh:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
rec = json.loads(line)
|
||||
# Record ALL enacted legislation honestly — bills AND joint/concurrent
|
||||
# resolutions. The classification frontmatter says which; filtering
|
||||
# (e.g. dropping TN's ceremonial resolutions) is a view's job, not the
|
||||
# mirror's.
|
||||
path, content = render_bill(rec, state, args.vintage)
|
||||
occ = used.get(path, 0) + 1
|
||||
used[path] = occ
|
||||
if occ > 1:
|
||||
path = path.with_name(f"{path.stem}-{occ}.md")
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(content + "\n", encoding="utf-8")
|
||||
n += 1
|
||||
print(f"{state}: wrote {n} enacted-legislation nodes to {dest.relative_to(REPO)}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
+1
-1
@@ -22,7 +22,7 @@ import sys
|
||||
from pathlib import Path
|
||||
|
||||
REPO = Path(__file__).resolve().parent.parent
|
||||
TREES = [REPO / "data" / "jurisdictions", REPO / "legal"]
|
||||
TREES = [REPO / "data" / "jurisdictions", REPO / "legal", REPO / "legislation"]
|
||||
SCHEMAS = REPO / "schemas"
|
||||
MAX_REPORT = 25
|
||||
|
||||
|
||||
Reference in New Issue
Block a user