From 2cc22fc8e2e262fa96991790b94539612f7ed5b4 Mon Sep 17 00:00:00 2001 From: Fabio Date: Mon, 6 Jul 2026 09:52:28 -0400 Subject: [PATCH] Ingest: preserve duplicate U.S. Code section numbers; title-scoped tags MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Code genuinely contains distinct sections sharing one number (two 5 U.S.C. 5757, two 10 U.S.C. 130g, two 5 U.S.C. 3598). The ingest silently overwrote the first with the second — data loss in a mirror. Later occurrences now get a deterministic -N suffix. The 'elections' tag was hardcoded from the Title 52 seed; it now applies only to Title 52, with a per-title EXTRA_TAGS map. Makefile gains TITLES override: make legal-us-code TITLES="7 10". Co-Authored-By: Claude Opus 4.8 --- Makefile | 6 ++++-- scripts/ingest_us_code.py | 17 ++++++++++++++--- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 66a43b39ad..988171060c 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,9 @@ board: ## Rebuild the Board (inlines fresh data into viz/board.html) changelog: ## What changed in the government since the last commit (BASE/HEAD overridable) python3 scripts/generate_changelog.py $(BASE) $(HEAD) -legal-us-code: ## Ingest the pinned U.S. Code Title 52 seed corpus - python3 scripts/ingest_us_code.py --title 52 +TITLES ?= 1 2 3 4 5 6 7 8 9 10 11 52 + +legal-us-code: ## Ingest the pinned U.S. Code titles (override with TITLES="…") + @for t in $(TITLES); do python3 scripts/ingest_us_code.py --title $$t; done check: build validate ## Build, then validate — the full gate diff --git a/scripts/ingest_us_code.py b/scripts/ingest_us_code.py index 29632a14f9..3c6eafe6dc 100644 --- a/scripts/ingest_us_code.py +++ b/scripts/ingest_us_code.py @@ -27,6 +27,9 @@ LEGAL_ROOT = REPO / "legal" / "us" / "code" USLM_NS = "http://xml.house.gov/schemas/uslm/1.0" NS = {"u": USLM_NS, "dc": "http://purl.org/dc/elements/1.1/"} +# Topic tags beyond the base ["legal", "us-code"], per title. +EXTRA_TAGS = {52: ["elections"]} + DEFAULT_RELEASE = "119-100" DEFAULT_PUBLIC_LAW_CONGRESS = "119" DEFAULT_PUBLIC_LAW_NUMBER = "100" @@ -224,7 +227,7 @@ def iter_code_sections(root): yield section -def write_section(section, parents, meta, args, archive_hash, xml_path): +def write_section(section, parents, meta, args, archive_hash, xml_path, used_paths): number = section_number(section) heading = child_text(section, "heading") chapter = nearest_ancestor(section, parents, "chapter") @@ -265,7 +268,7 @@ def write_section(section, parents, meta, args, archive_hash, xml_path): "text_hash": text_hash, "retrieved_at": args.retrieved_at, "confidence": "official", - "tags": ["legal", "us-code", "elections"], + "tags": ["legal", "us-code"] + EXTRA_TAGS.get(meta["title_number"], []), } yaml = ["---"] @@ -283,6 +286,13 @@ def write_section(section, parents, meta, args, archive_hash, xml_path): chapter_slug = f"chapter-{chapter_number.lower()}" if chapter_number else "chapter-unknown" out = LEGAL_ROOT / f"title-{meta['title_number']:02d}" / chapter_slug / f"section-{slugify(number)}.md" + # The Code genuinely contains duplicate section numbers (e.g. two 5 U.S.C. + # 5757 enacted by different laws). Keep every one: the first occurrence in + # document order gets the plain path, later ones a deterministic -N suffix. + occurrence = used_paths.get(out, 0) + 1 + used_paths[out] = occurrence + if occurrence > 1: + out = out.with_name(f"section-{slugify(number)}-{occurrence}.md") out.parent.mkdir(parents=True, exist_ok=True) content = "\n".join(yaml) out.write_text(content, encoding="utf-8") @@ -337,8 +347,9 @@ def main(): shutil.rmtree(out_title) files = [] + used_paths = {} for section in sorted(iter_code_sections(root), key=lambda s: section_number(s)): - files.append(write_section(section, parents, meta, args, archive_hash, raw_xml)) + files.append(write_section(section, parents, meta, args, archive_hash, raw_xml, used_paths)) manifest = { "type": "LegalManifest",