Ingest: preserve duplicate U.S. Code section numbers; title-scoped tags

The Code genuinely contains distinct sections sharing one number (two
5 U.S.C. 5757, two 10 U.S.C. 130g, two 5 U.S.C. 3598). The ingest
silently overwrote the first with the second — data loss in a mirror.
Later occurrences now get a deterministic -N suffix.

The 'elections' tag was hardcoded from the Title 52 seed; it now
applies only to Title 52, with a per-title EXTRA_TAGS map. Makefile
gains TITLES override: make legal-us-code TITLES="7 10".

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Fabio
2026-07-06 09:52:28 -04:00
parent 170c50d899
commit 2cc22fc8e2
2 changed files with 18 additions and 5 deletions
+4 -2
View File
@@ -19,7 +19,9 @@ board: ## Rebuild the Board (inlines fresh data into viz/board.html)
changelog: ## What changed in the government since the last commit (BASE/HEAD overridable) changelog: ## What changed in the government since the last commit (BASE/HEAD overridable)
python3 scripts/generate_changelog.py $(BASE) $(HEAD) python3 scripts/generate_changelog.py $(BASE) $(HEAD)
legal-us-code: ## Ingest the pinned U.S. Code Title 52 seed corpus TITLES ?= 1 2 3 4 5 6 7 8 9 10 11 52
python3 scripts/ingest_us_code.py --title 52
legal-us-code: ## Ingest the pinned U.S. Code titles (override with TITLES="…")
@for t in $(TITLES); do python3 scripts/ingest_us_code.py --title $$t; done
check: build validate ## Build, then validate — the full gate check: build validate ## Build, then validate — the full gate
+14 -3
View File
@@ -27,6 +27,9 @@ LEGAL_ROOT = REPO / "legal" / "us" / "code"
USLM_NS = "http://xml.house.gov/schemas/uslm/1.0" USLM_NS = "http://xml.house.gov/schemas/uslm/1.0"
NS = {"u": USLM_NS, "dc": "http://purl.org/dc/elements/1.1/"} NS = {"u": USLM_NS, "dc": "http://purl.org/dc/elements/1.1/"}
# Topic tags beyond the base ["legal", "us-code"], per title.
EXTRA_TAGS = {52: ["elections"]}
DEFAULT_RELEASE = "119-100" DEFAULT_RELEASE = "119-100"
DEFAULT_PUBLIC_LAW_CONGRESS = "119" DEFAULT_PUBLIC_LAW_CONGRESS = "119"
DEFAULT_PUBLIC_LAW_NUMBER = "100" DEFAULT_PUBLIC_LAW_NUMBER = "100"
@@ -224,7 +227,7 @@ def iter_code_sections(root):
yield section yield section
def write_section(section, parents, meta, args, archive_hash, xml_path): def write_section(section, parents, meta, args, archive_hash, xml_path, used_paths):
number = section_number(section) number = section_number(section)
heading = child_text(section, "heading") heading = child_text(section, "heading")
chapter = nearest_ancestor(section, parents, "chapter") chapter = nearest_ancestor(section, parents, "chapter")
@@ -265,7 +268,7 @@ def write_section(section, parents, meta, args, archive_hash, xml_path):
"text_hash": text_hash, "text_hash": text_hash,
"retrieved_at": args.retrieved_at, "retrieved_at": args.retrieved_at,
"confidence": "official", "confidence": "official",
"tags": ["legal", "us-code", "elections"], "tags": ["legal", "us-code"] + EXTRA_TAGS.get(meta["title_number"], []),
} }
yaml = ["---"] yaml = ["---"]
@@ -283,6 +286,13 @@ def write_section(section, parents, meta, args, archive_hash, xml_path):
chapter_slug = f"chapter-{chapter_number.lower()}" if chapter_number else "chapter-unknown" chapter_slug = f"chapter-{chapter_number.lower()}" if chapter_number else "chapter-unknown"
out = LEGAL_ROOT / f"title-{meta['title_number']:02d}" / chapter_slug / f"section-{slugify(number)}.md" out = LEGAL_ROOT / f"title-{meta['title_number']:02d}" / chapter_slug / f"section-{slugify(number)}.md"
# The Code genuinely contains duplicate section numbers (e.g. two 5 U.S.C.
# 5757 enacted by different laws). Keep every one: the first occurrence in
# document order gets the plain path, later ones a deterministic -N suffix.
occurrence = used_paths.get(out, 0) + 1
used_paths[out] = occurrence
if occurrence > 1:
out = out.with_name(f"section-{slugify(number)}-{occurrence}.md")
out.parent.mkdir(parents=True, exist_ok=True) out.parent.mkdir(parents=True, exist_ok=True)
content = "\n".join(yaml) content = "\n".join(yaml)
out.write_text(content, encoding="utf-8") out.write_text(content, encoding="utf-8")
@@ -337,8 +347,9 @@ def main():
shutil.rmtree(out_title) shutil.rmtree(out_title)
files = [] files = []
used_paths = {}
for section in sorted(iter_code_sections(root), key=lambda s: section_number(s)): for section in sorted(iter_code_sections(root), key=lambda s: section_number(s)):
files.append(write_section(section, parents, meta, args, archive_hash, raw_xml)) files.append(write_section(section, parents, meta, args, archive_hash, raw_xml, used_paths))
manifest = { manifest = {
"type": "LegalManifest", "type": "LegalManifest",