Ingest: preserve duplicate U.S. Code section numbers; title-scoped tags
The Code genuinely contains distinct sections sharing one number (two 5 U.S.C. 5757, two 10 U.S.C. 130g, two 5 U.S.C. 3598). The ingest silently overwrote the first with the second — data loss in a mirror. Later occurrences now get a deterministic -N suffix. The 'elections' tag was hardcoded from the Title 52 seed; it now applies only to Title 52, with a per-title EXTRA_TAGS map. Makefile gains TITLES override: make legal-us-code TITLES="7 10". Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -19,7 +19,9 @@ board: ## Rebuild the Board (inlines fresh data into viz/board.html)
|
|||||||
changelog: ## What changed in the government since the last commit (BASE/HEAD overridable)
|
changelog: ## What changed in the government since the last commit (BASE/HEAD overridable)
|
||||||
python3 scripts/generate_changelog.py $(BASE) $(HEAD)
|
python3 scripts/generate_changelog.py $(BASE) $(HEAD)
|
||||||
|
|
||||||
legal-us-code: ## Ingest the pinned U.S. Code Title 52 seed corpus
|
TITLES ?= 1 2 3 4 5 6 7 8 9 10 11 52
|
||||||
python3 scripts/ingest_us_code.py --title 52
|
|
||||||
|
legal-us-code: ## Ingest the pinned U.S. Code titles (override with TITLES="…")
|
||||||
|
@for t in $(TITLES); do python3 scripts/ingest_us_code.py --title $$t; done
|
||||||
|
|
||||||
check: build validate ## Build, then validate — the full gate
|
check: build validate ## Build, then validate — the full gate
|
||||||
|
|||||||
@@ -27,6 +27,9 @@ LEGAL_ROOT = REPO / "legal" / "us" / "code"
|
|||||||
USLM_NS = "http://xml.house.gov/schemas/uslm/1.0"
|
USLM_NS = "http://xml.house.gov/schemas/uslm/1.0"
|
||||||
NS = {"u": USLM_NS, "dc": "http://purl.org/dc/elements/1.1/"}
|
NS = {"u": USLM_NS, "dc": "http://purl.org/dc/elements/1.1/"}
|
||||||
|
|
||||||
|
# Topic tags beyond the base ["legal", "us-code"], per title.
|
||||||
|
EXTRA_TAGS = {52: ["elections"]}
|
||||||
|
|
||||||
DEFAULT_RELEASE = "119-100"
|
DEFAULT_RELEASE = "119-100"
|
||||||
DEFAULT_PUBLIC_LAW_CONGRESS = "119"
|
DEFAULT_PUBLIC_LAW_CONGRESS = "119"
|
||||||
DEFAULT_PUBLIC_LAW_NUMBER = "100"
|
DEFAULT_PUBLIC_LAW_NUMBER = "100"
|
||||||
@@ -224,7 +227,7 @@ def iter_code_sections(root):
|
|||||||
yield section
|
yield section
|
||||||
|
|
||||||
|
|
||||||
def write_section(section, parents, meta, args, archive_hash, xml_path):
|
def write_section(section, parents, meta, args, archive_hash, xml_path, used_paths):
|
||||||
number = section_number(section)
|
number = section_number(section)
|
||||||
heading = child_text(section, "heading")
|
heading = child_text(section, "heading")
|
||||||
chapter = nearest_ancestor(section, parents, "chapter")
|
chapter = nearest_ancestor(section, parents, "chapter")
|
||||||
@@ -265,7 +268,7 @@ def write_section(section, parents, meta, args, archive_hash, xml_path):
|
|||||||
"text_hash": text_hash,
|
"text_hash": text_hash,
|
||||||
"retrieved_at": args.retrieved_at,
|
"retrieved_at": args.retrieved_at,
|
||||||
"confidence": "official",
|
"confidence": "official",
|
||||||
"tags": ["legal", "us-code", "elections"],
|
"tags": ["legal", "us-code"] + EXTRA_TAGS.get(meta["title_number"], []),
|
||||||
}
|
}
|
||||||
|
|
||||||
yaml = ["---"]
|
yaml = ["---"]
|
||||||
@@ -283,6 +286,13 @@ def write_section(section, parents, meta, args, archive_hash, xml_path):
|
|||||||
|
|
||||||
chapter_slug = f"chapter-{chapter_number.lower()}" if chapter_number else "chapter-unknown"
|
chapter_slug = f"chapter-{chapter_number.lower()}" if chapter_number else "chapter-unknown"
|
||||||
out = LEGAL_ROOT / f"title-{meta['title_number']:02d}" / chapter_slug / f"section-{slugify(number)}.md"
|
out = LEGAL_ROOT / f"title-{meta['title_number']:02d}" / chapter_slug / f"section-{slugify(number)}.md"
|
||||||
|
# The Code genuinely contains duplicate section numbers (e.g. two 5 U.S.C.
|
||||||
|
# 5757 enacted by different laws). Keep every one: the first occurrence in
|
||||||
|
# document order gets the plain path, later ones a deterministic -N suffix.
|
||||||
|
occurrence = used_paths.get(out, 0) + 1
|
||||||
|
used_paths[out] = occurrence
|
||||||
|
if occurrence > 1:
|
||||||
|
out = out.with_name(f"section-{slugify(number)}-{occurrence}.md")
|
||||||
out.parent.mkdir(parents=True, exist_ok=True)
|
out.parent.mkdir(parents=True, exist_ok=True)
|
||||||
content = "\n".join(yaml)
|
content = "\n".join(yaml)
|
||||||
out.write_text(content, encoding="utf-8")
|
out.write_text(content, encoding="utf-8")
|
||||||
@@ -337,8 +347,9 @@ def main():
|
|||||||
shutil.rmtree(out_title)
|
shutil.rmtree(out_title)
|
||||||
|
|
||||||
files = []
|
files = []
|
||||||
|
used_paths = {}
|
||||||
for section in sorted(iter_code_sections(root), key=lambda s: section_number(s)):
|
for section in sorted(iter_code_sections(root), key=lambda s: section_number(s)):
|
||||||
files.append(write_section(section, parents, meta, args, archive_hash, raw_xml))
|
files.append(write_section(section, parents, meta, args, archive_hash, raw_xml, used_paths))
|
||||||
|
|
||||||
manifest = {
|
manifest = {
|
||||||
"type": "LegalManifest",
|
"type": "LegalManifest",
|
||||||
|
|||||||
Reference in New Issue
Block a user