diff --git a/Makefile b/Makefile new file mode 100644 index 0000000000..641828b301 --- /dev/null +++ b/Makefile @@ -0,0 +1,12 @@ +# Republic OS — repeatable pipeline commands. +# The build is deterministic: `make build` twice yields a byte-identical tree. + +.PHONY: build validate check + +build: ## Regenerate the entity tree from raw exports in data/ + python3 scripts/build.py + +validate: ## Check every entity file: OKF, schema, and link integrity + python3 scripts/validate.py + +check: build validate ## Build, then validate — the full gate diff --git a/schemas/body.schema.json b/schemas/body.schema.json new file mode 100644 index 0000000000..243d601586 --- /dev/null +++ b/schemas/body.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Body", + "description": "A government body or institution — a chamber, committee, or subcommittee. The 'repos' of the government-as-code model.", + "type": "object", + "required": ["type", "title", "classification", "chamber", "code", "confidence", "tags", "timestamp"], + "properties": { + "type": { "type": "string", "enum": ["Body"] }, + "title": { "type": "string" }, + "classification": { "type": "string", "enum": ["house", "senate", "executive", "committee", "subcommittee"] }, + "chamber": { "type": "string", "enum": ["house", "senate", "joint", "executive"] }, + "code": { "type": "string" }, + "parent": { "type": "string" }, + "leadership": { "type": "array" }, + "sources": { "type": "array" }, + "confidence": { "type": "string", "enum": ["official", "reported", "inferred", "unverified"] }, + "tags": { "type": "array" }, + "timestamp": { "type": "string" } + } +} diff --git a/schemas/candidate.schema.json b/schemas/candidate.schema.json new file mode 100644 index 0000000000..fbe83b8e8e --- /dev/null +++ b/schemas/candidate.schema.json @@ -0,0 +1,24 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Candidate", + "description": "A candidate for public office — someone running, distinct from a current officeholder. Sourced from FEC filings.", + "type": "object", + "required": ["type", "title", "office", "state", "confidence", "tags", "timestamp", "ids"], + "properties": { + "type": { "type": "string", "enum": ["Candidate"] }, + "title": { "type": "string" }, + "office": { "type": "string", "enum": ["U.S. House", "U.S. Senate"] }, + "state": { "type": "string" }, + "district": { "type": "string" }, + "party": { "type": "string" }, + "stance": { "type": "string", "enum": ["incumbent", "challenger", "open-seat"] }, + "election_year": { "type": "integer" }, + "status": { "type": "string" }, + "committee": { "type": "object" }, + "ids": { "type": "object" }, + "sources": { "type": "array" }, + "confidence": { "type": "string", "enum": ["official", "reported", "inferred", "unverified"] }, + "tags": { "type": "array" }, + "timestamp": { "type": "string" } + } +} diff --git a/schemas/jurisdiction.schema.json b/schemas/jurisdiction.schema.json new file mode 100644 index 0000000000..3fb078a059 --- /dev/null +++ b/schemas/jurisdiction.schema.json @@ -0,0 +1,20 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Jurisdiction", + "description": "A place with a government — a county or congressional district — carrying demographic context. A node in the fractal jurisdiction tree.", + "type": "object", + "required": ["type", "title", "classification", "confidence", "tags", "timestamp"], + "properties": { + "type": { "type": "string", "enum": ["Jurisdiction"] }, + "title": { "type": "string" }, + "classification": { "type": "string", "enum": ["county", "congressional-district"] }, + "fips": { "type": "string" }, + "state": { "type": "string" }, + "district": { "type": "string" }, + "demographics": { "type": "object" }, + "sources": { "type": "array" }, + "confidence": { "type": "string", "enum": ["official", "reported", "inferred", "unverified"] }, + "tags": { "type": "array" }, + "timestamp": { "type": "string" } + } +} diff --git a/schemas/person.schema.json b/schemas/person.schema.json new file mode 100644 index 0000000000..696a31f472 --- /dev/null +++ b/schemas/person.schema.json @@ -0,0 +1,23 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Person", + "description": "An officeholder — a person holding a public office at any level. OKF-conformant: only `type` is strictly required by OKF; this profile adds the fields Republic OS relies on. Unknown keys are tolerated.", + "type": "object", + "required": ["type", "title", "level", "confidence", "tags", "timestamp", "ids"], + "properties": { + "type": { "type": "string", "enum": ["Person"] }, + "title": { "type": "string" }, + "role": { "type": "string" }, + "party": { "type": "string" }, + "level": { "type": "string", "enum": ["federal", "state", "county", "municipal"] }, + "branch": { "type": "string", "enum": ["executive", "legislative", "judicial"] }, + "state": { "type": "string" }, + "leadership": { "type": "array" }, + "committees": { "type": "array" }, + "ids": { "type": "object" }, + "sources": { "type": "array" }, + "confidence": { "type": "string", "enum": ["official", "reported", "inferred", "unverified"] }, + "tags": { "type": "array" }, + "timestamp": { "type": "string" } + } +} diff --git a/scripts/validate.py b/scripts/validate.py new file mode 100644 index 0000000000..d688b2985e --- /dev/null +++ b/scripts/validate.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +"""Validate every entity file in the mirror. + +Three checks, no third-party dependencies: + 1. OKF conformance — every non-reserved .md has YAML frontmatter with a + non-empty `type`. + 2. Schema conformance — each entity validates against the JSON Schema for + its `type` in /schemas (required fields, property types, enums). The + schemas are a stricter profile over OKF's permissive base; unknown keys + are tolerated by design. + 3. Link integrity — internal /-rooted markdown links resolve to a file that + exists in the tree. + +Exit code is non-zero if any check fails, so this doubles as a CI gate. + +Usage: python scripts/validate.py +""" + +import json +import re +import sys +from pathlib import Path + +REPO = Path(__file__).resolve().parent.parent +TREE = REPO / "data" / "jurisdictions" +SCHEMAS = REPO / "schemas" +MAX_REPORT = 25 + +TYPE_PY = { + "string": str, "number": (int, float), "integer": int, + "boolean": bool, "array": list, "object": dict, +} + + +# --------------------------------------------------------------------------- # +# frontmatter parser (tuned to the deterministic output of build.py) +# --------------------------------------------------------------------------- # + +def extract_frontmatter(text): + lines = text.split("\n") + if not lines or lines[0].strip() != "---": + return None + out = [] + for line in lines[1:]: + if line.strip() == "---": + return out + out.append(line) + return None + + +def parse_scalar(s): + s = s.strip() + if s == "": + return None + try: + return json.loads(s) # "quoted", 123, 1.5, true, false, null + except (ValueError, json.JSONDecodeError): + return s # bare word, e.g. official + + +def parse_value(s): + s = s.strip() + if s.startswith("[") and s.endswith("]"): + inner = s[1:-1].strip() + return [parse_scalar(x) for x in inner.split(",")] if inner else [] + return parse_scalar(s) + + +def parse_frontmatter(text): + """Top-level parse. Block values become [] (array) or {} (object) + sentinels — sufficient for schema shape/enum/required checks.""" + fm = extract_frontmatter(text) + if fm is None: + return None + d = {} + i = 0 + while i < len(fm): + line = fm[i] + if not line.strip() or line.startswith((" ", "\t")): + i += 1 + continue + m = re.match(r"([A-Za-z0-9_]+):(.*)$", line) + if not m: + i += 1 + continue + key, rest = m.group(1), m.group(2).strip() + if rest: + d[key] = parse_value(rest) + else: + j = i + 1 + while j < len(fm) and not fm[j].strip(): + j += 1 + d[key] = [] if (j < len(fm) and fm[j].lstrip().startswith("- ")) else {} + i += 1 + return d + + +# --------------------------------------------------------------------------- # +# minimal JSON-Schema validation (required / type / enum) +# --------------------------------------------------------------------------- # + +def validate_obj(obj, schema): + errs = [] + for req in schema.get("required", []): + if req not in obj or obj[req] is None: + errs.append(f"missing required '{req}'") + for key, spec in schema.get("properties", {}).items(): + if key not in obj or obj[key] is None: + continue + val = obj[key] + t = spec.get("type") + if t and not isinstance(val, TYPE_PY[t]): + # bool is a subclass of int; reject bool where integer/number expected + if not (t in ("integer", "number") and isinstance(val, bool) is False): + errs.append(f"'{key}' expected {t}, got {type(val).__name__}") + if "enum" in spec and val not in spec["enum"]: + errs.append(f"'{key}'={val!r} not in {spec['enum']}") + return errs + + +# --------------------------------------------------------------------------- # +# main +# --------------------------------------------------------------------------- # + +def load_schemas(): + schemas = {} + for p in sorted(SCHEMAS.glob("*.schema.json")): + s = json.loads(p.read_text()) + t = s["properties"]["type"]["enum"][0] + schemas[t] = s + return schemas + + +LINK_RE = re.compile(r"\]\((/[^)\s]+\.md)\)") + + +def main(): + schemas = load_schemas() + if not schemas: + print("no schemas found in /schemas", file=sys.stderr) + return 1 + + files = sorted(TREE.rglob("*.md")) + counts = {t: 0 for t in schemas} + counts["(untyped)"] = 0 + okf_fail, schema_fail, link_fail = [], [], [] + existing = {("/" + str(p.relative_to(TREE))) for p in files} + + for p in files: + rel = p.relative_to(REPO) + text = p.read_text(encoding="utf-8") + fm = parse_frontmatter(text) + + # 1. OKF conformance + if fm is None or not fm.get("type"): + okf_fail.append(f"{rel}: no frontmatter / empty type") + counts["(untyped)"] += 1 + continue + + etype = fm["type"] + # 2. schema conformance + if etype in schemas: + counts[etype] += 1 + for e in validate_obj(fm, schemas[etype]): + schema_fail.append(f"{rel}: {e}") + else: + schema_fail.append(f"{rel}: unknown type {etype!r} (no schema)") + + # 3. link integrity + for m in LINK_RE.finditer(text): + if m.group(1) not in existing: + link_fail.append(f"{rel}: dead link {m.group(1)}") + + # ---- report ---- + total = len(files) + print(f"validated {total} entity files") + for t in sorted(counts): + if counts[t]: + print(f" {t}: {counts[t]}") + + def section(name, items): + if not items: + print(f"\n{name}: PASS (0)") + return 0 + print(f"\n{name}: FAIL ({len(items)})") + for line in items[:MAX_REPORT]: + print(f" - {line}") + if len(items) > MAX_REPORT: + print(f" … and {len(items) - MAX_REPORT} more") + return len(items) + + fails = (section("OKF conformance", okf_fail) + + section("Schema conformance", schema_fail) + + section("Link integrity", link_fail)) + + print() + if fails: + print(f"VALIDATION FAILED — {fails} issue(s)") + return 1 + print("VALIDATION PASSED — all files conform") + return 0 + + +if __name__ == "__main__": + sys.exit(main())