"""Build bounded, immutable Pages shards from an explicitly approved public packet.

No generated names or private corroboration files enter this artifact. Every
input spelling is checked again with the pinned existing hygiene/normalization.
"""
import argparse
from datetime import datetime, timezone
import gzip
import hashlib
import json
import os
from pathlib import Path
import shutil
import sys
import types
import unicodedata

ROOT = Path(__file__).resolve().parents[1]
POLICY = ROOT / "scripts_build/database_moat/ndr_lib.py"
POLICY_SHA = "5650212d3efd633e8806c8e2cbc7de53dfd9c8430a2f34db4b66f690c8aeccc3"
SCHEMA = "ndr-public-directory/1"
PAGE_ROWS = 256
PAGE_BYTES = 256 * 1024
BUCKET_ROWS = 4096
MANIFEST_BYTES = 512 * 1024
INDEX_BYTES = 128 * 1024
FILE_BUDGET = 18000  # Leave headroom for the existing site under Free's 20,000 files.
sys.dont_write_bytecode = True


def require(condition, message):
    if not condition:
        raise ValueError(message)


def sha(path):
    digest = hashlib.sha256()
    with Path(path).open("rb") as stream:
        for block in iter(lambda: stream.read(4 * 1024 * 1024), b""):
            digest.update(block)
    return digest.hexdigest()


def encoded(value):
    return (json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + "\n").encode("utf-8")


def write_json(path, value, maximum=None):
    data = encoded(value)
    require(maximum is None or len(data) <= maximum, "Static shard exceeds its size bound: " + path.name)
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_bytes(data)
    return {"bytes": len(data), "sha256": hashlib.sha256(data).hexdigest()}


def safe_url(value):
    return isinstance(value, str) and value.startswith("https://") and not any(ord(c) < 32 for c in value)


def policy():
    require(sha(POLICY) == POLICY_SHA, "Frozen policy changed")
    module = types.ModuleType("public_directory_frozen_policy")
    module.__file__ = str(POLICY)
    exec(compile(POLICY.read_bytes(), str(POLICY), "exec"), module.__dict__)
    return module


def written_script(form):
    scripts = set()
    for character in form:
        if not unicodedata.category(character).startswith("L"):
            continue
        name = unicodedata.name(character, "")
        if "LATIN" in name:
            scripts.add("Latin")
        elif name.startswith(("CJK ", "IDEOGRAPHIC ")):
            scripts.add("Han")
        else:
            found = next((script for script in ("Hiragana", "Katakana", "Hangul", "Cyrillic", "Greek", "Arabic", "Hebrew", "Devanagari", "Thai", "Armenian", "Georgian", "Bengali", "Tamil", "Telugu", "Kannada", "Malayalam", "Gujarati", "Gurmukhi", "Sinhala", "Myanmar", "Ethiopic") if name.startswith(script.upper() + " ")), None)
            scripts.add(found or "unclassified")
    if not scripts or "unclassified" in scripts:
        return "Script not classified"
    return " + ".join(sorted(scripts)) + " text"


def read_records(path, source, normalizer):
    rows = []
    seen = set()
    public_files = {entry["original_filename"]: entry for entry in source["download_files"] if entry["kind"] == "original"}
    opener = gzip.open if str(path).endswith(".gz") else open
    with opener(path, "rt", encoding="utf-8") as stream:
        for number, line in enumerate(stream, 1):
            require(len(line.encode("utf-8")) <= 65536, "Oversized public record")
            row = json.loads(line)
            key = row["key"]
            require(isinstance(key, str) and key not in seen, "Duplicate or invalid public key")
            seen.add(key)
            forms = sorted(set(row["forms"]))
            require(forms and row["name"] in forms, "Preferred spelling is not a literal source form")
            for form in forms:
                require(isinstance(form, str) and normalizer.name_key(form) == key and normalizer.hygiene_reason(normalizer.normalize_form(form)) is None,
                        "Public form differs from frozen source hygiene/normalization")
            roles = sorted(set(row["roles"]))
            corroborated = sorted(set(row.get("corroborated_roles", [])))
            require(roles and set(roles) <= {"given", "family"} and set(corroborated) <= set(roles), "Invalid public role")
            require(row["source_id"] == source["id"] and row["source_category"] == "public_training_corpus" and row["language"] is None,
                    "Unexpected source or inferred language")
            require(row["evidence_status"] == ("corroborated_name" if corroborated else "training_source"), "Incorrect evidence label")
            locators = []
            for original in row["primary_locators"]:
                entry = public_files.get(original["file"])
                require(entry and original["role"] in roles and isinstance(original["line"], int) and original["line"] > 0 and
                        original["sha256"] == entry["sha256"] and safe_url(original["raw_url"]), "Invalid public primary locator")
                locators.append({field: original[field] for field in ("role", "file", "line", "raw_url", "sha256")})
            require(locators and set(item["role"] for item in locators) == set(roles), "Source role is missing a primary locator")
            corroboration = []
            for original in row.get("corroboration", []):
                require(original["role"] in corroborated and isinstance(original["evidence_id"], str) and
                        all(isinstance(value, str) for value in original["source_ids"] + original["tiers"]), "Invalid corroboration metadata")
                corroboration.append({field: original[field] for field in ("role", "source_ids", "tiers", "evidence_id")})
            warnings = row.get("role_warnings", [])
            require(isinstance(warnings, list), "Invalid role warnings")
            # Explicit whitelist: PRIVATE_CORROBORATION and unknown packet fields never enter public output.
            rows.append({"key": key, "name": row["name"], "forms": forms, "roles": roles, "source_id": source["id"],
                         "source_category": "public_training_corpus", "evidence_status": row["evidence_status"],
                         "corroborated_roles": corroborated, "role_warnings": warnings, "language": None,
                         "written_script": written_script(row["name"]), "primary_locators": locators,
                         "corroboration": corroboration})
    return sorted(rows, key=lambda row: row["key"])


def page_groups(rows, release):
    group = []
    overhead = len(encoded({"schema": SCHEMA, "release": release, "records": []}))
    size = overhead
    for row in rows:
        row_size = len(encoded(row)) - 1
        added = row_size + (1 if group else 0)
        if group and (len(group) >= PAGE_ROWS or size + added > PAGE_BYTES):
            yield group
            group = []
            size = overhead
            added = row_size
        group.append(row)
        size += added
        require(size <= PAGE_BYTES, "Public record exceeds static page size")
    if group:
        yield group


def build(records_path, config_path, destination):
    records_path, config_path, destination = map(lambda value: Path(value).resolve(), (records_path, config_path, destination))
    config = json.loads(config_path.read_text(encoding="utf-8"))
    require(config.get("publication_approved") is True and config.get("fixture_only") is False, "Use an explicitly approved real public source config")
    require(config["records_sha256"] == sha(records_path), "Approved public records changed")
    source = config["source"]
    require(source["id"] == "namsor-japanese-public-2022" and safe_url(source["source_url"]), "Unsupported current public source")
    require(all(entry["kind"] in {"original", "license", "notice", "modifications", "readme", "compliance"} for entry in source["download_files"]), "Unapproved public download kind")
    require({"original", "license", "notice", "modifications"} <= {entry["kind"] for entry in source["download_files"]}, "Missing source/license/notice/modification availability")
    normalizer = policy()
    rows = read_records(records_path, source, normalizer)
    require(rows and len(rows) == config["expected_spellings"], "Eligible source count differs from approval")
    folds = {chr(point): chr(point).casefold() for point in range(0x110000) if chr(point).casefold() != chr(point)}
    identity = hashlib.sha256(encoded({"schema": SCHEMA, "records": sha(records_path), "config": sha(config_path),
                                      "builder": sha(__file__), "policy": POLICY_SHA, "unicode": unicodedata.unidata_version})).hexdigest()
    release = identity[:24]
    require(not (destination / release).exists(), "Use a fresh immutable release; verify rather than overwrite an existing one")
    require(destination.is_relative_to(ROOT / "site/assets/public-name-data"), "Build only in the current public directory namespace")
    staging_root = ROOT / "outputs/public-name-expansion-2026-11-01/website-implementation/staging"
    target = (staging_root / release).resolve()
    require(target.is_relative_to(ROOT / "outputs") and not target.exists(), "Use a fresh private staging directory")
    target.mkdir(parents=True)
    public_source = {field: source[field] for field in ("id", "title", "source_url", "license_label", "description")}
    downloads = []
    for number, entry in enumerate(source["download_files"]):
        path = Path(entry["path"]).resolve()
        require(path.is_file() and sha(path) == entry["sha256"] and path.stat().st_size <= 25 * 1024 * 1024,
                "Source download changed or exceeds the Pages asset limit")
        require("private" not in path.name.lower(), "Private corroboration cannot be published")
        name = "".join(c if c.isascii() and (c.isalnum() or c in "._-") else "-" for c in entry["original_filename"])
        filename = f"{number:02d}-{name}"
        copy = target / "downloads" / filename
        copy.parent.mkdir(exist_ok=True)
        shutil.copyfile(path, copy)
        downloads.append({"kind": entry["kind"], "label": entry["label"], "original_filename": entry["original_filename"],
                          "url": f"/assets/public-name-data/{release}/downloads/{filename}", "sha256": entry["sha256"], "bytes": copy.stat().st_size})
    transformed = target / "downloads/transformed-records.jsonl.gz"
    with transformed.open("wb") as raw:
        with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0) as packed:
            for row in rows:
                packed.write(encoded(row))
    require(transformed.stat().st_size <= 25 * 1024 * 1024, "Transformed source download needs another approved storage mechanism")
    downloads.append({"kind": "transformed", "label": "Transformed public records (JSONL gzip)", "original_filename": transformed.name,
                      "url": f"/assets/public-name-data/{release}/downloads/{transformed.name}", "sha256": sha(transformed), "bytes": transformed.stat().st_size})
    public_source["downloads"] = downloads
    views = {}
    file_count = len(downloads)
    for role in ("all", "given", "family"):
        selected = rows if role == "all" else [row for row in rows if role in row["roles"]]
        buckets = []
        for number, start in enumerate(range(0, len(selected), BUCKET_ROWS)):
            bucket_rows = selected[start:start + BUCKET_ROWS]
            pages = []
            for page_number, group in enumerate(page_groups(bucket_rows, release)):
                relative = f"{role}/{number:06d}/{page_number:05d}.json"
                fingerprint = write_json(target / relative, {"schema": SCHEMA, "release": release, "records": group}, PAGE_BYTES)
                pages.append({"path": f"/assets/public-name-data/{release}/{relative}", "first": group[0]["key"], "last": group[-1]["key"],
                              "count": len(group), **fingerprint})
                file_count += 1
            relative = f"{role}/{number:06d}/index.json"
            fingerprint = write_json(target / relative, {"schema": SCHEMA, "release": release, "role": role, "pages": pages}, INDEX_BYTES)
            buckets.append({"path": f"/assets/public-name-data/{release}/{relative}", "first": bucket_rows[0]["key"],
                            "last": bucket_rows[-1]["key"], "count": len(bucket_rows), **fingerprint})
            file_count += 1
        views[role] = {"count": len(selected), "buckets": buckets}
    totals = {"spellings": len(rows), "role_keys": sum(len(row["roles"]) for row in rows),
              "given_role_keys": views["given"]["count"], "family_role_keys": views["family"]["count"],
              "corroborated_spellings": sum(bool(row["corroborated_roles"]) for row in rows),
              "training_source_only_spellings": sum(not row["corroborated_roles"] for row in rows),
              "corroborated_role_keys": sum(len(row["corroborated_roles"]) for row in rows)}
    manifest = {"schema": SCHEMA, "release": release, "publication_approved": True, "fixture_only": False,
                "normalization": {"form": "NFC-casefold-NFC", "unicode_version": unicodedata.unidata_version,
                                  "policy_sha256": POLICY_SHA, "folds": folds},
                "totals": totals, "held": config["held"], "sources": [public_source], "views": views,
                "limitations": ["These are literal records from a public training release, not a population register or a guarantee of personal-name validity.",
                                "The source labels roles. Language, origin, meaning, gender and frequency are not established for these records.",
                                "A corroboration label means the same normalized spelling and role also occur in the referenced source-backed checkpoint; it is not identity verification.",
                                "Written-script labels describe the released characters, not the name's language or an asserted native-script translation.",
                                "Names Done Right is independent of the source publisher. Attribution does not imply endorsement."]}
    require(file_count + 1 < FILE_BUDGET, "Pages static-file budget exceeded; use an approved indexed storage service before expanding further")
    fingerprint = write_json(target / "manifest.json", manifest, MANIFEST_BYTES)
    # All generation occurs outside site/. Commit the checked release directory,
    # then replace the small pointer last; failed stages never become public.
    destination.mkdir(parents=True, exist_ok=True)
    final = (destination / release).resolve()
    require(final.is_relative_to(ROOT / "site/assets/public-name-data") and not final.exists(), "Public release target changed")
    os.replace(target, final)
    pointer = destination / "manifest.next.json"
    shutil.copyfile(final / "manifest.json", pointer)
    os.replace(pointer, destination / "manifest.json")
    return {"ok": True, "schema": SCHEMA, "release": release, "totals": totals, "held": config["held"],
            "manifest": fingerprint, "generated_files": file_count + 2, "records_sha256": sha(records_path),
            "builder_sha256": sha(__file__), "public_source_config_sha256": sha(config_path),
            "max_page_bytes": PAGE_BYTES, "max_page_rows": PAGE_ROWS, "max_bucket_rows": BUCKET_ROWS,
            "source_scope": "One approved public training-source packet, not the proprietary NamSor model or a complete Japanese-name census",
            "fifty_million_achieved": False}


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--records", required=True)
    parser.add_argument("--source-config", required=True)
    parser.add_argument("--out", required=True)
    parser.add_argument("--receipt", required=True)
    args = parser.parse_args()
    proof = Path(args.receipt)
    require(not proof.exists(), "Use a fresh build receipt")
    result = build(args.records, args.source_config, args.out)
    result["built_at"] = datetime.now(timezone.utc).isoformat()
    proof.parent.mkdir(parents=True, exist_ok=True)
    proof.write_text(json.dumps(result, ensure_ascii=True, indent=2) + "\n", encoding="utf-8")
    print(json.dumps(result, indent=2))


if __name__ == "__main__":
    main()
