"""Build a fresh, complete V4 public catalog in private staging. Never deploy.

All original public fields are recoverable from bounded evidence fragments;
only lookup heads omit witness bodies. Original release files remain untouched.
"""
from collections import Counter
from datetime import datetime, timezone
import copy
import gzip
import hashlib
import json
from pathlib import Path
import shutil
import sqlite3
import sys
import zipfile
sys.path.insert(0, str(Path(__file__).resolve().parents[3] / "scripts_build"))

import build_public_name_directory as base
from catalog_v4 import Catalog, ROOT, WORK, BASE_RELEASE
from public_name_segment_layout import SCHEMA, Segments, View
from public_name_v3_projection import EvidencePages, head, attach_evidence


def public_sources(old):
    sources = copy.deepcopy(old["sources"])
    datacite = next(s for s in sources if s["id"] == "datacite-public-2025")
    datacite["description"] += " V4 adds only complete closed continuation cohorts 4 through 9. Distinct DOI support and role-minority admission remain scoped to each complete native 100-member batch. Exact literal witnesses, source receipt hashes and downloadable native source/support proofs are retained; no cohort pooling or independent-publisher claim is made."
    orcid = next(s for s in sources if s["id"] == "orcid-public-2026")
    orcid["description"] += " V4 adds the minimized representation of disjoint acquisition batches 02, 03 and 04 once each, with respectively 379,097, 620,903 and 1,000,000 selected public-name profiles. Each displayed support count applies only within its own batch, never a sum of people or pooled support. Batch02 is a verified bounded prefix; its original 1,000,000-profile target was not reached. Individual profile identifiers and per-profile support locators remain private."
    sources.append({"id":"crossref-rest-bibliographic","title":"Crossref REST structured author-name facts", "source_url":"https://www.crossref.org/documentation/retrieve-metadata/",
        "license_label":"Reusable bibliographic author-name facts; no blanket content license", "license_url":"https://www.crossref.org/documentation/retrieve-metadata/", "policy_url":"https://www.crossref.org/documentation/retrieve-metadata/",
        "description":"Only explicit author given/family fields from five closed disjoint REST selections covering October1–12,2026 are included. Crossref describes bibliographic metadata as reusable facts and its own generated data as CC0; this is not a blanket CC0 license for deposited fields, abstracts or linked works. Every admitted spelling/role retains two distinct DOI witnesses, exact author-array field, literal form and response-row locator. Support and minority admission apply within the stated source packet, not across packets or people. Original selected responses remain private to avoid republishing nested contributor identifiers, affiliations or unrelated metadata. Language, origin, meaning, gender, identity and population frequency are unknown. NDR extracted source fields, applied frozen normalization/hygiene/support/role gates and deduplicated spelling keys. No resource contents or abstracts are included.","downloads":[]})
    return sources


def write_download(target, release, source, name, label, kind, data):
    path = target / "downloads" / name
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_bytes(data)
    base.require(path.stat().st_size <= 25 * 1024 * 1024, "Public download exceeds asset cap")
    source["downloads"].append({"kind": kind, "label": label, "original_filename": name,
        "url": f"/assets/public-name-data/{release}/downloads/{name}", "bytes": len(data), "sha256": base.sha(path)})


def add_source_proof_downloads(target, release, sources):
    inputs = json.loads((WORK / "INPUT_MANIFEST.json").read_text("utf-8"))
    by_id = {s["id"]: s for s in sources}
    for spec in inputs["inputs"]:
        source_id = {"datacite_native": "datacite-public-2025", "orcid_minimized": "orcid-public-2026", "crossref_bibliographic_facts": "crossref-rest-bibliographic"}[spec["kind"]]
        packet = (ROOT / spec["receipt_path"]).parent
        if spec["kind"] == "datacite_native":
            filenames = ["RECEIPT.json", "SOURCE_REPLAY_RECEIPT.json", "SOURCE_REPLAY_VERIFICATION.json", "policy_snapshot.py", "names.jsonl.gz"]
            notice = "DataCite deposited metadata: CC0-1.0. Only native admitted literal names and source/support/replay proof metadata are included here. Exact selected DOI-field observations are available in the separate V4 transformed-observation downloads. Support and minority admission apply only within this complete native100-member scope. Source archive/retained-attestation hashes bind the original complete native raw-source verification; this build did not rerun archive replay. Official source and reuse policy: https://datafiles.datacite.org/datafiles/public-2025 and https://support.datacite.org/docs/datacite-data-file-use-policy . No linked resources, abstracts or contact fields are redistributed.\n"
        else:
            filenames = ["RECEIPT.json", "NAMES.jsonl.gz", "NOTICE.txt", "SOURCE_RIGHTS_REVIEW.json", "SOURCE.json", "COUNTS.json", "COVERAGE.json", "MODIFICATIONS.json"]
            filenames = [name for name in filenames if (packet / name).exists()]
            notice = "Original closed public name packet, represented once. Retain its original source-specific NOTICE, rights and modification statements. Original unrelated ORCID profile fields and Crossref response bodies are private and are not included. This package is not a whole-corpus transfer or identity-validation claim.\n"
        archive = WORK / "private-build" / release / (spec["id"].replace(":", "--") + "-source-proofs.zip")
        with zipfile.ZipFile(archive, "x", compression=zipfile.ZIP_DEFLATED) as bundle:
            for name in filenames:
                value = (packet / name).read_bytes()
                info = zipfile.ZipInfo(name, date_time=(2026, 11, 2, 0, 0, 0)); info.compress_type = zipfile.ZIP_DEFLATED
                bundle.writestr(info, value)
            info = zipfile.ZipInfo("V4-REUSE-AND-TRANSFORMATION.txt", date_time=(2026, 11, 2, 0, 0, 0)); info.compress_type = zipfile.ZIP_DEFLATED
            bundle.writestr(info, notice.encode("utf-8"))
        write_download(target, release, by_id[source_id], archive.name, "Original admitted source packet/proofs: " + spec["id"], "source", archive.read_bytes())
    public_cutoff = {"release": release, "prior_public_release": BASE_RELEASE,
        "description": "Closed V4 scope only; ORCID05 and DataCite10 or any later packets are excluded.",
        "sources": [{"id": s["id"], "kind": s["kind"], "receipt_sha256": s["receipt_sha256"], "admitted_names_sha256": s["sha256"], "role_counts": s["expected_roles"]} for s in inputs["inputs"]],
        "input_manifest_sha256": base.sha(WORK / "INPUT_MANIFEST.json"),
        "reuse_conditions": {"DataCite": "CC0 deposited metadata; no linked-resource rights inferred", "ORCID": "CC0 public fields, with privacy/publicity/marks caveats; per-profile identifiers remain private", "Crossref": "Reusable bibliographic author-name facts, not blanket CC0 for all deposited fields or linked works", "Wiktionary": "Retained exact revision/history attribution, CC-BY-SA4 and transformation/source availability", "Japanese_training": "Retained LGPL2.1 source, notices, original files and training-source limitations"}}
    for source in sources:
        write_download(target, release, source, "V4-SOURCE-CUTOFF.json", "Frozen V4 source scopes and source-specific reuse conditions", "notice", base.encoded(public_cutoff))


def compile_indexes(views, target, release, cap=512 * 1024):
    result = {}
    def visit(ref, height, role):
        if height <= 1:
            return
        path = target / "segments" / Path(ref["path"]).name
        with path.open("rb") as stream:
            stream.seek(ref["offset"]); packed = stream.read(ref["bytes"])
        base.require(hashlib.sha256(packed).hexdigest() == ref["sha256"], "Compiled index source differs")
        raw = gzip.decompress(packed)
        base.require(len(raw) == ref["inflated_bytes"] <= 128 * 1024, "Compiled index inflation differs")
        value = json.loads(raw)
        base.require(value["schema"] == SCHEMA and value["release"] == release and value["role"] == role and value["kind"] == "branches", "Compiled index shape differs")
        entries = value["entries"]
        base.require(0 < len(entries) <= 128 and entries[0]["first"] == ref["first"] and entries[-1]["last"] == ref["last"] and
                     sum(x["count"] for x in entries) == ref["count"] and
                     all(a["last"] < b["first"] for a, b in zip(entries, entries[1:])), "Compiled index ranges differ")
        key = ref["path"] + ":" + str(ref["offset"])
        candidate = {**result, key: {"sha256": ref["sha256"], "value": value}}
        if len(base.encoded(candidate)) <= cap:
            result[key] = candidate[key]
        for entry in entries:
            visit(entry, height - 1, role)
    for role, view in views.items():
        if view["count"]:
            visit(view["root"], view["depth"], role)
    return result


def build():
    expected_path = WORK / "EXPECTED_SCOPE.json"
    expected = json.loads(expected_path.read_text("utf-8"))
    base.require(expected["ok"] and expected["expected_public_directory_keys"] == 1211234, "Missing exact frozen V4 census")
    export_audit = ROOT / "outputs/public-name-expansion-2026-11-01/datacite/CHECKPOINT10_UNIFORM_04_07_VERIFICATION.json"
    base.require(base.sha(export_audit) == "4e9e2557396600230a7e5f861b30cae44667a1e85aaa115b718a7197bc6e2bfc" and
                 json.loads(export_audit.read_text("utf-8"))["ok"], "Missing closed independent04–07 exact export proof")
    base.require(base.sha(ROOT / "scripts_build/build_public_name_expansion.py") == "2ab98d7e9bdf90d91f815e86fffed31af73adcd9606af803d209c0379569ae34", "Original V2 builder changed")
    closed = json.loads((WORK / "CATALOG_CLOSED_RECEIPT.json").read_text("utf-8"))
    base.require(closed["ok"] and closed["input_manifest_sha256"] == expected["input_manifest_sha256"], "Catalog is not closed against frozen inputs")
    catalog = Catalog(WORK / "private-catalog/catalog.sqlite3")
    units = list(catalog.db.execute("SELECT id,sha256,rows FROM units ORDER BY id"))
    base.require([list(u) for u in units] == closed["units"], "Closed source units changed")
    base.require(catalog.db.execute("SELECT count(*) FROM names").fetchone()[0] == 1211234, "Merged key census differs")
    helpers = [WORK / name for name in ["catalog_v4.py", "source_common.py", "freeze_inputs.py", "census.py", "export_native_witnesses.py", "prepare_catalog_v4.py"]] + [ROOT / "scripts_build/public_name_v3_projection.py", ROOT / "scripts_build/public_name_segment_layout.py"]
    code = {p.name: base.sha(p) for p in helpers + [Path(__file__)]}
    release = hashlib.sha256(base.encoded({"schema": SCHEMA, "units": units, "code": code, "census": base.sha(expected_path), "verified_source_export": base.sha(export_audit)})).hexdigest()[:24]
    stage = WORK / "staging" / release
    base.require(not stage.exists(), "Use a fresh immutable staging release")
    target = stage / "assets/public-name-data" / release
    target.mkdir(parents=True)
    private_work = WORK / "private-build" / release
    private_work.mkdir(parents=True)
    old = json.loads((ROOT / "site/assets/public-name-data" / BASE_RELEASE / "manifest.json").read_text("utf-8"))
    sources = public_sources(old)
    projection = sqlite3.connect(private_work / "projection.sqlite3")
    projection.execute("CREATE TABLE heads(key TEXT PRIMARY KEY,data TEXT,roles TEXT)")
    evidence = Segments(target, release, segment_bytes=512 * 1024, start_number=500000)
    pages = EvidencePages(evidence, projection)
    totals = Counter(); languages = Counter(); largest_record = (0, ""); count = 0
    key_index = WORK / f"KEY_ROLE_INDEX_{release}.jsonl.gz"
    with key_index.open("wb") as raw_index:
        with gzip.GzipFile(filename="", mode="wb", fileobj=raw_index, mtime=0, compresslevel=6) as index:
            for row in catalog.records():
                count += 1
                totals["spellings"] += 1; totals["role_keys"] += len(row["roles"])
                for role in row["roles"]: totals[role + "_role_keys"] += 1
                totals["corroborated_spellings"] += bool(row["corroborated_roles"])
                totals["training_source_spellings"] += bool(row["primary_locators"])
                totals["source_attested_spellings"] += bool(row.get("attestations"))
                totals["training_source_only_spellings"] += bool(row["primary_locators"]) and not row["corroborated_roles"] and not row.get("attestations")
                totals["source_role_differences"] += row["source_role_difference"]
                for item in row.get("attestations", []):
                    if item["tier"] == "lexical_sharealike": languages[(item["language"], item["language_name"])] += 1
                largest_record = max(largest_record, (len(base.encoded(row)), row["key"]))
                parts = pages.add(row)
                compact = head(row, parts)
                projection.execute("INSERT INTO heads VALUES(?,?,?)", (row["key"], base.encoded(compact).decode("utf-8"), json.dumps(row["roles"])))
                index.write(base.encoded({"key": row["key"], "roles": row["roles"]}))
                if count % 25000 == 0:
                    projection.commit(); print(json.dumps({"stage": "evidence_projected", "keys": count}), flush=True)
    pages.flush(); evidence.close(); projection.commit()
    totals["additional_public_spellings"] = count - 911217
    base.require(totals["spellings"] == expected["expected_public_directory_keys"] and totals["given_role_keys"] == expected["given_role_keys"] and
                 totals["family_role_keys"] == expected["family_role_keys"] and totals["corroborated_spellings"] == 31818 and
                 totals["training_source_only_spellings"] == expected["training_source_only_spellings"] and totals["source_attested_spellings"] == expected["qualified_source_spellings"] and
                 len({code for code, _ in languages}) == 224, "Frozen public role/evidence/language census differs")
    hot = Segments(target, release, segment_bytes=128 * 1024)
    writers = {role: View(hot, role, target_page_bytes=32 * 1024, page_rows=25) for role in ("all", "given", "family")}
    largest_head = (0, "")
    for key, encoded, roles in projection.execute("SELECT key,data,roles FROM heads ORDER BY key"):
        row = attach_evidence(json.loads(encoded), projection)
        largest_head = max(largest_head, (len(base.encoded(row)), key))
        writers["all"].add(row)
        for role in json.loads(roles): writers[role].add(row)
    views = {role: writer.finish() for role, writer in writers.items()}
    hot.close()
    print(json.dumps({"stage": "heads_indexed", "release": release, "hot_segments": len(hot.files), "evidence_segments": len(evidence.files)}), flush=True)
    # Retained downloads cover old observations. Only new scope events are
    # exported again; each part embeds its own source/license/modification header.
    for source in sources:
        if source["id"] == "namsor-japanese-public-2022":
            continue
        notice = {"type": "source_notice", "source": {k: source[k] for k in ("id", "title", "source_url", "license_label", "license_url", "policy_url", "description")},
                  "modifications": "Additional closed V4 observations only. Exact role/language/DOI or minimized aggregate support retained; frozen normalization, deduplication, compact heads and lossless evidence pagination. Retained earlier download parts remain available. Covered lexical adaptations remain CC-BY-SA 4.0; preserve source/license/modification notices."}
        cursor = catalog.db.execute("SELECT key,data FROM events WHERE source_id=? AND unit_rank>0 ORDER BY key,unit_rank,seq", (source["id"],))
        part = 0; chunk = []; chunk_bytes = 0; current_key = None; group = []
        def emit(items, number):
            path = private_work / f"source-{source['id']}-{number:03d}.jsonl.gz"
            with path.open("wb") as raw:
                with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0, compresslevel=6) as stream:
                    stream.write(base.encoded(notice))
                    for item in items: stream.write(base.encoded(item))
            write_download(target, release, source, path.name, f"Additional V4 source observations, part {number + 1} (JSONL gzip with notice)", "transformed", path.read_bytes())
        for key, encoded in cursor:
            if current_key is not None and key != current_key:
                item = {"key": current_key, "attestations": group}; group = []
                size = len(base.encoded(item))
                base.require(size <= 192 * 1024, "Source download row needs lossless pagination")
                if chunk and chunk_bytes + size > 16 * 1024 * 1024: emit(chunk, part); part += 1; chunk = []; chunk_bytes = 0
                chunk.append(item); chunk_bytes += size
            current_key = key; group.append(json.loads(encoded))
        if current_key is not None: chunk.append({"key": current_key, "attestations": group})
        if chunk: emit(chunk, part)
    for source in sources:
        for script in helpers + [Path(__file__)]:
            write_download(target, release, source, script.name, "V4 transformation source: " + script.name, "modifications", script.read_bytes())
    # Preserve the producer's public notices without publishing its private
    # profile/member audit locators or source snapshots.
    for filename in ():
        p = ROOT / "outputs/public-name-expansion-2026-11-01/orcid-continuation-01/packet-02" / filename
        write_download(target, release, sources[-1], "ORCID-" + filename, "ORCID public minimization " + filename, "notice", p.read_bytes())
    add_source_proof_downloads(target, release, sources)
    manifest = {**old, "schema": SCHEMA, "release": release, "totals": dict(totals), "sources": sources, "views": views,
        "storage": {"transport": "bounded_stream_segments/1", "hot_segments": len(hot.files), "evidence_segments": len(evidence.files),
                    "hot_segment_bytes": 128 * 1024, "evidence_segment_bytes": 512 * 1024,
                    "head_page_inflated_bytes": 32 * 1024, "evidence_page_inflated_bytes": 64 * 1024,
                    "maximum_records": 50, "default_head_records": 25, "maximum_head_data_reads": 1, "maximum_index_reads": 8},
        "record_format": "compact_heads_with_lossless_paged_evidence",
        "lexical_language_attestations": [{"code": code, "name": name, "observations": number} for (code, name), number in sorted(languages.items(), key=lambda x: (x[0][1].casefold(), x[0][0]))],
        "limitations": old["limitations"] + ["ORCID observations are public self-asserted fields with aggregate support, not identity verification. Individual profile identifiers and support locators are not republished.",
            "Lookup heads retain original roles, forms, warnings, source-role/language summaries and corroboration flags. Complete original public fields and every assertion are recoverable through paginated evidence and source downloads. Counts are spelling keys, not people."]}
    base.write_json(target / "manifest.json", manifest, 512 * 1024)
    pointer = stage / "assets/public-name-data/manifest.json"; pointer.write_bytes((target / "manifest.json").read_bytes())
    indexes = compile_indexes(views, target, release)
    generated = stage / "functions/_lib/active-public-name-release.js"; generated.parent.mkdir(parents=True)
    data = json.dumps({"manifest": manifest, "indexes": indexes}, ensure_ascii=True, separators=(",", ":"))
    generated.write_text("// Build-pinned immutable release metadata and validated upper indexes. No visitor state.\nconst freeze=(value)=>{if(value&&typeof value==='object'){Object.freeze(value);for(const child of Object.values(value))freeze(child);}return value;};\nconst data=freeze(" + data + ");\nexport const activeManifest=data.manifest;\nexport const compiledIndexes=data.indexes;\n", "utf-8")
    current_static = [p for p in (ROOT / "site").rglob("*") if p.is_file() and not any(x in p.relative_to(ROOT / "site").parts for x in (".wrangler", "functions"))]
    new_static = [p for p in (stage / "assets").rglob("*") if p.is_file()]
    total_files = len(current_static) + len(new_static) - 1
    base.require(total_files <= 20000 and max(p.stat().st_size for p in new_static) <= 25 * 1024 * 1024, "Actual retained-file/asset budget exceeded")
    result = {"ok": True, "built_at": datetime.now(timezone.utc).isoformat(), "release": release, "schema": SCHEMA,
        "stage": str(stage), "units": units, "code_sha256": code, "totals": dict(totals), "public_lexical_codes": 224,
        "hot_segments": len(hot.files), "evidence_segments": len(evidence.files), "head_pages": {role: writer.page_count for role, writer in writers.items()},
        "index_depth": {role: view["depth"] for role, view in views.items()}, "compiled_index_nodes": len(indexes),
        "compiled_index_bytes": len(base.encoded(indexes)), "generated_module_bytes": generated.stat().st_size,
        "largest_full_record": largest_record, "largest_head": largest_head, "largest_evidence_page_bytes": pages.largest_inflated,
        "evidence_pages": pages.pages, "new_static_files": len(new_static), "total_static_files_including_retained": total_files,
        "new_static_bytes": sum(p.stat().st_size for p in new_static), "manifest_sha256": base.sha(pointer),
        "active_module_sha256": base.sha(generated), "key_role_index": str(key_index), "key_role_index_sha256": base.sha(key_index),
        "fifty_million_achieved": False, "canonical_site_changed": False, "deployed": False,
        "capacity_scope": "Actual current1211234 release only; all future expansion remains subject to file/byte/request and real provider CPU/allowance checks."}
    receipt = WORK / f"BUILD_RECEIPT_{release}.json"
    base.require(not receipt.exists(), "Immutable build receipt exists")
    receipt.write_text(json.dumps(result, indent=2) + "\n", "utf-8")
    projection.close(); catalog.close()
    print(json.dumps(result, indent=2), flush=True)
    return result


if __name__ == "__main__":
    build()
