"""Build a fresh, complete V3 public catalog in private staging. Never deploy.

All original public fields are recoverable from bounded evidence fragments;
only lookup heads omit witness bodies. Original release files remain untouched.
"""
from collections import Counter
from datetime import datetime, timezone
import copy
import gzip
import hashlib
import json
from pathlib import Path
import shutil
import sqlite3

import build_public_name_directory as base
from public_name_v3_catalog import Catalog, ROOT, WORK, BASE_RELEASE
from public_name_segment_layout import SCHEMA, Segments, View
from public_name_v3_projection import EvidencePages, head, attach_evidence


def public_sources(old):
    sources = copy.deepcopy(old["sources"])
    sources[1]["description"] = "Admitted DataCite public-2025 personal givenName/familyName fields from the retained current 200-member and cohort 0 scopes, plus complete continuation cohorts 1, 2 and 3. Every assertion retains its exact DOI/member/field witnesses and support scope. Continuation support applies within each batch of 100 complete archive members; it is never pooled into people, population frequency or independent-source corroboration. Language and origin are unknown; linked resources and abstracts are excluded."
    sources.append({"id": "orcid-public-2026", "title": "ORCID Public Data File 2026 — minimized public name fields",
        "source_url": "https://doi.org/10.23640/07243.34697766.v1", "license_label": "CC0 1.0 public-domain dedication for public name fields",
        "license_url": "https://creativecommons.org/publicdomain/zero/1.0/", "policy_url": "https://info.orcid.org/public-data-file-use-policy/",
        "description": "Admitted public self-asserted given-names/family-name fields from one bounded batch of 100,000 public profiles in the 2026 release. A minimum of two distinct public records supports each admitted spelling/role within this batch; that is not a count of distinct people or identity validation. Public outputs exclude individual ORCID IDs, contact details and per-profile/member audit locators. Source language, origin, meaning, gender and population frequency are unknown. Source archive publisher MD5: a55a16fe0b292bda03cc6016546c8984. Privacy/publicity and trademark rights are not waived by CC0. Packet 02 is the minimized representation of this one batch, not additional acquisition.",
        "downloads": []})
    return sources


def write_download(target, release, source, name, label, kind, data):
    path = target / "downloads" / name
    path.parent.mkdir(parents=True, exist_ok=True)
    path.write_bytes(data)
    base.require(path.stat().st_size <= 25 * 1024 * 1024, "Public download exceeds asset cap")
    source["downloads"].append({"kind": kind, "label": label, "original_filename": name,
        "url": f"/assets/public-name-data/{release}/downloads/{name}", "bytes": len(data), "sha256": base.sha(path)})


def compile_indexes(views, target, release, cap=512 * 1024):
    result = {}
    def visit(ref, height, role):
        if height <= 1:
            return
        path = target / "segments" / Path(ref["path"]).name
        with path.open("rb") as stream:
            stream.seek(ref["offset"]); packed = stream.read(ref["bytes"])
        base.require(hashlib.sha256(packed).hexdigest() == ref["sha256"], "Compiled index source differs")
        raw = gzip.decompress(packed)
        base.require(len(raw) == ref["inflated_bytes"] <= 128 * 1024, "Compiled index inflation differs")
        value = json.loads(raw)
        base.require(value["schema"] == SCHEMA and value["release"] == release and value["role"] == role and value["kind"] == "branches", "Compiled index shape differs")
        entries = value["entries"]
        base.require(0 < len(entries) <= 128 and entries[0]["first"] == ref["first"] and entries[-1]["last"] == ref["last"] and
                     sum(x["count"] for x in entries) == ref["count"] and
                     all(a["last"] < b["first"] for a, b in zip(entries, entries[1:])), "Compiled index ranges differ")
        key = ref["path"] + ":" + str(ref["offset"])
        candidate = {**result, key: {"sha256": ref["sha256"], "value": value}}
        if len(base.encoded(candidate)) <= cap:
            result[key] = candidate[key]
        for entry in entries:
            visit(entry, height - 1, role)
    for role, view in views.items():
        if view["count"]:
            visit(view["root"], view["depth"], role)
    return result


def build():
    expected_path = ROOT / "outputs/public-name-expansion-2026-11-01/PUBLIC_EXPANSION_EXPECTED_SCOPE_AUDIT_V3.json"
    expected = json.loads(expected_path.read_text("utf-8"))
    base.require(expected["ok"] and expected["expected_public_directory_keys"] == 911217, "Missing independent frozen census")
    base.require(base.sha(ROOT / "scripts_build/build_public_name_expansion.py") == "2ab98d7e9bdf90d91f815e86fffed31af73adcd9606af803d209c0379569ae34", "Original V2 builder changed")
    catalog = Catalog(WORK / "private-catalog/catalog.sqlite3")
    units = list(catalog.db.execute("SELECT id,sha256,rows FROM units ORDER BY id"))
    wanted = {"baseline597769": 597769, "datacite-continuation-cohort-0001": 1039245,
              "datacite-continuation-cohort-0002": 470861, "datacite-continuation-cohort-0003": 589281,
              "dictionary-tranche03": 13273, "orcid-continuation-01-public02": 14883}
    base.require({name: count for name, _, count in units} == wanted, "Only all six complete frozen source units may be published")
    base.require(catalog.db.execute("SELECT count(*) FROM names").fetchone()[0] == 911217, "Merged key census differs")
    helpers = [ROOT / "scripts_build" / name for name in ["public_name_v3_catalog.py", "public_name_v3_projection.py", "public_name_segment_layout.py"]]
    code = {p.name: base.sha(p) for p in helpers + [Path(__file__)]}
    release = hashlib.sha256(base.encoded({"schema": SCHEMA, "units": units, "code": code, "census": base.sha(expected_path)})).hexdigest()[:24]
    stage = WORK / "staging" / release
    base.require(not stage.exists(), "Use a fresh immutable staging release")
    target = stage / "assets/public-name-data" / release
    target.mkdir(parents=True)
    private_work = WORK / "private-build" / release
    private_work.mkdir(parents=True)
    old = json.loads((ROOT / "site/assets/public-name-data" / BASE_RELEASE / "manifest.json").read_text("utf-8"))
    sources = public_sources(old)
    projection = sqlite3.connect(private_work / "projection.sqlite3")
    projection.execute("CREATE TABLE heads(key TEXT PRIMARY KEY,data TEXT,roles TEXT)")
    evidence = Segments(target, release, segment_bytes=512 * 1024, start_number=500000)
    pages = EvidencePages(evidence, projection)
    totals = Counter(); languages = Counter(); largest_record = (0, ""); count = 0
    key_index = WORK / f"KEY_ROLE_INDEX_{release}.jsonl.gz"
    with key_index.open("wb") as raw_index:
        with gzip.GzipFile(filename="", mode="wb", fileobj=raw_index, mtime=0, compresslevel=6) as index:
            for row in catalog.records():
                count += 1
                totals["spellings"] += 1; totals["role_keys"] += len(row["roles"])
                for role in row["roles"]: totals[role + "_role_keys"] += 1
                totals["corroborated_spellings"] += bool(row["corroborated_roles"])
                totals["training_source_spellings"] += bool(row["primary_locators"])
                totals["source_attested_spellings"] += bool(row.get("attestations"))
                totals["training_source_only_spellings"] += bool(row["primary_locators"]) and not row["corroborated_roles"] and not row.get("attestations")
                totals["source_role_differences"] += row["source_role_difference"]
                for item in row.get("attestations", []):
                    if item["tier"] == "lexical_sharealike": languages[(item["language"], item["language_name"])] += 1
                largest_record = max(largest_record, (len(base.encoded(row)), row["key"]))
                parts = pages.add(row)
                compact = head(row, parts)
                projection.execute("INSERT INTO heads VALUES(?,?,?)", (row["key"], base.encoded(compact).decode("utf-8"), json.dumps(row["roles"])))
                index.write(base.encoded({"key": row["key"], "roles": row["roles"]}))
                if count % 25000 == 0:
                    projection.commit(); print(json.dumps({"stage": "evidence_projected", "keys": count}), flush=True)
    pages.flush(); evidence.close(); projection.commit()
    totals["additional_public_spellings"] = count - 597769
    base.require(totals["spellings"] == expected["expected_public_directory_keys"] and totals["given_role_keys"] == expected["given_role_keys"] and
                 totals["family_role_keys"] == expected["family_role_keys"] and totals["corroborated_spellings"] == 31818 and
                 totals["training_source_only_spellings"] == 94219 and totals["source_attested_spellings"] == 803186 and
                 len({code for code, _ in languages}) == 224, "Frozen public role/evidence/language census differs")
    hot = Segments(target, release, segment_bytes=128 * 1024)
    writers = {role: View(hot, role, target_page_bytes=32 * 1024, page_rows=25) for role in ("all", "given", "family")}
    largest_head = (0, "")
    for key, encoded, roles in projection.execute("SELECT key,data,roles FROM heads ORDER BY key"):
        row = attach_evidence(json.loads(encoded), projection)
        largest_head = max(largest_head, (len(base.encoded(row)), key))
        writers["all"].add(row)
        for role in json.loads(roles): writers[role].add(row)
    views = {role: writer.finish() for role, writer in writers.items()}
    hot.close()
    print(json.dumps({"stage": "heads_indexed", "release": release, "hot_segments": len(hot.files), "evidence_segments": len(evidence.files)}), flush=True)
    # Retained downloads cover old observations. Only new scope events are
    # exported again; each part embeds its own source/license/modification header.
    for source in sources:
        if source["id"] == "namsor-japanese-public-2022":
            continue
        notice = {"type": "source_notice", "source": {k: source[k] for k in ("id", "title", "source_url", "license_label", "license_url", "policy_url", "description")},
                  "modifications": "Additional closed V3 observations only. Exact role/language/DOI or minimized aggregate support retained; frozen normalization, deduplication, compact heads and lossless evidence pagination. Retained earlier download parts remain available. Covered lexical adaptations remain CC-BY-SA 4.0; preserve source/license/modification notices."}
        cursor = catalog.db.execute("SELECT key,data FROM events WHERE source_id=? AND unit_rank>0 ORDER BY key,unit_rank,seq", (source["id"],))
        part = 0; chunk = []; current_key = None; group = []
        def emit(items, number):
            path = private_work / f"source-{source['id']}-{number:03d}.jsonl.gz"
            with path.open("wb") as raw:
                with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0, compresslevel=6) as stream:
                    stream.write(base.encoded(notice))
                    for item in items: stream.write(base.encoded(item))
            write_download(target, release, source, path.name, f"Additional V3 source observations, part {number + 1} (JSONL gzip with notice)", "transformed", path.read_bytes())
        for key, encoded in cursor:
            if current_key is not None and key != current_key:
                chunk.append({"key": current_key, "attestations": group}); group = []
                if len(chunk) == 30000: emit(chunk, part); part += 1; chunk = []
            current_key = key; group.append(json.loads(encoded))
        if current_key is not None: chunk.append({"key": current_key, "attestations": group})
        if chunk: emit(chunk, part)
    for source in sources:
        for script in helpers + [Path(__file__)]:
            write_download(target, release, source, script.name, "V3 transformation source: " + script.name, "modifications", script.read_bytes())
    # Preserve the producer's public notices without publishing its private
    # profile/member audit locators or source snapshots.
    for filename in ("NOTICE.txt", "MODIFICATIONS.json"):
        p = ROOT / "outputs/public-name-expansion-2026-11-01/orcid-continuation-01/packet-02" / filename
        write_download(target, release, sources[-1], "ORCID-" + filename, "ORCID public minimization " + filename, "notice", p.read_bytes())
    manifest = {**old, "schema": SCHEMA, "release": release, "totals": dict(totals), "sources": sources, "views": views,
        "storage": {"transport": "bounded_stream_segments/1", "hot_segments": len(hot.files), "evidence_segments": len(evidence.files),
                    "hot_segment_bytes": 128 * 1024, "evidence_segment_bytes": 512 * 1024,
                    "head_page_inflated_bytes": 32 * 1024, "evidence_page_inflated_bytes": 64 * 1024,
                    "maximum_records": 50, "default_head_records": 25, "maximum_head_data_reads": 1, "maximum_index_reads": 8},
        "record_format": "compact_heads_with_lossless_paged_evidence",
        "lexical_language_attestations": [{"code": code, "name": name, "observations": number} for (code, name), number in sorted(languages.items(), key=lambda x: (x[0][1].casefold(), x[0][0]))],
        "limitations": old["limitations"] + ["ORCID observations are public self-asserted fields with aggregate support, not identity verification. Individual profile identifiers and support locators are not republished.",
            "Lookup heads retain original roles, forms, warnings, source-role/language summaries and corroboration flags. Complete original public fields and every assertion are recoverable through paginated evidence and source downloads. Counts are spelling keys, not people."]}
    base.write_json(target / "manifest.json", manifest, 512 * 1024)
    pointer = stage / "assets/public-name-data/manifest.json"; pointer.write_bytes((target / "manifest.json").read_bytes())
    indexes = compile_indexes(views, target, release)
    generated = stage / "functions/_lib/active-public-name-release.js"; generated.parent.mkdir(parents=True)
    data = json.dumps({"manifest": manifest, "indexes": indexes}, ensure_ascii=True, separators=(",", ":"))
    generated.write_text("// Build-pinned immutable release metadata and validated upper indexes. No visitor state.\nconst freeze=(value)=>{if(value&&typeof value==='object'){Object.freeze(value);for(const child of Object.values(value))freeze(child);}return value;};\nconst data=freeze(" + data + ");\nexport const activeManifest=data.manifest;\nexport const compiledIndexes=data.indexes;\n", "utf-8")
    current_static = [p for p in (ROOT / "site").rglob("*") if p.is_file() and not any(x in p.relative_to(ROOT / "site").parts for x in (".wrangler", "functions"))]
    new_static = [p for p in (stage / "assets").rglob("*") if p.is_file()]
    total_files = len(current_static) + len(new_static) - 1
    base.require(total_files <= 20000 and max(p.stat().st_size for p in new_static) <= 25 * 1024 * 1024, "Actual retained-file/asset budget exceeded")
    result = {"ok": True, "built_at": datetime.now(timezone.utc).isoformat(), "release": release, "schema": SCHEMA,
        "stage": str(stage), "units": units, "code_sha256": code, "totals": dict(totals), "public_lexical_codes": 224,
        "hot_segments": len(hot.files), "evidence_segments": len(evidence.files), "head_pages": {role: writer.page_count for role, writer in writers.items()},
        "index_depth": {role: view["depth"] for role, view in views.items()}, "compiled_index_nodes": len(indexes),
        "compiled_index_bytes": len(base.encoded(indexes)), "generated_module_bytes": generated.stat().st_size,
        "largest_full_record": largest_record, "largest_head": largest_head, "largest_evidence_page_bytes": pages.largest_inflated,
        "evidence_pages": pages.pages, "new_static_files": len(new_static), "total_static_files_including_retained": total_files,
        "new_static_bytes": sum(p.stat().st_size for p in new_static), "manifest_sha256": base.sha(pointer),
        "active_module_sha256": base.sha(generated), "key_role_index": str(key_index), "key_role_index_sha256": base.sha(key_index),
        "fifty_million_achieved": False, "canonical_site_changed": False, "deployed": False,
        "capacity_scope": "Actual current911217 release only; all future expansion remains subject to file/byte/request and real provider CPU/allowance checks."}
    receipt = WORK / f"BUILD_RECEIPT_{release}.json"
    base.require(not receipt.exists(), "Immutable build receipt exists")
    receipt.write_text(json.dumps(result, indent=2) + "\n", "utf-8")
    projection.close(); catalog.close()
    print(json.dumps(result, indent=2), flush=True)
    return result


if __name__ == "__main__":
    build()
