"""Publish only completed, admitted source observations; no acquisition or deployment.

Shared terms are kept in the source manifest. Per-record attestations retain exact
roles, DOI fields or primary revisions. Existing training evidence is unchanged.
"""
import argparse
from collections import Counter
from datetime import datetime, timezone
import gzip
import hashlib
from itertools import chain
import json
import os
from pathlib import Path
import shutil
import unicodedata

import build_public_name_directory as base

ROOT = base.ROOT
WORK = ROOT / "outputs/public-name-expansion-2026-11-01/public-expansion-v2"
DATA = ROOT / "site/assets/public-name-data"
SCHEMA = "ndr-public-directory/2"
MAX_ASSET = 25 * 1024 * 1024


def jsonl(path):
    with gzip.open(path, "rt", encoding="utf-8") as stream:
        for line in stream:
            yield json.loads(line)


def pack(path, rows):
    with path.open("wb") as raw:
        with gzip.GzipFile(filename="", mode="wb", fileobj=raw, mtime=0, compresslevel=6) as stream:
            for row in rows:
                stream.write(base.encoded(row))


def witness(item):
    # No abstracts, contact details, resource contents, local paths or other DOI fields.
    result = {key: item[key] for key in ("doi", "field", "form", "role", "member", "line")}
    base.require(isinstance(result["doi"], str) and result["doi"].startswith("10.") and
                 result["role"] in {"given", "family"} and isinstance(result["line"], int) and
                 result["line"] > 0 and result["field"].endswith(".givenName" if result["role"] == "given" else ".familyName"), "Invalid DOI witness")
    return result


def event(row, cohort, normalizer):
    role, key, form, proof = row["role"], row["key"], row["form"], row["provenance"]
    base.require(role in {"given", "family"} and normalizer.name_key(form) == key and
                 normalizer.hygiene_reason(normalizer.normalize_form(form)) is None, "Non-admitted source spelling")
    if row["tier"] == "bibliographic":
        base.require(proof["license"] == "CC0-1.0" and row["language"] is None, "Unexpected DataCite terms/language")
        witnesses = [witness(item) for item in proof["support_witnesses"][:3]]
        base.require(isinstance(proof["distinct_doi_support"], int) and proof["distinct_doi_support"] >= 2 and
                     len({item["doi"] for item in witnesses}) >= 2 and
                     all(item["role"] == role and normalizer.name_key(item["form"]) == key for item in witnesses), "Missing DOI support")
        scope = proof.get("source_scope_id", cohort)
        base.require(scope == cohort or (cohort == "datacite-first1000-cohort0" and scope.startswith(cohort + ":")), "Unexpected source scope")
        return {"source_id": "datacite-public-2025", "cohort": scope, "tier": "bibliographic", "role": role,
                "form": form, "language": None, "distinct_doi_support": proof["distinct_doi_support"],
                "witnesses": witnesses}
    base.require(row["tier"] == "lexical_sharealike" and proof["license"] == "CC-BY-SA-4.0" and
                 proof["literal_primary_title_not_generated"] is True and proof["page_title"] == form and
                 isinstance(row["language"], str) and base.safe_url(proof["source_url"]) and
                 base.safe_url(proof["history_url"]), "Unexpected lexical source evidence")
    evidence = [{k: item[k] for k in ("definition_line", "definition_line_number", "language_heading", "language_heading_line", "template_language", "template_name")}
                for item in proof["primary_role_and_language_evidence"]]
    base.require(evidence and all(item["template_language"] == row["language"] and
                 item["template_name"] == ("surname" if role == "family" else "given name") for item in evidence), "Role/language assertion differs")
    return {"source_id": "wiktionary-en-primary", "cohort": cohort, "tier": "lexical_sharealike", "role": role,
            "form": form, "language": row["language"], "language_name": row["language_name"],
            "page_title": proof["page_title"], "page_id": proof["page_id"], "revision_id": proof["revision_id"],
            "revision_timestamp": proof["revision_timestamp"], "source_url": proof["source_url"],
            "history_url": proof["history_url"], "wikitext_sha256": proof["primary_wikitext_sha256"],
            "primary_evidence": evidence}


def source_definitions(old):
    return old["sources"] + [
        {"id": "datacite-public-2025", "title": "DataCite public-2025 personal-name metadata",
         "source_url": "https://datafiles.datacite.org/datafiles/public-2025",
         "license_label": "CC0 1.0 public-domain dedication for deposited DOI metadata",
         "license_url": "https://creativecommons.org/publicdomain/zero/1.0/",
         "policy_url": "https://support.datacite.org/docs/datacite-data-file-use-policy",
         "snapshot_sha256": "678cbb64d04f24a8793f3b205ab18e5421dfdd20b89faa67fa62cce7453ec006",
         "description": "Admitted givenName/familyName fields from completed 200-member and first-1,000-member cohorts. At least two distinct DOI records support each admitted source role within its cohort. DOI counts are not counts of people or independent identity verification. Only names and exact field locators are republished; linked resources, abstracts and contact details are excluded. Language and origin are unknown.",
         "downloads": []},
        {"id": "wiktionary-en-primary", "title": "Wiktionary contributors — exact primary revisions",
         "source_url": "https://en.wiktionary.org/", "license_label": "Creative Commons Attribution-ShareAlike 4.0 International",
         "license_url": "https://creativecommons.org/licenses/by-sa/4.0/",
         "policy_url": "https://foundation.wikimedia.org/wiki/Policy:Terms_of_Use#7._Licensing_of_Content",
         "description": "Wiktionary contributors are credited beside every lexical observation through its exact revision and contributor-history links. Kaikki/Wiktextract supplied discovery candidates; only literal primary titles with matching explicit given-name/surname templates and language headings were admitted. Source-asserted language is an attestation, not an origin or native-script claim. NDR extracted fields, normalized spelling keys and deduplicated records; covered lexical adaptations and NDR contributions to them are offered under CC-BY-SA 4.0. Original roles and evidence text are retained; no meanings, translations or gender claims are added.",
         "downloads": []}]


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--cohort0", required=True)
    parser.add_argument("--cohort0-sha256", required=True)
    args = parser.parse_args()
    base.require(not (WORK / "BUILD_RECEIPT.json").exists(), "Immutable build receipt exists")
    expected_scope = json.loads((ROOT / "outputs/public-name-expansion-2026-11-01/PUBLIC_EXPANSION_EXPECTED_SCOPE_AUDIT_V2.json").read_text("utf-8"))
    base.require(expected_scope["ok"] is True and expected_scope["public_directory_expected_keys"] == 597769 and
                 expected_scope["running_cohort1_counted"] is False, "Independent frozen scope audit is missing")
    expected_hashes = {str(Path(item["path"]).resolve()): item["sha256"] for item in expected_scope["input_hashes"]}
    old = json.loads((WORK / "baseline/site/assets/public-name-data/manifest.json").read_text("utf-8"))
    base.require(old["release"] == "3daa61530faeb8a4f7259b1a", "Unexpected frozen public baseline")
    base.require(old["normalization"]["unicode_version"] == unicodedata.unidata_version and
                 old["normalization"]["policy_sha256"] == base.POLICY_SHA, "Unicode or frozen normalization differs from the published search map")
    normalizer = base.policy()
    training = next(item for item in old["sources"][0]["downloads"] if item["kind"] == "transformed")
    training_path = ROOT / "site" / training["url"].lstrip("/")
    base.require(base.sha(training_path) == training["sha256"], "Published baseline changed")
    rows = {row["key"]: row for row in jsonl(training_path)}
    base.require(len(rows) == 135249, "Published baseline count changed")
    baseline_keys = set(rows)
    inputs = [
        (ROOT / "outputs/public-name-expansion-2026-11-01/datacite/packet-02/ELIGIBLE.jsonl.gz", "datacite-current200"),
        (Path(args.cohort0).resolve(), "datacite-first1000-cohort0"),
        (ROOT / "outputs/public-name-expansion-2026-11-01/kaikki-continuation-01/packet-01/NAMES.jsonl.gz", "dictionary-tranche01"),
        (ROOT / "outputs/public-name-expansion-2026-11-01/kaikki-continuation-02/packet-01/NAMES.jsonl.gz", "dictionary-tranche02")]
    input_proofs = []
    event_counts = Counter()
    for path, cohort in inputs:
        fingerprint = base.sha(path)
        base.require(fingerprint == (args.cohort0_sha256 if cohort == "datacite-first1000-cohort0" else expected_hashes.get(str(path.resolve()))), "Pinned admitted input changed")
        count = 0
        for row in jsonl(path):
            attestation = event(row, cohort, normalizer)
            key = row["key"]
            if key not in rows:
                rows[key] = {"key": key, "name": row["form"], "forms": [], "roles": [], "source_id": None,
                             "source_category": "public_source_attestations", "evidence_status": "source_attested",
                             "corroborated_roles": [], "role_warnings": [], "language": None,
                             "written_script": base.written_script(row["form"]), "primary_locators": [], "corroboration": []}
            item = rows[key]
            item["forms"] = sorted(set(item["forms"] + [row["form"]]))
            item["roles"] = sorted(set(item["roles"] + [row["role"]]))
            item.setdefault("attestations", []).append(attestation)
            count += 1
        event_counts[cohort] = count
        input_proofs.append({"path": str(path.relative_to(ROOT)), "sha256": fingerprint, "admitted_events": count})
        print(json.dumps({"stage": "source_projected", "cohort": cohort, "events": count, "global_keys": len(rows)}), flush=True)
    records = sorted(rows.values(), key=lambda item: item["key"])
    base.require(len(records) == 597769 and sum("given" in row["roles"] for row in records) == 171694 and
                 sum("family" in row["roles"] for row in records) == 479513, "Public union differs from independent frozen-input scope audit")
    base.require(dict(event_counts) == {"datacite-current200": 87372, "datacite-first1000-cohort0": 1082627,
                 "dictionary-tranche01": 9849, "dictionary-tranche02": 9558}, "Completed admitted event scope changed")
    pack(WORK / "KEY_ROLE_INDEX.jsonl.gz", ({"key": row["key"], "roles": row["roles"]} for row in records))
    for row in records:
        source_roles = {}
        if row["primary_locators"]:
            source_roles[row["source_id"]] = {item["role"] for item in row["primary_locators"]}
        for item in row.get("attestations", []):
            source_roles.setdefault(item["source_id"], set()).add(item["role"])
        row["source_role_difference"] = len({tuple(sorted(value)) for value in source_roles.values()}) > 1
    largest_record_bytes, largest_record_key = max((len(base.encoded(row)) - 1, row["key"]) for row in records)
    base.require(largest_record_bytes <= 192 * 1024, "A complete public record exceeds the API's individual response budget; split its evidence into bounded public detail assets before publication")
    fixtures = {}
    for row in records:
        for label, matches in [
            ("bibliographic", any(a["tier"] == "bibliographic" for a in row.get("attestations", []))),
            ("primary_lexical", any(a["tier"] == "lexical_sharealike" for a in row.get("attestations", []))),
            ("source_role_difference", row["source_role_difference"]),
            ("retained_training_corroboration", bool(row["corroborated_roles"]))]:
            if matches and label not in fixtures:
                fixtures[label] = row
        if len(fixtures) == 4:
            break
    (WORK / "EVIDENCE_FIXTURES.json").write_bytes(base.encoded(fixtures))
    release = hashlib.sha256(base.encoded({"schema": SCHEMA, "inputs": input_proofs, "baseline": old["release"],
        "builder": base.sha(__file__), "policy": base.POLICY_SHA})).hexdigest()[:24]
    target = WORK / "staging" / release
    base.require(not target.exists() and not (DATA / release).exists(), "Immutable release already exists")
    target.mkdir(parents=True)
    sources = source_definitions(old)
    def download(path, label, kind, source):
        filename = path.name
        dest = target / "downloads" / filename
        dest.parent.mkdir(exist_ok=True)
        if path != dest:
            shutil.copyfile(path, dest)
        base.require(dest.stat().st_size <= MAX_ASSET, "Source download exceeds Pages asset cap")
        source["downloads"].append({"kind": kind, "label": label, "original_filename": filename,
            "url": f"/assets/public-name-data/{release}/downloads/{filename}", "bytes": dest.stat().st_size, "sha256": base.sha(dest)})
    notices = [
        ("DATACITE-NOTICE.txt", "CC0 metadata only. Policy: https://support.datacite.org/docs/datacite-data-file-use-policy\nLicense: https://creativecommons.org/publicdomain/zero/1.0/\nNDR extracts admitted names and exact DOI fields, applies frozen normalization and deduplicates keys. Distinct DOI support is not distinct-person support. No linked resources, abstracts, email addresses or generated names are included. Privacy, publicity, trademark and other non-copyright rights are not waived by this metadata publication.\n", sources[1]),
        ("WIKTIONARY-NOTICE.txt", "Wiktionary contributors, attributed through each exact revision and history URL.\nCreative Commons Attribution-ShareAlike 4.0 International: https://creativecommons.org/licenses/by-sa/4.0/\nKaikki/Wiktextract discovery, primary Wiktionary revision corroboration. NDR changes: field extraction, NFC/BOM/whitespace normalization, casefold keys, explicit role/language selection, deduplication and source relationships. Covered lexical adaptations and NDR contributions are CC-BY-SA 4.0; retain source attribution, license and modification notices. Exact primary source titles, language headings and role templates remain linked and downloadable. No generated forms, inferred meanings, origins, genders, translations or native-script claims. This license does not relicense the separately labelled training-source material or DataCite metadata.\n", sources[2])]
    for name, content, source in notices:
        path = WORK / name; path.write_text(content, "utf-8"); download(path, name, "notice", source)
    for tranche in ("01", "02"):
        source_path = ROOT / f"outputs/public-name-expansion-2026-11-01/kaikki-continuation-{tranche}/packet-01/NOTICE.txt"
        path = WORK / f"dictionary-tranche{tranche}-original-NOTICE.txt"
        shutil.copyfile(source_path, path)
        download(path, f"Original dictionary tranche {tranche} notice", "notice", sources[2])
    # Separate source exports preserve source licenses rather than assigning one
    # blanket license to a mixed-source collection. All records also remain in shards.
    for source in sources[1:]:
        notice = {"schema": "ndr-public-source-observations/1", "type": "source_notice",
                  "source": {key: source[key] for key in ("id", "title", "source_url", "license_label", "license_url", "policy_url", "description")},
                  "format": "First line is this source/license/modification notice; subsequent lines contain a spelling key and its source-specific attestations. Preserve this notice with the covered observations."}
        selected = ({"key": row["key"], "attestations": [a for a in row.get("attestations", []) if a["source_id"] == source["id"]]}
                    for row in records if any(a["source_id"] == source["id"] for a in row.get("attestations", [])))
        # Bounded row-count chunks keep every downloadable asset below 25 MiB.
        chunk = []; part = 0
        for row in selected:
            chunk.append(row)
            if len(chunk) == 40000:
                path = WORK / f'{source["id"]}-{part:03d}.jsonl.gz'; pack(path, chain([notice], chunk))
                download(path, f"Transformed observations with source/license header, part {part + 1} (JSONL gzip)", "transformed", source)
                chunk = []; part += 1
        if chunk:
            path = WORK / f'{source["id"]}-{part:03d}.jsonl.gz'; pack(path, chain([notice], chunk))
            download(path, f"Transformed observations with source/license header, part {part + 1} (JSONL gzip)", "transformed", source)
        print(json.dumps({"stage": "source_downloads_packed", "source": source["id"], "parts": part + bool(chunk)}), flush=True)
    download(Path(__file__), "Transformation source (Python)", "modifications", sources[1])
    download(Path(__file__), "Transformation source (Python)", "modifications", sources[2])
    for script, label in [(ROOT / "scripts_build/build_public_name_directory.py", "Shared sharding/normalization loader (Python)"),
                          (base.POLICY, "Frozen spelling normalization and hygiene source (Python)")]:
        download(script, label, "modifications", sources[1])
        download(script, label, "modifications", sources[2])
    views = {}
    for role in ("all", "given", "family"):
        selected = records if role == "all" else [row for row in records if role in row["roles"]]
        buckets = []
        for bucket_number, start in enumerate(range(0, len(selected), base.BUCKET_ROWS)):
            group = selected[start:start + base.BUCKET_ROWS]; pages = []
            for number, page in enumerate(base.page_groups(group, release)):
                relative = f"{role}/{bucket_number:06d}/{number:05d}.json.gz"
                raw = base.encoded({"schema": SCHEMA, "release": release, "records": page})
                base.require(len(raw) <= base.PAGE_BYTES, "Decompressed shard exceeds the request read bound")
                packed = gzip.compress(raw, compresslevel=6, mtime=0)
                base.require(len(packed) <= base.PAGE_BYTES, "Compressed shard exceeds the request read bound")
                path = target / relative; path.parent.mkdir(parents=True, exist_ok=True); path.write_bytes(packed)
                fingerprint = {"bytes": len(packed), "sha256": hashlib.sha256(packed).hexdigest()}
                pages.append({"path": f"/assets/public-name-data/{release}/{relative}", "first": page[0]["key"], "last": page[-1]["key"], "count": len(page), **fingerprint})
            relative = f"{role}/{bucket_number:06d}/index.json"
            fingerprint = base.write_json(target / relative, {"schema": SCHEMA, "release": release, "role": role, "pages": pages}, base.INDEX_BYTES)
            buckets.append({"path": f"/assets/public-name-data/{release}/{relative}", "first": group[0]["key"], "last": group[-1]["key"], "count": len(group), **fingerprint})
        views[role] = {"count": len(selected), "buckets": buckets}
        print(json.dumps({"stage": "view_sharded", "role": role, "records": len(selected), "buckets": len(buckets)}), flush=True)
    totals = {"spellings": len(records), "role_keys": sum(len(row["roles"]) for row in records),
              "given_role_keys": views["given"]["count"], "family_role_keys": views["family"]["count"],
              "corroborated_spellings": old["totals"]["corroborated_spellings"],
              "training_source_only_spellings": sum(bool(row["primary_locators"]) and not row["corroborated_roles"] and not row.get("attestations") for row in records),
              "source_attested_spellings": sum(bool(row.get("attestations")) for row in records),
              "additional_public_spellings": len(records) - len(baseline_keys), "training_source_spellings": len(baseline_keys),
              "source_role_differences": sum(row["source_role_difference"] for row in records)}
    base.require(totals["source_attested_spellings"] == expected_scope["keys_with_new_qualified_bibliographic_or_lexical_evidence"], "Qualified public source-key count differs from independent audit")
    languages = Counter((a["language"], a["language_name"]) for row in records for a in row.get("attestations", [])
                        if a["tier"] == "lexical_sharealike")
    manifest = {**old, "schema": SCHEMA, "release": release, "totals": totals, "sources": sources, "views": views,
        "lexical_language_attestations": [{"code": code, "name": name, "observations": count}
            for (code, name), count in sorted(languages.items(), key=lambda item: (item[0][1].casefold(), item[0][0]))],
        "evidence_counts": event_counts, "limitations": [
            "Spellings are deduplicated globally. Counts are source spellings, not people, populations or a global census.",
            "Training-source observations, same-role checkpoint corroboration, DOI-field support and primary lexical role/language evidence are distinct labels.",
            "DataCite evidence is a source-declared personal givenName/familyName field supported by at least two distinct DOI records within its stated cohort. Language is unknown; DOI support is not identity verification.",
            "Lexical language labels are explicit assertions in the linked Wiktionary revision. They do not establish origin, meaning, gender or a native-script translation.",
            "Each source retains its role labels. Different roles across sources are visible; they are not silently swapped or treated as errors.",
            "No meaning, origin, gender, suitability or popularity is inferred. Character-script labels describe only the written form. Attribution does not imply endorsement."]}
    base.write_json(target / "manifest.json", manifest, base.MANIFEST_BYTES)
    staged = list(target.rglob("*")); staged_files = [p for p in staged if p.is_file()]
    # Count the actual static publication tree, including every retained release.
    existing = [p for p in (ROOT / "site").rglob("*") if p.is_file() and not any(x in p.relative_to(ROOT / "site").parts for x in (".wrangler", "functions"))]
    total_files = len(existing) + len(staged_files)
    base.require(total_files <= 20000, f"Free Pages total file cap exceeded: {total_files}")
    base.require(max(p.stat().st_size for p in staged_files) <= MAX_ASSET, "Pages asset size cap exceeded")
    os.replace(target, DATA / release)
    shutil.copyfile(DATA / release / "manifest.json", DATA / "manifest.next.json")
    os.replace(DATA / "manifest.next.json", DATA / "manifest.json")
    receipt = {"ok": True, "built_at": datetime.now(timezone.utc).isoformat(), "schema": SCHEMA, "release": release,
        "totals": totals, "inputs": input_proofs, "event_counts": event_counts, "retained_baseline_release": old["release"],
        "new_release_files": len(staged_files), "total_static_files_including_retained_releases": total_files,
        "new_release_bytes": sum(p.stat().st_size for p in (DATA / release).rglob("*") if p.is_file()),
        "manifest_sha256": base.sha(DATA / "manifest.json"), "builder_sha256": base.sha(__file__),
        "key_role_index_sha256": base.sha(WORK / "KEY_ROLE_INDEX.jsonl.gz"),
        "evidence_fixtures_sha256": base.sha(WORK / "EVIDENCE_FIXTURES.json"),
        "largest_inflated_record_bytes": largest_record_bytes, "largest_record_key": largest_record_key,
        "limits": {"plan": "Pages Free", "file_cap": 20000, "asset_bytes_cap": MAX_ASSET,
                   "data_shard_stored_and_decompressed_bytes_cap": base.PAGE_BYTES, "data_format": "deterministic-gzip-json",
                   "response_records_cap": 50, "data_reads_cap": 4},
        "fifty_million_achieved": False, "deployed": False}
    (WORK / "BUILD_RECEIPT.json").write_text(json.dumps(receipt, indent=2) + "\n", "utf-8")
    print(json.dumps(receipt, indent=2))


if __name__ == "__main__":
    main()
