"""Deterministic normalization, hygiene rules, rights gate and source parsers.

Everything here is pure / streaming and has no wall-clock or randomness dependence,
so the same input bytes always produce the same output bytes.

Counting units (see AUDIT_REPORT.md):
  * source row   - one publisher data row. Its shape is source-specific:
                     long sources (SSA, Insee, Statistik Austria, Alberta): name x sex x year;
                     Census lists: one name (per list/sheet);
                     wide sources (Retraite Québec, BC): one name x sex row spanning many
                     year columns. A wide row is ONE source row, not one row per year.
  * year cell    - for wide sources only: one name x sex x year cell. Each cell is
                   numeric (published count), suppressed (e.g. '<5': a positive count
                   withheld by the publisher) or absent (source-specific marker such as
                   Québec '.' / BC '0': not listed / no births that year).
  * attested form - a NORMALIZED source form: NFC, BOM (U+FEFF) removed, internal
                   whitespace runs collapsed to one space, trimmed. It is not byte-exact;
                   the original spelling is recoverable from the hashed raw file + locator.
  * name key     - NFC(casefold(attested form)), i.e. NFC applied again after casefold
                   (casefold can decompose). Case-only variants (MARY / Mary) share a key.
                   Accented and unaccented spellings (Zoé / Zoe), transliterations and
                   different scripts are NOT merged: they are separate keys.

Quantitative evidence: quantity(obs) normalizes every parser observation to a
count_status in {exact, partial, unknown} (parsers that only publish exact counts just set
obs["count"]; wide parsers set the fields below explicitly).
  exact   - obs["count"] is the publisher's complete count for that source row.
  partial - some contributing cells were suppressed; obs["numeric_count_sum"] is only the
            sum of disclosed numeric cells (a lower bound on disclosed values, NOT a total)
            and obs has no "count".
  unknown - no numeric count is published for the row (all relevant cells suppressed, or
            the source publishes no frequency); obs has no "count". Never read as zero.
obs["suppressed_cells"] is the number of suppressed cells in the row.
"""
import csv
import hashlib
import io
import re
import unicodedata
import zipfile

NORMALIZATION_VERSION = "ndr-norm/1"

_WS = re.compile(r"\s+")
_DIGIT = re.compile(r"\d")
# Allowed non-letter characters inside a name: space, hyphen, apostrophes, middle dot,
# ZWJ/ZWNJ (needed by several Indic scripts). Combining marks are allowed (category M*).
_ALLOWED_PUNCT = set(" -'’ʼʻ·‌‍")
_PLACEHOLDER = re.compile(
    r"^(_|all other|total\b|unknown\b|not stated|name not|no name|inconnu|sans pr|unbekannt|\(s\)$)",
    re.IGNORECASE,
)
MAX_LEN = 60


def sha256_bytes(b):
    return hashlib.sha256(b).hexdigest()


def sha256_file(path):
    h = hashlib.sha256()
    with open(path, "rb") as f:
        for chunk in iter(lambda: f.read(1 << 20), b""):
            h.update(chunk)
    return h.hexdigest()


def normalize_form(raw):
    """NFC + trim + collapse internal whitespace. Returns '' for empty input."""
    if raw is None:
        return ""
    s = unicodedata.normalize("NFC", str(raw))
    s = s.replace("﻿", "")
    s = _WS.sub(" ", s).strip()
    return s


def name_key(form):
    """Case-insensitive identity key: NFC(casefold(NFC(form)))."""
    return unicodedata.normalize("NFC", normalize_form(form).casefold())


def bucket_of(key, n_buckets):
    """Stable bucket index (blake2b, not Python's salted hash())."""
    d = hashlib.blake2b(key.encode("utf-8"), digest_size=8).digest()
    return int.from_bytes(d, "big") % n_buckets


def hygiene_reason(form):
    """Return None if the form passes the name-shape rules, else a reason code.

    These rules only reject strings that cannot be a personal-name form as published
    (aggregate rows, numerals, punctuation-only, overlong strings) plus one conservative
    inclusion policy: a single cased letter is not counted (reason code kept as the
    legacy label 'single_letter_initial'). That is a counting choice, NOT a finding that
    the publisher's entry is an initial - e.g. Retraite Québec publishes one-letter
    entries. Rejected rows stay listed in excluded/<source>.tsv with their locators.
    Passing these rules does NOT make a string a verified name.
    """
    if not form:
        return "empty"
    if _PLACEHOLDER.search(form):
        return "placeholder_or_aggregate_row"
    if len(form) > MAX_LEN:
        return "overlong"
    if _DIGIT.search(form):
        return "contains_digit"
    letters = 0
    for ch in form:
        cat = unicodedata.category(ch)
        if cat.startswith("L"):
            letters += 1
        elif cat.startswith("M"):
            continue
        elif ch in _ALLOWED_PUNCT:
            continue
        else:
            return "disallowed_character"
    if letters == 0:
        return "no_letters"
    if letters == 1:
        ch = next(c for c in form if unicodedata.category(c).startswith("L"))
        # Policy: a lone cased letter (Latin/Greek/Cyrillic...) is excluded from counts because
        # it is commonly an initial and cannot be told apart from one; this is not proof
        # that the source entry is an initial. Single uncased letters (e.g. one Han
        # character) can be complete given names and are kept.
        if ch.lower() != ch.upper():
            return "single_letter_initial"
    if form[0] in "-'’" or form[-1] in "-":
        return "edge_punctuation"
    tokens = form.split(" ")
    if len(tokens) > 1 and any(len(t) == 1 and t.lower() != t.upper() and t.casefold() not in _PARTICLES for t in tokens):
        # "A ALEXANDER", "J Smith": a person-record fragment with an initial, not a name form.
        return "initial_token"
    return None


# One-letter words that are genuine name particles/conjunctions (Spanish y/e, Irish Ó, Catalan d...).
_PARTICLES = {"ó", "o", "y", "e", "i", "d", "á", "à"}


def display_form(forms_with_support):
    """Pick a deterministic display form from {form: n_sources}.

    Prefer forms that are not ALL-CAPS (publishers such as Census/Insee/Québec publish
    upper case only), then higher source support, then code-point order.
    """
    def rank(item):
        form, support = item
        all_caps = form == form.upper() and form != form.lower()
        return (all_caps, -support, form)

    return sorted(forms_with_support.items(), key=rank)[0][0]


# ---------------------------------------------------------------- rights gate

REQUIRED_SOURCE_FIELDS = ("source_id", "publisher", "category", "role", "files", "parser", "terms_url", "terms_statement", "attribution")
TIER1_CATEGORIES = {"official_aggregate", "official_name_list"}


def tier1_eligibility(src):
    """Return (eligible, [reasons]). Tier 1 = official publisher, documented terms,
    raw bytes held locally with a recorded URL and verified hash."""
    reasons = []
    for f in REQUIRED_SOURCE_FIELDS:
        if not src.get(f):
            reasons.append(f"missing_field:{f}")
    if src.get("category") not in TIER1_CATEGORIES:
        reasons.append(f"category_not_tier1:{src.get('category')}")
    if src.get("rights_block"):
        reasons.append("rights_block")
    for item in src.get("files", []):
        if not item.get("url"):
            reasons.append(f"file_without_url:{item.get('local_name')}")
    return (not reasons, reasons)


# ---------------------------------------------------------------- parsers
# Each parser yields (raw_form, locator, obs) where obs is a dict with optional
# keys count, year, year_min, year_max, sex, and (wide parsers) count_status,
# numeric_count_sum, suppressed_cells.  locator is "<member-or-sheet>:<1-based line/row>".

COUNT_STATUSES = ("exact", "partial", "unknown")


def quantity(obs):
    """Return (count_status, numeric_count_sum, suppressed_cells, exact_count_or_None).

    'count' in obs always means a complete publisher count. Without it the count is not
    zero - it is partial (some cells suppressed) or unknown (nothing numeric published).
    """
    status = obs.get("count_status")
    if status is None:
        if obs.get("count") is not None:
            return "exact", int(obs["count"]), 0, int(obs["count"])
        return "unknown", 0, int(obs.get("suppressed_cells", 0)), None
    if status not in COUNT_STATUSES:
        raise ValueError(f"bad count_status {status!r}")
    exact = obs.get("count") if status == "exact" else None
    if status == "exact" and exact is None:
        raise ValueError("count_status 'exact' without count")
    if status != "exact" and obs.get("count") is not None:
        raise ValueError(f"count_status {status!r} must not carry a complete count")
    return status, int(obs.get("numeric_count_sum", 0)), int(obs.get("suppressed_cells", 0)), exact


def combine_status(statuses):
    """Aggregate per-row statuses: exact only if every row is exact; unknown only if no
    row disclosed any numeric value; otherwise partial."""
    st = set(statuses)
    if not st or st == {"exact"}:
        return "exact"
    if st == {"unknown"}:
        return "unknown"
    return "partial"

def parse_ssa_national_zip(paths):
    with zipfile.ZipFile(paths[0]) as z:
        members = sorted(n for n in z.namelist() if re.fullmatch(r"yob\d{4}\.txt", n))
        for m in members:
            year = int(m[3:7])
            with z.open(m) as fh:
                for i, line in enumerate(io.TextIOWrapper(fh, encoding="utf-8", newline=""), 1):
                    line = line.rstrip("\r\n")
                    if not line:
                        continue
                    name, sex, count = line.split(",")
                    yield name, f"{m}:{i}", {"count": int(count), "year": year, "sex": sex}


def parse_census2010_zip(paths):
    with zipfile.ZipFile(paths[0]) as z:
        with z.open("Names_2010Census.csv") as fh:
            rdr = csv.reader(io.TextIOWrapper(fh, encoding="utf-8", newline=""))
            header = next(rdr)
            assert header[0] == "name", header
            for i, row in enumerate(rdr, 2):
                if not row:
                    continue
                yield row[0], f"Names_2010Census.csv:{i}", {"count": int(row[2]), "year": 2010}


def _xlsx_rows(path):
    import openpyxl  # imported lazily so tests without xlsx do not need it

    wb = openpyxl.load_workbook(path, read_only=True, data_only=True)
    try:
        for ws in wb.worksheets:
            for i, row in enumerate(ws.iter_rows(values_only=True), 1):
                yield ws.title, i, row
    finally:
        wb.close()


def parse_census2020_xlsx(paths):
    for p in paths:
        base = p.replace("\\", "/").rsplit("/", 1)[-1]
        started = False
        for sheet, i, row in _xlsx_rows(p):
            if not started:
                if row and isinstance(row[0], str) and row[0].strip().upper() in ("FIRST NAME", "LAST NAME"):
                    started = True
                continue
            if not row or row[0] is None or not isinstance(row[1], (int, float)):
                continue  # blank or footnote rows
            yield str(row[0]), f"{base}!{sheet}:{i}", {"count": int(row[2]), "year": 2020}


def parse_insee_nat_zip(paths):
    with zipfile.ZipFile(paths[0]) as z:
        member = [n for n in z.namelist() if n.endswith(".csv")][0]
        with z.open(member) as fh:
            rdr = csv.reader(io.TextIOWrapper(fh, encoding="utf-8", newline=""), delimiter=";")
            header = next(rdr)
            assert header == ["sexe", "preusuel", "annais", "nombre"], header
            for i, row in enumerate(rdr, 2):
                if not row:
                    continue
                sexe, name, year, count = row
                obs = {"count": int(count), "sex": {"1": "M", "2": "F"}.get(sexe, sexe)}
                if year.isdigit():
                    obs["year"] = int(year)
                yield name, f"{member}:{i}", obs


def parse_at_vornamen_csv(paths):
    with open(paths[0], encoding="utf-8", newline="") as fh:
        rdr = csv.reader(fh, delimiter=";")
        header = next(rdr)
        idx = header.index("F-VORNAME_NORMALISIERT")
        for i, row in enumerate(rdr, 2):
            if not row:
                continue
            yield row[idx], f"OGDEXT_VORNAMEN_1.csv:{i}", {
                "count": int(row[4]), "year": int(row[0]), "sex": {"1": "M", "2": "F"}.get(row[2], row[2])}


_SUPPRESSED = re.compile(r"^<\s*\d+$")


def _wide_year_csv(paths, absent_values, sexes):
    """One source row = one name x sex across many year columns.

    Cell classes: digits -> published count; '<N' -> suppressed positive count (value
    withheld, NOT zero); a source-specific absent marker -> not listed that year. Any
    other cell value raises (fail closed rather than guess). Non-year columns (e.g. BC's
    ' TOTAL') are never summed; when a TOTAL column exists and the row is fully numeric
    it must equal the sum of year cells.
    """
    for p, sex in zip(paths, sexes):
        base = p.replace("\\", "/").rsplit("/", 1)[-1]
        with open(p, encoding="utf-8-sig", newline="") as fh:
            rdr = csv.reader(fh)
            header = next(rdr)
            cols = [(j, h.strip()) for j, h in enumerate(header) if j > 0]
            year_cols = [(j, int(h)) for j, h in cols if h.isdigit()]
            total_cols = [j for j, h in cols if h.upper() == "TOTAL"]
            other = [h for j, h in cols if not h.isdigit() and h.upper() != "TOTAL"]
            if other:
                raise ValueError(f"{base}: unexpected non-year columns {other!r}")
            for i, row in enumerate(rdr, 2):
                if not row or not row[0].strip():
                    continue
                loc = f"{base}:{i}"
                numeric = suppressed = 0
                present = []
                for j, year in year_cols:
                    v = row[j].strip() if j < len(row) else ""
                    if v in absent_values:
                        continue
                    if v.isdigit():
                        numeric += int(v)
                    elif _SUPPRESSED.match(v):
                        suppressed += 1
                    else:
                        raise ValueError(f"{loc}: unrecognized cell {v!r} for {year}")
                    present.append(year)
                if suppressed == 0:
                    status = "exact"
                elif numeric > 0:
                    status = "partial"
                else:
                    status = "unknown"
                obs = {"sex": sex, "count_status": status, "numeric_count_sum": numeric,
                       "suppressed_cells": suppressed}
                if status == "exact":
                    obs["count"] = numeric
                    for j in total_cols:
                        t = row[j].strip() if j < len(row) else ""
                        if not t.isdigit() or int(t) != numeric:
                            raise ValueError(f"{loc}: TOTAL {t!r} != sum of year cells {numeric}")
                if present:
                    obs["year_min"], obs["year_max"] = min(present), max(present)
                yield row[0], loc, obs


def parse_qc_wide_csv(paths):
    # grande_listeg = garçons (M), grande_listef = filles (F).
    # '.' = not listed that year (absent); '<5' = given to 1-4 children (suppressed, positive).
    return _wide_year_csv(paths, {".", ""}, ["M", "F"])


def parse_bc_wide_csv(paths):
    # BC publishes '0' for years without a listed count (absent). The retained files contain
    # no '<5' cells; if one appears it is treated as suppressed positive evidence.
    return _wide_year_csv(paths, {"0", ""}, ["M", "F"])


def parse_generic_xlsx_name_column(paths):
    for p in paths:
        base = p.replace("\\", "/").rsplit("/", 1)[-1]
        col = None
        for sheet, i, row in _xlsx_rows(p):
            if col is None:
                if row and "First Name" in [str(c).strip() if c is not None else "" for c in row]:
                    col = [str(c).strip() if c is not None else "" for c in row]
                continue
            if not row or row[col.index("First Name")] is None:
                continue
            obs = {}
            if "Frequency" in col and isinstance(row[col.index("Frequency")], (int, float)):
                obs["count"] = int(row[col.index("Frequency")])
            if "Year" in col and isinstance(row[col.index("Year")], (int, float)):
                obs["year"] = int(row[col.index("Year")])
            if "Gender" in col and row[col.index("Gender")]:
                obs["sex"] = {"Boy": "M", "Girl": "F"}.get(str(row[col.index("Gender")]).strip(), str(row[col.index("Gender")]).strip())
            yield str(row[col.index("First Name")]), f"{base}!{sheet}:{i}", obs


PARSERS = {
    "ssa_national_zip": parse_ssa_national_zip,
    "census2010_zip": parse_census2010_zip,
    "census2020_xlsx": parse_census2020_xlsx,
    "insee_nat_zip": parse_insee_nat_zip,
    "at_vornamen_csv": parse_at_vornamen_csv,
    "qc_wide_csv": parse_qc_wide_csv,
    "bc_wide_csv": parse_bc_wide_csv,
    "generic_xlsx_name_column": parse_generic_xlsx_name_column,
}
