"""Finish packaging from an unchanged, complete data/index stage.

Attempt01's source/data/index generation completed. A Windows 265-character
download path failed. Preserve that whole attempt; verify/copy its exact gzip
segments into fresh s2 staging and finish only metadata/download packaging with
short filenames. No source replay, admission or name/evidence regeneration.
"""
from collections import Counter
from datetime import datetime,timezone
import gzip,hashlib,json,shutil,sqlite3,sys,zipfile,zlib
from pathlib import Path
from freeze_inputs import ROOT,WORK,SOURCE,BASE_RELEASE,read,rows,sha,write,require
from source_common import encoded
from catalog_v4 import Catalog
from build_v4_attempt01 import public_sources,compile_indexes
sys.path.insert(0,str(ROOT/'scripts_build'))
from public_name_v3_projection import recover

def main():
    snapshot=read(WORK/'DATA_STAGE_COMPLETION_SNAPSHOT.json');release=snapshot['release']
    require(sha(WORK/'build_v4.py')==sha(WORK/'build_v4_attempt01.py')==snapshot['original_writer_sha256'],'Original failed writer changed')
    require(snapshot['hot_segments']==978 and snapshot['evidence_segments']==997 and snapshot['all_key_role_export_lines_equal_exact_census']==1211234,'Original completed stage differs')
    expected=read(WORK/'EXPECTED_SCOPE.json');closed=read(WORK/'CATALOG_CLOSED_RECEIPT.json')
    require(closed['ok'] and sha(WORK/'private-catalog/catalog.sqlite3')==closed['catalog_sha256'],'Closed source catalog changed')
    require(sha(SOURCE/'datacite/CHECKPOINT10_UNIFORM_04_07_VERIFICATION.json')=='4e9e2557396600230a7e5f861b30cae44667a1e85aaa115b718a7197bc6e2bfc','Closed source verification differs')
    stage=WORK/'s3'/release;target=stage/'assets/public-name-data'/release;private=WORK/'packaging-attempt03'/release
    require(not stage.exists() and not private.exists() and not list(WORK.glob('BUILD_RECEIPT_*.json')),'Fresh packaging stage required')
    (target/'segments').mkdir(parents=True);private.mkdir(parents=True)
    original=WORK/'staging'/release/'assets/public-name-data'/release
    for item in snapshot['segments']:
        p=original/'segments'/item['file'];require(sha(p)==item['sha256'] and p.stat().st_size==item['bytes'],'Preserved data segment changed')
        dest=target/'segments'/item['file'];shutil.copyfile(p,dest);require(sha(dest)==item['sha256'],'Copied segment differs')
    require(sha(WORK/f'KEY_ROLE_INDEX_{release}.jsonl.gz')==snapshot['key_role_index_sha256'],'Original full role/key index changed')
    nodes={};children=set();page_counts=Counter();max_head=(0,'');max_evidence=0;physical=Counter()
    def pages(evidence):
        nonlocal max_evidence,max_head
        for p in sorted((target/'segments').glob('*.bin')):
            if (int(p.stem)>=500000)!=evidence:continue
            data=p.read_bytes();require(len(data)==(512 if evidence else 128)*1024,'Segment cap differs');offset=0
            while offset<len(data) and data[offset]:
                obj=zlib.decompressobj(31);raw=obj.decompress(data[offset:],128*1024+1)
                require(obj.eof and not obj.unconsumed_tail and len(raw)<=128*1024,'Incomplete or oversized gzip member')
                size=len(data)-offset-len(obj.unused_data);value=json.loads(raw.decode('utf-8',errors='strict'))
                require(value['release']==release and value['schema']=='ndr-public-directory/3','Mixed source release')
                ref={'path':f'/assets/public-name-data/{release}/segments/{p.name}','segment_bytes':len(data),'offset':offset,'bytes':size,'sha256':hashlib.sha256(data[offset:offset+size]).hexdigest(),'inflated_bytes':len(raw)}
                physical[value['kind']]+=1
                if evidence:
                    require(value['kind']=='evidence' and len(raw)<=64*1024,'Evidence page bound differs');max_evidence=max(max_evidence,len(raw));yield value
                elif value['kind']=='heads':
                    require(len(raw)<=32*1024 and 0<len(value['records'])<=25,'Head page bound differs')
                    for row in value['records']:max_head=max(max_head,(len(encoded(row)),row['key']))
                else:
                    require(value['kind'] in ('pages','branches') and 0<len(value['entries'])<=128,'Index node shape differs')
                    entries=value['entries'];require(all(a['last']<b['first'] for a,b in zip(entries,entries[1:])),'Index codepoint ranges differ')
                    full={**ref,'first':entries[0]['first'],'last':entries[-1]['last'],'count':sum(e['count'] for e in entries)}
                    nodes[(ref['path'],offset)]={'ref':full,'value':value}
                    if value['kind']=='branches':children.update((e['path'],e['offset']) for e in entries)
                    else:page_counts[value['role']]+=len(entries)
                offset+=size
            require(not any(data[offset:]),'Nonzero unreferenced data padding')
    for _ in pages(False):pass
    def depth(node):
        if node['value']['kind']=='pages':return 1
        heights={depth(nodes[(e['path'],e['offset'])]) for e in node['value']['entries']};require(len(heights)==1,'Unequal index branch heights');return 1+heights.pop()
    roots=[n for key,n in nodes.items() if key not in children];require(len(roots)==3,'Expected three complete original source-view roots')
    views={n['value']['role']:{'count':n['ref']['count'],'depth':depth(n),'root':n['ref']} for n in roots}
    require({role:v['count'] for role,v in views.items()}=={'all':1211234,'given':331262,'family':995840} and max(v['depth'] for v in views.values())<=4,'Original completed views differ')
    totals=Counter();languages=Counter();largest_record=(0,'');current=None;parts=[]
    def recount(group):
        nonlocal largest_record
        row=recover(group);totals['spellings']+=1;totals['role_keys']+=len(row['roles'])
        for role in row['roles']:totals[role+'_role_keys']+=1
        totals['corroborated_spellings']+=bool(row['corroborated_roles']);totals['training_source_spellings']+=bool(row['primary_locators'])
        totals['source_attested_spellings']+=bool(row.get('attestations'));totals['training_source_only_spellings']+=bool(row['primary_locators']) and not row['corroborated_roles'] and not row.get('attestations')
        totals['source_role_differences']+=row['source_role_difference'];largest_record=max(largest_record,(len(encoded(row)),row['key']))
        for a in row.get('attestations',[]):
            if a['tier']=='lexical_sharealike':languages[(a['language'],a['language_name'])]+=1
    for page in pages(True):
        for part in page['fragments']:
            key=part['record']['key']
            if current is not None and key!=current:require(current<key,'Original evidence key order differs');recount(parts);parts=[]
            current=key;parts.append(part)
    if parts:recount(parts)
    totals['additional_public_spellings']=totals['spellings']-911217
    require(totals['spellings']==1211234 and totals['given_role_keys']==331262 and totals['family_role_keys']==995840 and totals['corroborated_spellings']==31818 and
        totals['source_attested_spellings']==1105795 and totals['training_source_only_spellings']==92714 and totals['training_source_spellings']==135249 and len({k[0] for k in languages})==224,'Recovered source/evidence totals differ')
    print(json.dumps({'stage':'original_data_stage_rehashed_recovered','release':release,'keys':totals['spellings'],'physical_members':sum(physical.values())}),flush=True)
    old=read(ROOT/'site/assets/public-name-data'/BASE_RELEASE/'manifest.json');sources=public_sources(old);by_id={s['id']:s for s in sources}
    def download(source,name,label,kind,data,original_name=None):
        p=target/'downloads'/name;p.parent.mkdir(exist_ok=True);require(len(str(p))<250 and len(data)<=25*1024*1024,'Bounded short download path required');p.write_bytes(data)
        source['downloads'].append({'kind':kind,'label':label,'original_filename':original_name or name,'url':f'/assets/public-name-data/{release}/downloads/{name}','bytes':len(data),'sha256':sha(p)})
    catalog=Catalog(WORK/'private-catalog/catalog.sqlite3')
    for source in sources:
        if source['id']=='namsor-japanese-public-2022':continue
        short={'datacite-public-2025':'dc','wiktionary-en-primary':'lex','orcid-public-2026':'oc','crossref-rest-bibliographic':'cr'}[source['id']]
        notice={'type':'source_notice','source':{k:source[k] for k in ('id','title','source_url','license_label','license_url','policy_url','description')},
            'modifications':'Additional closed V4 source observations only. Exact roles, literal forms and native DOI-field/minimized aggregate support scopes retained. Frozen spelling normalization, global deduplication, bounded lossless evidence and short download packaging. Earlier source downloads remain available. Each source retains its own reuse conditions; covered lexical adaptations remain CC-BY-SA4.'}
        chunk=[];chunk_bytes=0;current=None;group=[];number=0
        def emit(items,n):
            p=private/f'v4-{short}-{n:03d}.jsonl.gz'
            with p.open('xb') as raw,gzip.GzipFile(filename='',mode='wb',fileobj=raw,mtime=0,compresslevel=6) as out:
                out.write(encoded(notice))
                for item in items:out.write(encoded(item))
            download(source,p.name,f'Additional V4 source observations, part{n+1} (JSONL gzip with source notice)','transformed',p.read_bytes(),original_name=f"source-{source['id']}-{n:03d}.jsonl.gz")
        for key,text in catalog.db.execute('SELECT key,data FROM events WHERE source_id=? AND unit_rank>0 ORDER BY key,unit_rank,seq',(source['id'],)):
            if current is not None and key!=current:
                item={'key':current,'attestations':group};size=len(encoded(item));require(size<=192*1024,'Source observation group requires further lossless pagination')
                if chunk and chunk_bytes+size>16*1024*1024:emit(chunk,number);number+=1;chunk=[];chunk_bytes=0
                chunk.append(item);chunk_bytes+=size;group=[]
            current=key;group.append(json.loads(text))
        if current is not None:chunk.append({'key':current,'attestations':group})
        if chunk:emit(chunk,number)
    catalog.close()
    for spec in read(WORK/'INPUT_MANIFEST.json')['inputs']:
        packet=(ROOT/spec['receipt_path']).parent
        source=by_id[{'datacite_native':'datacite-public-2025','orcid_minimized':'orcid-public-2026','crossref_bibliographic_facts':'crossref-rest-bibliographic'}[spec['kind']]]
        names=['RECEIPT.json','SOURCE_REPLAY_RECEIPT.json','SOURCE_REPLAY_VERIFICATION.json','policy_snapshot.py','names.jsonl.gz'] if spec['kind']=='datacite_native' else ['RECEIPT.json','NAMES.jsonl.gz','NOTICE.txt','SOURCE_RIGHTS_REVIEW.json','SOURCE.json','COUNTS.json','COVERAGE.json','MODIFICATIONS.json']
        name='p-'+hashlib.sha256(spec['id'].encode()).hexdigest()[:12]+'.zip';p=private/name
        with zipfile.ZipFile(p,'x',compression=zipfile.ZIP_DEFLATED) as z:
            for filename in names:
                if not (packet/filename).exists():continue
                info=zipfile.ZipInfo(filename,date_time=(2026,11,2,0,0,0));info.compress_type=zipfile.ZIP_DEFLATED;z.writestr(info,(packet/filename).read_bytes())
            notice='Original admitted name packet and source/support proof metadata for '+spec['id']+'. Retain the source-specific original notices and conditions. DataCite deposited metadata is CC0; ORCID public fields are CC0 with privacy/publicity/marks caveats and no per-profile identifiers republished; Crossref author-name facts are reusable bibliographic facts, not a blanket CC0 license for all fields or linked works. Linked resources, abstracts, unrelated contributor metadata and private original ORCID/Crossref response bodies are excluded. Each support count remains within its original native batch/packet.\n'
            info=zipfile.ZipInfo('V4-NOTICE.txt',date_time=(2026,11,2,0,0,0));info.compress_type=zipfile.ZIP_DEFLATED;z.writestr(info,notice.encode())
        download(source,name,'Original admitted source packet/proofs: '+spec['id'],'source',p.read_bytes(),original_name=spec['id'].replace(':','--')+'-source-proofs.zip')
    helpers=[WORK/n for n in ['catalog_v4.py','source_common.py','freeze_inputs.py','census.py','export_native_witnesses.py','prepare_catalog_v4.py','build_v4.py','finish_data_v4_attempt03.py']]+[ROOT/'scripts_build'/n for n in ['public_name_v3_projection.py','public_name_segment_layout.py']]
    cutoff={'release':release,'prior_public_release':BASE_RELEASE,'source_scopes':[{k:s[k] for k in ['id','kind','receipt_sha256','sha256','expected_roles']} for s in read(WORK/'INPUT_MANIFEST.json')['inputs']],
        'excluded':['ORCID05','ORCID06','DataCite10','all future sources'],'input_manifest_sha256':sha(WORK/'INPUT_MANIFEST.json'),
        'transformation_source':'Lossless record/evidence generation plus bounded short-filename source packaging; source licenses remain separate.'}
    for source in sources:
        download(source,'v4-cutoff.json','Frozen V4 source scopes, conditions and transformation notes','notice',encoded(cutoff),original_name='V4-SOURCE-CUTOFF.json')
        for p in helpers:download(source,p.name,'V4 transformation source: '+p.name,'modifications',p.read_bytes())
    manifest={**old,'release':release,'totals':dict(totals),'sources':sources,'views':views,
        'storage':{'transport':'bounded_stream_segments/1','hot_segments':978,'evidence_segments':997,'hot_segment_bytes':128*1024,'evidence_segment_bytes':512*1024,
            'head_page_inflated_bytes':32*1024,'evidence_page_inflated_bytes':64*1024,'maximum_records':50,'default_head_records':25,'maximum_head_data_reads':1,'maximum_index_reads':8},
        'record_format':'compact_heads_with_lossless_paged_evidence',
        'lexical_language_attestations':[{'code':code,'name':name,'observations':n} for (code,name),n in sorted(languages.items(),key=lambda x:(x[0][1].casefold(),x[0][0]))],
        'limitations':old['limitations']+['Additional DataCite and ORCID counts remain within their native scopes. Crossref author-name fields are reusable bibliographic facts, not a blanket CC0 claim. All new source language/origin/meaning/gender and identity claims remain unknown.']}
    require(len(encoded(manifest))<=512*1024,'Manifest transport cap exceeded')
    (target/'manifest.json').write_bytes(encoded(manifest));pointer=stage/'assets/public-name-data/manifest.json';pointer.write_bytes(encoded(manifest))
    indexes=compile_indexes(views,target,release);generated=stage/'functions/_lib/active-public-name-release.js';generated.parent.mkdir(parents=True)
    packet=json.dumps({'manifest':manifest,'indexes':indexes},ensure_ascii=True,separators=(',',':'))
    generated.write_text("// Closed immutable V4 release metadata and upper indexes; no visitor state.\nconst freeze=(v)=>{if(v&&typeof v==='object'){Object.freeze(v);for(const c of Object.values(v))freeze(c);}return v;};\nconst data=freeze("+packet+");\nexport const activeManifest=data.manifest;\nexport const compiledIndexes=data.indexes;\n",'utf-8')
    current=[p for p in (ROOT/'site').rglob('*') if p.is_file() and not any(x in p.relative_to(ROOT/'site').parts for x in ('.wrangler','functions'))]
    new=[p for p in (stage/'assets').rglob('*') if p.is_file()];count=len(current)+len(new)-1
    require(count<=20000 and max(p.stat().st_size for p in new)<=25*1024*1024,'Actual retained static/asset budget exceeded')
    require(sha(WORK/'private-catalog/catalog.sqlite3')==closed['catalog_sha256'],'Catalog bytes changed during packaging')
    result={'ok':True,'built_at':datetime.now(timezone.utc).isoformat(),'release':release,'schema':'ndr-public-directory/3','stage':str(stage),
        'units':closed['units'],'totals':dict(totals),'public_lexical_codes':224,'hot_segments':978,'evidence_segments':997,'head_pages':dict(page_counts),
        'index_depth':{r:v['depth'] for r,v in views.items()},'compiled_index_nodes':len(indexes),'generated_module_bytes':generated.stat().st_size,
        'largest_full_record':largest_record,'largest_head':max_head,'largest_evidence_page_bytes':max_evidence,'evidence_pages':physical['evidence'],
        'new_static_files':len(new),'total_static_files_including_retained':count,'new_static_bytes':sum(p.stat().st_size for p in new),
        'manifest_sha256':sha(pointer),'active_module_sha256':sha(generated),'key_role_index':str(WORK/f'KEY_ROLE_INDEX_{release}.jsonl.gz'),'key_role_index_sha256':snapshot['key_role_index_sha256'],
        'data_generation_namespace_preserved':True,'all1975_complete_data_segments_reused_byte_exact':True,'data_stage_snapshot_sha256':sha(WORK/'DATA_STAGE_COMPLETION_SNAPSHOT.json'),
        'original_failed_data_writer_sha256':snapshot['original_writer_sha256'],'packaging_finisher_sha256':sha(__file__),'failed_attempt_preserved':True,'superseded_attempt02_preserved':True,'original_descriptive_filenames_retained_in_download_metadata':True,
        'fifty_million_achieved':False,'canonical_site_changed':False,'deployed':False,'capacity_scope':'Actual selected1,211,234-key catalog only; no50M storage or Workers-plan guarantee.'}
    write(WORK/f'BUILD_RECEIPT_{release}.json',result);print(json.dumps(result),flush=True)

if __name__=='__main__':main()
