"""Reviewed-release-only V7 private builder. No canonical/provider operations."""
import argparse,copy,gzip,hashlib,json,math,os,shutil,sqlite3,subprocess,sys,time,zipfile
from pathlib import Path
from collections import Counter
from common import *
from catalog import Catalog
from sources import closed_sources,add_sources,SCB_ID
import gnd_adapter as G
from authority_head import extend_head
from direct_packed_segments import PackedSegments,capacity
from public_name_segment_layout import SCHEMA,View
from public_name_v3_projection import EvidencePages,head,attach_evidence
import build_public_name_directory as helpers
OWNED={'assets/public-name-data/manifest.json','assets/public-name-static.js','assets/public-name-data-store.js',
 'assets/public-names.js','public-names.html','public-names/sources.html','functions/_lib/public-name-store.js',
 'functions/public-names/api.js','functions/_lib/active-public-name-release.js','functions/_lib/active-public-name-leaves.js'}

def baseline(cfg,gate):
 meta=read(checked(cfg['baseline']['inventory'],gate));need(meta['ok'] and meta['source_files']==19574 and meta['conservative_static_files']==19561,'Published baseline inventory differs')
 wanted={v['path']:v for v in meta['files']};actual=inventory(ROOT/'site',gate)
 need(set(wanted)==set(actual),'Another owner changed published inventory')
 for n,p in actual.items():need((p['bytes'],p['sha256'])==(wanted[n]['bytes'],wanted[n]['sha256']),'Published owner byte mismatch: '+n)
 need(not (ROOT/'site/wrangler.jsonc').exists() and not (ROOT/'site/wrangler.toml').exists(),'Unexpected production configuration')
 return actual

def census(c,base,work,gate):
 c.db.execute('ATTACH DATABASE ? AS old', (base.resolve().as_uri()+'?mode=ro&immutable=1',))
 expected_keys='SELECT key FROM old.names UNION SELECT key FROM v7_source_rows'
 expected_roles='SELECT key,role FROM old.roles UNION SELECT key,role FROM v7_source_rows'
 for wanted,actual in [(expected_keys,'SELECT key FROM names'),(expected_roles,'SELECT key,role FROM roles')]:
  need(c.db.execute('SELECT * FROM ('+wanted+') EXCEPT '+actual+' LIMIT 1').fetchone() is None and
       c.db.execute(actual+' EXCEPT SELECT * FROM ('+wanted+') LIMIT 1').fetchone() is None,'Exact source/baseline role union differs')
 result={'keys':c.db.execute('SELECT count(*) FROM names').fetchone()[0],
  'given_role_keys':c.db.execute("SELECT count(*) FROM roles WHERE role='given'").fetchone()[0],
  'family_role_keys':c.db.execute("SELECT count(*) FROM roles WHERE role='family'").fetchone()[0],
  'new_public_keys':c.db.execute('SELECT count(*) FROM (SELECT key FROM names EXCEPT SELECT key FROM old.names)').fetchone()[0]}
 result['overlap']=result['given_role_keys']+result['family_role_keys']-result['keys']
 out=work/'EXPECTED_KEY_ROLES.jsonl.gz'
 with out.open('xb') as raw,gzip.GzipFile(filename='',fileobj=raw,mode='wb',mtime=0) as f:
  for key, in c.db.execute('SELECT key FROM names ORDER BY key'):
   gate();f.write(encoded({'key':key,'roles':[v[0] for v in c.db.execute('SELECT role FROM roles WHERE key=? ORDER BY role',(key,))]}))
 c.db.execute('DETACH DATABASE old');save(work/'EXPECTED_SCOPE.json',{'ok':True,**result,'index':pin(out,gate)})
 return result

def compile_nodes(hot,views):
 indexes={};leaves={};text_bytes=refs=0
 def visit(ref,height,role):
  nonlocal text_bytes,refs
  value=hot.read_member(ref);need(value['schema']==SCHEMA and value['release']==hot.release and value['role']==role,'Compiler role/release differs')
  entries=value['entries'];need(0<len(entries)<=128 and entries[0]['first']==ref['first'] and entries[-1]['last']==ref['last'] and sum(e['count'] for e in entries)==ref['count'] and all(a['last']<b['first'] for a,b in zip(entries,entries[1:])),'Compiler exact ranges differ')
  key=ref['path']+':'+str(ref['offset'])
  if height>1:
   need(value['kind']=='branches','Compiler branch shape differs');indexes[key]={'sha256':ref['sha256'],'value':value}
   need(len(encoded(indexes))<=512*1024,'Compiled upper indexes exceed cap')
   for e in entries:visit(e,height-1,role)
  else:
   need(value['kind']=='pages','Compiler leaf shape differs')
   tuples=[[int(Path(e['path']).stem),e['offset'],e['bytes'],e['inflated_bytes'],e['sha256'],e['first'],e['last'],e['count']] for e in entries]
   text=json.dumps(tuples,ensure_ascii=True,separators=(',',':'))
   if len(text)<=64*1024 and text_bytes+2*len(text)<=12*1024*1024 and refs+len(entries)<=80000 and len(indexes)+len(leaves)<1024:
    leaves[key]={'sha256':ref['sha256'],'role':role,'text':text};text_bytes+=2*len(text);refs+=len(entries)
 for role,v in views.items():
  if v['count']:visit(v['root'],v['depth'],role)
 # New branches can be discovered after earlier leaves were selected. Trim only
 # optional compiled leaves; static source membership is never removed.
 while len(indexes)+len(leaves)>1024:leaves.popitem()
 need(len(indexes)<=1024,'Upper tree alone exceeds compiled-node budget')
 return indexes,{'format':'ndr-compiled-leaf-tuples/1','release':hot.release,'leaves':leaves}

def download(target,release,source,name,data,kind='source',label=None):
 need(len(data)<=15*1024*1024,'Public download exceeds15MiB transport cap')
 p=target/'downloads'/name;p.parent.mkdir(exist_ok=True)
 if p.exists():need(p.read_bytes()==data,'Conflicting public download path')
 else:p.write_bytes(data)
 source['downloads'].append({'url':f'/assets/public-name-data/{release}/downloads/{name}','original_filename':name,'label':label or name,'kind':kind,'bytes':len(data),'sha256':sha(p)})
def source_downloads(cfg,closed,target,release,sources):
 for kind,packet,id in [('gnd',closed['gnd'].parent,G.SOURCE_ID)]+([('scb',closed['scb'].parent,SCB_ID)] if 'scb' in closed else []):
  source=next(s for s in sources if s['id']==id);parts=[]
  with (packet/'NAMES.jsonl.gz').open('rb') as f:
   while True:
    b=f.read(15*1024*1024)
    if not b:break
    name=kind+'-names.gz.part'+str(len(parts)).zfill(3);download(target,release,source,name,b)
    parts.append({'part':len(parts),'filename':name,'bytes':len(b),'sha256':hashlib.sha256(b).hexdigest()})
  download(target,release,source,kind+'-parts.json',encoded({'original_filename':'NAMES.jsonl.gz','original_sha256':cfg[kind]['names']['sha256'],'parts':parts,'instruction':'Concatenate the consecutive binary parts in listed order, then gzip-decompress ONE original minimized packet.'}))
  for name in ['NOTICE.txt','SOURCE.json','MODIFICATIONS.json']:
   need((packet/name).is_file(),'Missing original source notice/conditions')
   download(target,release,source,kind+'-'+name,(packet/name).read_bytes(),'notice')
  for p in [HERE/'build.py',HERE/'sources.py',HERE/'catalog.py',HERE/'common.py',PREP/'gnd_adapter.py',PREP/'catalog_authority.py',PREP/'authority_head.py',PREP/'direct_packed_segments.py',PREP/'packed_logical_fetcher.mjs',PREP/'authority_public.mjs',HERE/'scb_public.mjs',HERE/'prepare_runtime.mjs']:
   download(target,release,source,'v7-'+p.name,p.read_bytes(),'modifications')

def main():
 a=argparse.ArgumentParser();a.add_argument('--config',required=True);a.add_argument('--deadline',required=True,type=float);v=a.parse_args()
 cfg,release_control,work=frozen_config(v.config);gate=Gate(work,v.deadline);gate(True)
 need((work/'SUPERVISOR_RUNNING.json').exists(),'Run only under original-handle resource supervisor')
 until=time.monotonic()+30
 while not (work/'ALLOW_BUILD.json').exists():need(time.monotonic()<until,'Parent did not assign original child lifetime job');time.sleep(.1)
 need(read(work/'ALLOW_BUILD.json')['original']['pid']==os.getpid(),'Original builder PID release differs')
 old_files=baseline(cfg,gate);closed=closed_sources(cfg,gate);old=read(checked(cfg['baseline']['manifest'],gate))
 base=checked(cfg['baseline']['catalog'],gate);need(base.stat().st_size==8890732544,'V6 catalog size differs')
 path=work/'catalog.sqlite3';need(not path.exists(),'One fresh catalog only; failed outputs require a fresh lineage')
 shutil.copy2(base,path);need(sha(path,gate)==cfg['baseline']['catalog']['sha256'],'Fresh catalog copy differs')
 c=Catalog(path);policy=helpers.policy();new_sources=add_sources(c,cfg,closed,policy,gate);expected=census(c,base,work,gate)
 release=hashlib.sha256(encoded({'baseline':cfg['baseline']['release'],'config':sha(v.config),'catalog':sha(path,gate),'code':sha(HERE/'CODE_FREEZE.json')})).hexdigest()[:24]
 stage=work/'payload';target=stage/'assets/public-name-data'/release;target.mkdir(parents=True)
 projection=sqlite3.connect(work/'projection.sqlite3');projection.executescript('PRAGMA cache_size=-16384;PRAGMA mmap_size=0;CREATE TABLE heads(key TEXT PRIMARY KEY,data TEXT,roles TEXT);')
 evidence=PackedSegments(target,release,524288,500000,gate=gate);pages=EvidencePages(evidence,projection);totals=Counter();languages=Counter()
 for row in c.records():
  gate();totals['spellings']+=1
  for role in row['roles']:totals[role+'_role_keys']+=1
  totals['corroborated_spellings']+=bool(row['corroborated_roles']);totals['training_source_spellings']+=bool(row['primary_locators'])
  totals['source_attested_spellings']+=bool(row.get('attestations'));totals['training_source_only_spellings']+=bool(row['primary_locators']) and not row['corroborated_roles'] and not row.get('attestations')
  totals['source_role_differences']+=row['source_role_difference']
  for item in row.get('attestations',[]):
   if item['tier']=='lexical_sharealike':languages[(item['language'],item['language_name'])]+=1
  compact=extend_head(head(row,pages.add(row)),row)
  projection.execute('INSERT INTO heads VALUES(?,?,?)',(row['key'],encoded(compact).decode(),json.dumps(row['roles'])))
  if totals['spellings']%10000==0:projection.commit()
 pages.flush();evidence.close();projection.commit();hot=PackedSegments(target,release,131072,gate=gate)
 writers={r:View(hot,r,target_page_bytes=32768,page_rows=25) for r in ('all','given','family')}
 for key,data,roles in projection.execute('SELECT key,data,roles FROM heads ORDER BY key'):
  gate();value=attach_evidence(json.loads(data),projection);writers['all'].add(value)
  for role in json.loads(roles):writers[role].add(value)
 views={r:w.finish() for r,w in writers.items()};hot.close();projection.close();c.close()
 need(totals['spellings']==expected['keys'] and totals['given_role_keys']==expected['given_role_keys'] and totals['family_role_keys']==expected['family_role_keys'],'Full directory census differs')
 need(totals['corroborated_spellings']==31818 and totals['training_source_spellings']==135249 and len(languages)==len(old['lexical_language_attestations']),'Historical flags/languages changed')
 sources=copy.deepcopy(old['sources']);sources.append(G.source_catalog_entry())
 if 'scb' in closed:
  meta=closed['scb_closure']['source'];sources.append({'id':SCB_ID,'source_group':'scb','title':'Statistics Sweden (SCB) registered surname, forename and call-name fields',
   'source_url':meta['original_download_url'],'policy_url':meta['terms_url'],
   'license_label':'Statistics Sweden (SCB), CC BY4.0','license_url':'https://creativecommons.org/licenses/by/4.0/',
   'description':'Statistics Sweden (SCB), reference2022-12-31, workbook updated2024-11-21. Literal typed surname/forename/call-name cells, exact decoded-value hash labels and separate table counts. NDR normalized keys and omitted mean-age values. Publisher sex-table categories are not name gender; Sweden is not name language or origin. Overlapping source categories/counts are never summed. SCB is one source group, without identity or independent-corroboration claims.','downloads':[]})
 source_downloads(cfg,closed,target,release,sources)
 totals['additional_public_spellings']=expected['new_public_keys'];totals['role_keys']=totals['given_role_keys']+totals['family_role_keys']
 manifest={**old,'release':release,'totals':dict(totals),'views':views,'sources':sources,
  'storage':{**old['storage'],'physical_packing':'direct04m','hot_segments':len(hot.files),'evidence_segments':len(evidence.files)},
  'limitations':old['limitations']+['DNB/GND preferred and variant typed components retain their exact source roles/forms and overlapping per-scope authority support. One DNB group never creates independent-source corroboration. Name attributes remain unknown.']+
   (['SCB source table categories overlap and are not summed; sex categories are not name gender. Mean-age values are absent from NDR public records and downloads.'] if 'scb' in closed else [])}
 need(len(encoded(manifest))<=524288,'Actual manifest exceeds512KiB');save(target/'manifest.json',manifest);save(stage/'assets/public-name-data/manifest.json',manifest)
 indexes,leaves=compile_nodes(hot,views);generated=stage/'functions/_lib';generated.mkdir(parents=True)
 (generated/'active-public-name-release.js').write_text('export const activeManifest='+encoded(manifest).decode().rstrip()+';\nexport const compiledIndexes='+encoded(indexes).decode().rstrip()+';\n','utf-8')
 (generated/'active-public-name-leaves.js').write_text('export const compiledLeafData='+encoded(leaves).decode().rstrip()+';\n','utf-8')
 # All data packs/downloads are already measured. Admit the exact composition
 # BEFORE cloning the retained site or constructing a runnable candidate.
 data_files=inventory(stage,gate);fresh_static=sum(is_static(n) and n not in old_files for n in data_files)+2
 actual_total=sum(is_static(n) for n in old_files)+fresh_static
 need(actual_total<=20000 and all(p['bytes']<25*1024*1024 for p in data_files.values()),'Exact retained+fresh file/asset admission failed; old files will not be removed')
 save(work/'PRE_CANDIDATE_ASSET_ADMISSION.json',{'ok':True,'retained_static':19561,'new_static':fresh_static,'static_total':actual_total,'all_data_files':data_files,'candidate_not_yet_created':True})
 site=work/'site';need(not site.exists(),'Candidate already exists');site.mkdir()
 for n,p in old_files.items():
  gate();dest=site/n;dest.parent.mkdir(parents=True,exist_ok=True);shutil.copy2(ROOT/'site'/n,dest)
  need(sha(dest,gate)==p['sha256'],'Retained candidate copy differs')
 for n,p in data_files.items():
  gate();dest=site/n;dest.parent.mkdir(parents=True,exist_ok=True)
  if dest.exists():need(n in OWNED,'Unexpected old-file overlay');dest.unlink()
  os.link(stage/n,dest)
 node=Path('C:/Users/bhara/.cache/codex-runtimes/codex-primary-runtime/dependencies/node/bin/node.exe')
 child=subprocess.Popen([str(node),'--max-old-space-size=256',str(HERE/'prepare_runtime.mjs'),str(site)],stdout=subprocess.PIPE,stderr=subprocess.PIPE)
 out,err=child.communicate(timeout=max(1,min(120,v.deadline-time.time())))
 (work/'runtime.stdout').write_bytes(out);(work/'runtime.stderr').write_bytes(err);need(child.returncode==0,'Private runtime composition failed; preserve outputs')
 gate(True);final=inventory(site,gate);need(sum(is_static(n) for n in final)==actual_total,'Post-runtime file inventory differs from prior admission')
 changed={n for n in old_files if final[n]['sha256']!=old_files[n]['sha256']};need(changed<=OWNED and set(old_files)<=set(final),'Unowned baseline bytes changed/removed')
 for n in set(final)-set(old_files):need(n in {'assets/public-name-authority.js','assets/public-name-packed.js','functions/_lib/public-name-authority.js','functions/_lib/public-name-packed.js'} or n.startswith('assets/public-name-data/'+release+'/'),'New file outside V7 allowlist')
 need(sha(base,gate)==cfg['baseline']['catalog']['sha256'],'Immutable baseline catalog changed during build')
 save(work/'BUILD_RECEIPT.json',{'ok':True,'completed_at':now(),'release':release,'config_sha256':sha(v.config),'baseline':cfg['baseline'],'new_source_rows':new_sources,'expected':expected,'manifest':pin(target/'manifest.json',gate),'catalog':pin(path,gate),'site':str(site),'files':final,'source_files':len(final),'static_files':actual_total,'owned_changes':sorted(changed),'new_paths':sorted(set(final)-set(old_files)),'physical_packs':len(hot.physical_files)+len(evidence.physical_files),'all_old_files_preserved':True,'minimum_observed_RAM':gate.minimum_ram,'maximum_private_bytes':gate.maximum_seen,'canonical_or_provider_mutated':False})
if __name__=='__main__':main()
