#!/usr/bin/env python3
"""Supplemental read-only saved history/ranking/rate-limit evidence extraction."""
from build_audit import *
import heapq
def pathread(p):return json.load(open(p))
def history_details():
 cases=pathread(HERE/'evidence/history-cases.json');out=[]
 for rid in sorted(set(c['run_id']for c in cases)):
  s=pathread(OLD/rid/'summary.json');src=g.Source(s['source']);history=pathread(REVIEW/'history'/f'{rid}.json');byid={h['input_id']:h for h in history}
  qs=[]
  for r in rows(OLD/rid/'requests.csv'):
   if r.get('request_ref')in [None,'NA','']:continue
   q=objref(src,r['request_ref'])
   if q and 'history_provenance'in q:qs.append((r['request_ref'],q))
  for case in [c for c in cases if c['run_id']==rid]:
   p=next(p for p in pathread(REVIEW/'probe-decision-audit.json')if p['run_id']==rid and p['step']==case['step'])
   q=pathread(REVIEW/p['request_ref']);audit=pathread(REVIEW/p['local_audit_not_model_input_ref']);currentversion=audit['abstraction_version'];traces=[]
   for prev in case['equivalent_history']:
    h=byid.get(prev.get('input_id'));tree=h.get('source_gui_id')if h else None;found=[]
    for ref,query in qs:
     for hid,pr in query.get('history_provenance',{}).items():
      if pr['source_tree']==tree:found.append({'request_id':query['request_id'],'review_step':query['step'],'history_id':hid,**pr,'request_ref':ref})
    found.sort(key=lambda x:x['review_step']);near=sorted(found,key=lambda x:abs(x['review_step']-case['request_step']))[:6]
    traces.append({'prior_step':prev['step'],'input_id':prev.get('input_id'),'source_gui_id':tree,'original_state':h.get('state_id')if h else None,'original_action':h.get('model_action_id')if h else None,
     'original_version':h.get('version')if h else None,'probe':h.get('probe')if h else None,'inventory_ref':h.get('inventory_ref')if h else None,'execution_ref':prev.get('execution_ref'),
     'same_query_scope_appearances':found,'nearest_query_scope_appearances':near,'appeared_in_exact_review':any(x['request_id']==case['request_id']for x in found)})
   wire=[]
   for hid,h in audit['history'].items():wire.append({'id':hid,'step':h['input']['step'],'text':h['concrete']['text'],'input_id':h['input']['input_id']})
   out.append({**case,'version_at_review':currentversion,'prior_traces':traces,'wire_witnesses':wire})
  src.close()
 dump(HERE/'evidence/history-case-traces.json',out)

def control_from_gui(src,h):
 gui=objref(src,h.get('source_gui_ref'))
 if gui is None:
  inv=objref(src,h.get('inventory_ref'));gui=src.native_obj((inv or{}).get('input',{}).get('source_gui',{}).get('path'))
 if gui is None:return dict(h['concrete']),False,None
 concrete=h['concrete'];nodes=gui['nodes'];node=next((n for n in nodes if n.get('resource_id')==concrete.get('resource_id')and n.get('native_text')==concrete.get('text')and n.get('bounds')==concrete.get('bounds')),None)
 if node is None:raise ValueError('Cannot uniquely recover node '+str(h['input_id']))
 byid={n['id']:n for n in nodes};collected=[]
 def collect(n,depth):
  if depth>3 or len(collected)>=32:return
  ro=n.get('raw_observation',{});norm=lambda v:unicodedata.normalize('NFC',' '.join((v or'').replace('"','').split()))
  raw=lambda f,native:norm(ro[f])if ro.get(f)is not None else n.get(native,'')
  collected.append({'text':raw('text','native_text'),'content_desc':raw('content_desc','native_content_desc'),'class':n.get('class',''),'resource_id':n.get('resource_id','')})
  for cid in n.get('children',[]):collect(byid[cid],depth+1)
 collect(node,0);control={**collected[0],'action_type':concrete['action_type'],'target_subtree':collected};return control,True,gui['gui_id']

def ranking_inputs():
 rid='run_0c300af1c9614168998ce1f37b6aadfe';src=g.Source(pathread(OLD/rid/'summary.json')['source']);history=pathread(REVIEW/'history'/f'{rid}.json');inputs=[]
 for qid,entry in [('semantic-35','semantic-35-c2'),('semantic-103','semantic-103-c2')]:
  p=REVIEW/'evidence'/rid/entry;q=pathread(p/'request-receipt.json');a=pathread(p/'local-audit-not-sent-to-model.json');candidate=a['candidates']['c2'];cg=src.native_obj(candidate['gui']['path'])
  h={'source_gui_ref':src.ref(src.native+'/'+candidate['gui']['path']),'concrete':candidate['concrete'],'input_id':entry};cc,ok,_=control_from_gui(src,h);assert ok
  members=[]
  for hid,pr in q['history_provenance'].items():
   assert pr['current_model_action']==candidate['concrete']['model_action_id']
   candidates=[h for h in history if h.get('source_gui_id')==pr['source_tree']and h.get('concrete',{}).get('action_type')==cc['action_type']];assert len(candidates)==1
   h=candidates[0];control,complete,_=control_from_gui(src,h)
   members.append({'id':hid,'step':int(h['step']),'input_id':h['input_id'],'probe':h['probe'],'control':control,'raw_descriptor_complete':complete,
    'raw_text_if_retained':control.get('text')if complete else None,'native_text':h['concrete'].get('text'),'input_confirmed':h.get('input_confirmed'),'execution_ref':h.get('execution_ref'),
    'execution_confirmed':h.get('execution_confirmed'),'business_success':h.get('business_success'),'target_activity':h.get('target_activity_from_native'),'target_gui_labels':h.get('target_gui',{}).get('labels'),'target_gui_ref':h.get('target_gui_ref'),
    'inventory_ref':h['inventory_ref'],'source_gui_ref':h.get('source_gui_ref'),'provenance':pr})
  inputs.append({'case':qid,'candidate':cc,'candidate_step':q['step'],'history':members,'expected_selected':q['history_coverage']['c2']['selected_history_ids'],
   'request_ref':str(p/'request-receipt.json'),'limitation':'Missing raw descriptors use explicitly flagged native-text projections; they are not asserted to be exact hidden ranking input.'})
 dump(HERE/'evidence/witness-inputs.json',inputs);src.close()
 cp=str(ROOT/'native/ape-java.jar')+':'+str(ROOT/'tests/lib/android-json-0.0.20131108.vaadin1.jar')
 outdir=HERE/'evidence/replay-classes';outdir.mkdir(exist_ok=True)
 subprocess.run(['javac','-encoding','UTF-8','-cp',cp,'-d',str(outdir),str(HERE/'WitnessReplay.java')],check=True)
 subprocess.run(['java','-cp',str(outdir)+':'+cp,'com.android.commands.monkey.ape.jev.WitnessReplay',str(HERE/'evidence/witness-inputs.json'),str(HERE/'evidence/witness-replay.json')],check=True)
 rs=[]
 for case in pathread(HERE/'evidence/witness-replay.json'):
  assert case['selected_matches']
  for r in case['rows']:
   h=r['source'];rs.append({'request_id':case['case'],'history_id':r['id'],'execution_step':h['step'],'native_label':h['native_text'],'raw_label':h['raw_text_if_retained'],'probe':h['probe'],
    'label_score':r['label_score']if h['raw_descriptor_complete']else None,'word_score':r['word_score']if h['raw_descriptor_complete']else None,
    'resource_score':r['resource_id_score'],'other_score':r['other_score'],'score_replay_projection':r['score'],'score_exact_archived_descriptor':r['score']if h['raw_descriptor_complete']else None,
    'rank_in_replay_projection':r['rank'],'selected':r['selected'],'raw_descriptor_complete':h['raw_descriptor_complete'],'feature_set':r['features'],'matched_features':r['matched_features'],
    'selection_reason':'first max-score; then unseen whole-feature signature'if r['selected']else'same signature as h1, diversity skip'if r['id']=='h2'else'behind earlier equal-score distinct signatures in replay',
    'input_confirmed':h['input_confirmed'],'execution_confirmed':h['execution_confirmed'],'business_success':h['business_success'],'target_activity':h['target_activity'],'target_gui_labels':h['target_gui_labels'],
    'execution_ref':h['execution_ref'],'target_gui_ref':h['target_gui_ref'],'source_ref':h['inventory_ref'],'gui_ref':h['source_gui_ref']})
 csvout('witness-ranking.csv',rs)

def rate_details():
 snap=pathread(HERE/'evidence/global-jev-calls.json');calls=snap['calls'];formal=rows(HERE/'request-waits.csv');formal=[r for r in formal if r['condition']=='JEV_PAIRWISE'];formalmap={r['gateway_call_id']:r for r in formal};assert len(formalmap)==686
 preflights=[]
 for c in calls:
  if c['id']in formalmap:continue
  src=g.Source(pathread(OLD/c['run_id']/'summary.json')['source']);name='preflight/preflight-1-request.json';raw=src.read(name);receipt=src.obj('preflight/preflight-1-receipt.json')
  match=hashlib.sha256(raw).hexdigest()==c['body_sha256'];assert match and receipt['status']=='PASS'
  preflights.append({'call_id':c['id'],'run_id':c['run_id'],'gateway_body_sha256':c['body_sha256'],'preflight_raw_sha256_matches':match,'preflight_status':receipt['status'],
   'preflight_request_id':receipt['request_id'],'request_ref':src.ref(name),'receipt_ref':src.ref('preflight/preflight-1-receipt.json')});src.close()
 dump(HERE/'evidence/preflight-call-matches.json',preflights)
 calls.sort(key=lambda c:(c['received_at'],c['id']));simrows=[];simcheck={}
 for rpm in [10,14]:
  last=-float('inf');active=[];result=[]
  for c in calls:
   arrival=c['received_at'];service=c['completed_at']-c['sent_at'];sent=max(arrival,last+60/rpm)
   while active and active[0]<=sent:heapq.heappop(active)
   if len(active)>=2:sent=max(sent,heapq.heappop(active))
   while active and active[0]<=sent:heapq.heappop(active)
   heapq.heappush(active,sent+service);last=sent
   r={'rpm':rpm,'call_id':c['id'],'run_id':c['run_id'],'formal':c['id']in formalmap,'arrival_host_s':arrival,'actual_sent_host_s':c['sent_at'],'actual_queue_s':c['sent_at']-arrival,
    'simulated_sent_host_s':sent,'simulated_queue_s':sent-arrival,'queue_reduction_vs_observed_s':c['sent_at']-sent,'upstream_fixed_service_s':service,'status':c['status']};result.append(r)
  simrows.extend(result)
  err=[r['actual_queue_s']-r['simulated_queue_s']for r in result];simcheck[str(rpm)]={'calls':len(result),'formal':sum(r['formal']for r in result),'signed_observed_minus_sim_queue_p50_s':pct(err,.5),'p95_s':pct(err,.95),'max_s':max(err),'min_s':min(err)}
 # Isolated dispatch sensitivity: hold the preceding observed send and residual
 # dispatch delay fixed. This does not accumulate ideal pacing improvements.
 last_observed=None;sensitivity={}
 for c in sorted(calls,key=lambda c:c['sent_at']):
  ready10=max(c['received_at'],last_observed+6)if last_observed is not None else c['received_at']
  ready14=max(c['received_at'],last_observed+60/14)if last_observed is not None else c['received_at']
  sensitivity[c['id']]=max(0.,ready10-ready14);last_observed=c['sent_at']
 for r in simrows:r['isolated_previous_actual_send_fixed_reduction_s']=sensitivity[r['call_id']]
 csvout('rate-limit-simulation.csv',simrows)
 summary=[]
 for rid in sorted(set(c['run_id']for c in calls)):
  cs=sorted([c for c in calls if c['run_id']==rid],key=lambda c:c['sent_at']);fs=[c for c in cs if c['id']in formalmap];gaps=[b['sent_at']-a['sent_at']for a,b in zip(fs,fs[1:])];base=next(r for r in formal if r['run_id']==rid)
  ten={r['call_id']:r for r in simrows if r['rpm']==10 and r['run_id']==rid and r['formal']};fourteen={r['call_id']:r for r in simrows if r['rpm']==14 and r['run_id']==rid and r['formal']}
  summary.append({'run_id':rid,'app':base['app'],'formal_requests':len(fs),'other_gateway_calls_same_run':len(cs)-len(fs),'actual_queue_s':sum(c['sent_at']-c['received_at']for c in fs),
   'actual_send_gap_min_s':min(gaps),'actual_send_gap_p50_s':pct(gaps,.5),'actual_send_gap_p95_s':pct(gaps,.95),'actual_send_gap_max_s':max(gaps),
   'sim10_queue_s':sum(r['simulated_queue_s']for r in ten.values()),'sim14_queue_s':sum(r['simulated_queue_s']for r in fourteen.values()),
   'sim10_minus_sim14_queue_s':sum(ten[k]['simulated_queue_s']-fourteen[k]['simulated_queue_s']for k in ten),
   'isolated_previous_actual_send_fixed_reduction_s':sum(sensitivity[k]for k in ten),
   'observed_minus_sim14_queue_s':sum(r['queue_reduction_vs_observed_s']for r in fourteen.values()),'assumption':'fixed archived arrivals, fixed upstream duration, FCFS approximation, shared 2 inflight leases, all 691 observed calls included; no changed APE trajectory'})
 csvout('rate-limit-summary.csv',summary)
 sends=sorted(c['sent_at']for c in calls);gaps=[b-a for a,b in zip(sends,sends[1:])];maxactive=max(sum(c['sent_at']<=t<c['completed_at']for c in calls)for t in sends)
 rolling=max(bisect.bisect_left(sends,t+60)-bisect.bisect_left(sends,t)for t in sends)
 dump(HERE/'evidence/rate-checks.json',{'global_calls':len(calls),'formal_calls':len(formalmap),'unmatched_call_ids':[c['id']for c in calls if c['id']not in formalmap],'actual_global_gap_min_s':min(gaps),'actual_global_gap_p50_s':pct(gaps,.5),'max_observed_upstream_concurrent':maxactive,'max_observed_sends_in_half_open_60s':rolling,'simulations':simcheck,'outside_this_database_traffic':'UNKNOWN'})

def code_identity():
 files=[p for p in (ROOT/'native/src').rglob('*.java')];comparison=[]
 for p in files:
  relative=str(p.relative_to(ROOT));frozen=subprocess.check_output(['git','show',FROZEN+':'+relative]);comparison.append({'path':relative,'sha256':sha(p),'frozen_sha256':hashlib.sha256(frozen).hexdigest(),'matches':p.read_bytes()==frozen})
 dex=HERE/'evidence/dex/com/android/commands/monkey/ape/jev';tokens={'JevRuntime.smali':['ape.jev.probeTimeoutMs','0x1388','Lcom/android/commands/monkey/ape/jev/JevSemanticReview;-><init>'],
  'JevSemanticReview.smali':['awaitUntil','submitRetained','STALE_HISTORY_OR_VERSION','archiveOnArrival','RETAINED_CAPACITY'],
  'JevTransport$Call.smali':['awaitUntil','CountDownLatch;->await'],'JevHistoryPairwiseProtocol.smali':['selectHistoryFeatures','label:','resource_id:']}
 dexchecks=[]
 for name,terms in tokens.items():
  data=(dex/name).read_text();dexchecks.extend({'file':str((dex/name).relative_to(HERE)),'token':t,'present':t in data}for t in terms)
 assert all(r['present']for r in dexchecks)
 dump(HERE/'evidence/code-identity.json',{'head':subprocess.check_output(['git','rev-parse','HEAD'],text=True).strip(),'frozen_commit':FROZEN,'native_java_files':len(comparison),'all_native_sources_match_frozen':all(r['matches']for r in comparison),
  'native_ape_jar_sha256':sha(ROOT/'native/ape.jar'),'native_java_jar_sha256':sha(ROOT/'native/ape-java.jar'),'source_comparison':comparison,'dex_key_call_checks':dexchecks,
  'boundary':'Source identity plus saved build identity plus key DEX call checks; no claim of a new full reproducible Android build.'})
 # Snapshot current platform source only. This is NOT a historical deployed source attestation.
 p=Path('/home/hy/darwin-deploy/experiment_platform/lab_platform/provider.py');(HERE/'evidence/provider-current.py').write_bytes(p.read_bytes());dump(HERE/'evidence/provider-current-identity.json',{'path':str(p),'sha256':sha(p),'historical_deployment_source_snapshot':'NOT AVAILABLE','use':'Current implementation corroboration; frozen config and actual timings are runtime authority.'})
def saved_evidence_views():
 # Read existing historical fixture logs only; never execute their local server.
 p=ROOT/'artifacts/probe-persistent-20261102/final-failopen-v3/retained/same_tree-5000/jev-events.jsonl';ev=[json.loads(l)for l in p.read_text().splitlines()]
 keep=[{'line':i,'event':e}for i,e in enumerate(ev,1)if e.get('event')in ['SEMANTIC_FAIL_OPEN','SEMANTIC_RESPONSE_CONSUMED','DECISION_PROBE','RUN_FINISHED']]
 dump(HERE/'evidence/historical-soft-timeout-health.json',{'source':str(p.relative_to(ROOT)),'sha256':sha(p),'scope':'Historical synthetic test, not a formal Run and not executed in this task.','events':keep})
 p=Path('/home/hy/darwin-deploy/experiment_platform/lab_platform/results.py');(HERE/'evidence/platform-results-current.py').write_bytes(p.read_bytes())
 for commit in ['2763d2f','855653c','9a5b577']:
  data=subprocess.check_output(['git','show','--format=fuller',commit,'--','native/src/com/android/commands/monkey/ape/jev/JevSemanticReview.java','native/src/com/android/commands/monkey/ape/jev/JevTransport.java'],cwd=ROOT)
  (HERE/f'evidence/design-{commit}.diff').write_bytes(data)
 wanted={('scarletnotes','JEV_PAIRWISE'),('activitydiary','JEV_PAIRWISE'),('omninotes','STARTLUX_PAIRWISE'),('omninotes','JEV_PAIRWISE')};req=rows(HERE/'request-waits.csv')
 csvout('longest-measured-waits.csv',[max((r for r in req if r['app']==app and r['condition']==cond),key=lambda r:float(r['ape_measured_await_s']))for app,cond in sorted(wanted)])
 from PIL import Image,ImageDraw
 items=[('ActivityDiary Statistics 475','run_2b38f68437b6480c8f26d01cec61b440','semantic-9-c2'),('ActivityDiary Statistics 564','run_2b38f68437b6480c8f26d01cec61b440','semantic-15-c3'),('Omni Lock 246','run_0c300af1c9614168998ce1f37b6aadfe','semantic-4-c2'),('Omni Lock 664','run_0c300af1c9614168998ce1f37b6aadfe','semantic-35-c2'),('Omni Lock 1531','run_0c300af1c9614168998ce1f37b6aadfe','semantic-103-c2')]
 out=Image.new('RGB',(1500,len(items)*350),'#dddddd');draw=ImageDraw.Draw(out)
 for i,(label,rid,entry)in enumerate(items):
  for j,phase in enumerate(['before','after']):
   im=Image.open(OLD/rid/'screenshots'/f'{entry}-{phase}.png').convert('RGB');im.thumbnail((730,320));out.paste(im,(j*750+(750-im.width)//2,i*350+27));draw.text((j*750+10,i*350+7),label+' '+phase,fill='black')
 out.save(HERE/'evidence/gui-cases.jpg')
if __name__=='__main__':
 history_details();ranking_inputs();rate_details();code_identity();saved_evidence_views()