import os, pathlib, subprocess, threading, http.server, json, datetime, re, signal
root=pathlib.Path(__file__).parent
run=root/datetime.datetime.now(datetime.UTC).strftime('%Y%m%dT%H%M%SZ'); run.mkdir()
requests=[]
class Endpoint(http.server.BaseHTTPRequestHandler):
 def do_POST(self):
  body=json.loads(self.rfile.read(int(self.headers['Content-Length'])))
  requests.append({'path':self.path,'model':body.get('model'),'syntheticBearerReceived':self.headers.get('Authorization')=='Bearer synthetic-evidence-token'})
  self.send_response(400); self.send_header('Content-Type','application/json'); self.end_headers(); self.wfile.write(b'{"error":{"message":"Evidence endpoint reached; no paid model was called","type":"invalid_request_error"}}')
 def log_message(self,*args): pass
server=http.server.ThreadingHTTPServer(('127.0.0.1',0),Endpoint)
threading.Thread(target=server.serve_forever,daemon=True).start()
config='model_provider = "cliproxy"\n[model_providers.cliproxy]\nbase_url = "http://127.0.0.1:%d/v1"\nwire_api = "responses"\nexperimental_bearer_token = "synthetic-evidence-token"\n'%server.server_port
for mode in ['configured-home','missing-config','codex-home']:
 home=run/mode; home.mkdir()
 if mode=='configured-home':
  (home/'.codex').mkdir(); (home/'.codex/config.toml').write_text(config)
 if mode=='codex-home':
  (home/'custom-codex').mkdir(); (home/'custom-codex/config.toml').write_text(config)
for side,work in [('before','factory-issue-38563-evidence-1-base'),('after','factory-issue-38563-evidence-1')]:
 lines=[f'{side.upper()} — '+('6a29d58d147522f85112c0c7149f2f9032cd955f' if side=='before' else 'b928e4216ece7b0f19ed34a511670a192f94ff5c'), 'Linux; Python 3.12.3; real pnpm eval commands; isolated synthetic HOME.', 'Local HTTP capture endpoint returns 400 intentionally; no paid model calls.','']
 for mode in ['configured-home','missing-config','codex-home']:
  for name,args in [('test-signal',['--model','gpt-6.1-sol','--case','copy-text-at-every-call-site','--samples','1']),('mcp-tool-use',['--model','gpt-6.1-sol','--case','NODEV-02','--samples','1'])]:
   env=os.environ.copy(); env.pop('CODEX_HOME',None); env['HOME']=str(run/mode)
   if mode=='codex-home': env['CODEX_HOME']=str(run/mode/'custom-codex')
   cmd=['pnpm',name,*args,'--out',str(run/f'{side}-{mode}-{name}-out')]
   start=len(requests)
   p=subprocess.Popen(cmd,cwd=pathlib.Path.home()/'wt'/work/'evals',env=env,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,text=True,start_new_session=True)
   try: output,_=p.communicate(timeout=35); status=str(p.returncode)
   except subprocess.TimeoutExpired:
    os.killpg(p.pid,signal.SIGTERM); output,_=p.communicate(); status='stopped after 35 seconds'
   output=output.replace(str(pathlib.Path.home()/'wt'/work),'<checkout>').replace(str(root),'<scratch>')
   output=re.sub(r'\x1b\[[0-9;]*m','',output)
   lines += [f'=== {mode} / {name} ===','$ '+ ' '.join(cmd[:-1])+ ' <output-dir>',f'Exit: {status}',output.strip(), 'Endpoint requests: '+json.dumps(requests[start:]),'']
 (run/f'{side}.txt').write_text('\n'.join(lines).replace(str(root),'<scratch>'))
 print(side, 'complete',flush=True)
server.shutdown()
print(run)
