#!/usr/bin/env python3
"""Validate captures, preserve raw diffs, make transparent normalised views and sheet."""
import datetime, difflib, hashlib, json, pathlib, re, subprocess, tarfile
from capture_missing import ROOT, PINS, ENVIRONMENT

SESSION = pathlib.Path((ROOT / 'latest-runs.txt').read_text())
BUILD = pathlib.Path((ROOT / 'latest-build.txt').read_text())

class Normalizer:
    def __init__(self, run):
        self.run = run
        self.events = json.loads((run / 'events.json').read_text())
        ps = next(e['data']['tracked_pids'] for e in self.events if e['kind'] == 'PROCESS_STATE')
        self.pids = {str(p): f'<PID{i+1}>' for i, p in enumerate(ps)}
    def text(self, text):
        text = text.replace(str(self.run), '<RUN>')
        text = re.sub(re.escape(str(BUILD)) + r'/(before|after)', '<BUILD>', text)
        text = text.replace(str(ROOT), '<SCRATCH>')
        text = re.sub(r'http://127\.0\.0\.1:\d+', 'http://127.0.0.1:<PORT>', text)
        text = re.sub(r'\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d\.\d+(?:\+00:00|Z)', '<UTC>', text)
        text = re.sub(r'\+\d+\.\d+(?= [A-Z_]+ )', '+<T>', text)
        text = re.sub(r'("(?:request_t|mono|duration_seconds)":\s*)\d+(?:\.\d+)?', r'\1"<TIME>"', text)
        text = re.sub(r'\.staging-[a-zA-Z0-9]+', '.staging-<STAGING>', text)
        text = re.sub(r'tmp\.[a-zA-Z0-9]{6,}', 'tmp.<TEMP>', text)
        for pid, label in self.pids.items(): text = re.sub(r'(?<![\dA-Za-z])' + pid + r'(?![\dA-Za-z])', label, text)
        return text
    def behavioural(self):
        channels = []
        manifest = next(e['data'] for e in self.events if e['kind'] == 'MANIFEST')
        channels.append('COMMAND ' + self.text(manifest['command']))
        channels.append('ENV_CHECK ' + self.text(next(e['data']['output'] for e in self.events if e['kind'] == 'ENV_CHECK')))
        requests = [e['data'] for e in self.events if e['kind'] == 'SERVER' and e['data'].get('event') == 'request']
        previous = {}
        for request in requests:
            item = {k:v for k,v in request.items() if k != 'request_t'}
            endpoint = request['endpoint']
            if endpoint in previous: item['same_endpoint_gap_seconds_0.1s'] = round(request['request_t'] - previous[endpoint], 1)
            previous[endpoint] = request['request_t']
            channels.append('REQUEST ' + json.dumps(item, sort_keys=True))
        for e in self.events:
            if e['kind'] == 'TRANSPORT':
                channels.append('TRANSPORT ' + self.text(json.dumps({k:v for k,v in e['data'].items() if k not in ['utc','mono']}, sort_keys=True)))
            elif e['kind'] in ['ACTION', 'DIRECTORY_BEFORE_SIGNAL', 'POST_SIGNAL_OBSERVATION', 'PROCESS_STATE', 'DIRECTORY_AFTER_EXIT', 'TMP_AFTER_EXIT', 'SERVER_BEFORE_CLEANUP', 'EXIT', 'EXECUTABLE_CHECK']:
                channels.append(e['kind'] + ' ' + self.text(json.dumps(e['data'], sort_keys=True)))
        # Concatenating preserves terminal byte order while removing incidental read chunk boundaries.
        terminal = ''.join(e['data'] for e in self.events if e['kind'] == 'OUTPUT')
        channels.append('TERMINAL_BEGIN\n' + self.text(terminal) + '\nTERMINAL_END')
        return '\n'.join(channels) + '\n'

def diff(a, b, name):
    text = ''.join(difflib.unified_diff(a.splitlines(True), b.splitlines(True), fromfile='before ' + PINS['before'], tofile='after ' + PINS['after']))
    name.write_text(text)
    return {'bytes': len(text.encode()), 'lines': len(text.splitlines())}

def reqs(r): return ' → '.join(f"{q['endpoint']}:{q['status']}" for q in r['requests'])

def main():
    results = json.loads((SESSION / 'results.json').read_text())
    assert len(results) == 28
    summary = []
    for pair in sorted(p for p in SESSION.iterdir() if (p/'before/result.json').is_file()):
        rs = {s: json.loads((pair / s / 'result.json').read_text()) for s in PINS}
        for side, r in rs.items():
            assert r['sha'] == PINS[side] and not r['timeout']
            assert not r['state']['remaining_processes'], r
            ev = json.loads((pair / side / 'events.json').read_text())
            manifest = next(e['data'] for e in ev if e['kind'] == 'MANIFEST')
            assert '/home/user/.t3' not in json.dumps(manifest['env'])
            assert pathlib.Path(manifest['env']['HOME']).is_relative_to(ROOT)
            assert pathlib.Path(manifest['env']['T3CODE_HOME']).is_relative_to(ROOT)
            assert manifest['fixture_payloads'] == json.loads((SESSION / 'artifact-fingerprints.json').read_text())
            assert next(e['data']['thread_alive'] for e in ev if e['kind']=='FIXTURE_CLOSED') is False
            if r['row'] == 'S31':
                assert r['exit_code'] == (1 if side=='before' else 130)
                assert r['interrupt_sent'] == (side=='after')
                assert len(r['requests']) == 1 and r['requests'][0]['status'] == 504
                assert r['requests_after_signal'] == (None if side=='before' else 0)
                assert not r['state']['staging'] and not r['state']['markers']
                if side=='after':
                    action = next(e['data'] for e in ev if e['kind']=='ACTION')
                    assert action['sleep_process']['state'].startswith('S')
                    assert '/usr/bin/sleep 1' in action['sleep_process']['cmdline']
                    assert action['sleep_process']['pid'] != action['foreground_pgid']
                    before_dirs = next(e['data'] for e in ev if e['kind']=='DIRECTORY_BEFORE_SIGNAL')
                    if r['case']['target']=='sums': assert any('.staging-' in line for line in before_dirs)
            elif r['row'] in ['E02','E03']:
                assert (r['exit_code']==0) == (side=='after')
                target = r['case']['target']
                codes = [q['status'] for q in r['requests'] if q['endpoint']==target]
                failure = 502 if r['row']=='E02' else 504
                assert codes == ([failure] if side=='before' else [failure,200]), codes
                if side=='after' and r['case']['surface'] in ['S','L']: assert len(r['state']['markers']) == 1
            elif r['row']=='E04':
                assert r['exit_code']==130 and r['interrupt_sent']
                assert len(r['state']['staging'])==1 and not r['state']['markers']
                leftover = pair / side / r['state']['staging'][0]
                assert len(list(leftover.iterdir()))==1
                assert (leftover / 't3-1.2.3-linux-x64.tar.gz').stat().st_size > 0
            if r['case']['surface']=='L': assert any(e['kind']=='NODE_VERSION' and e['data']['output']=='v20.11.1\n' for e in ev)
            if r['case']['surface']=='P':
                text = (pair / side / 'output.txt').read_text()
                expected = 'terminal=True interactive=True' if r['case']['mech']=='H' else 'terminal=False interactive=False'
                assert expected in text, text
        raw = diff((pair/'before/transcript.txt').read_text(), (pair/'after/transcript.txt').read_text(), pair/'Diff0.diff')
        ns = {side: Normalizer(pair/side) for side in PINS}
        for side, n in ns.items():
            (pair/side/'timestamp-masked.txt').write_text(n.text((pair/side/'transcript.txt').read_text()))
            (pair/side/'behavioural.txt').write_text(n.behavioural())
        dt = diff((pair/'before/timestamp-masked.txt').read_text(), (pair/'after/timestamp-masked.txt').read_text(), pair/'DiffT.diff')
        db = diff((pair/'before/behavioural.txt').read_bytes().decode(), (pair/'after/behavioural.txt').read_bytes().decode(), pair/'DiffB.diff')
        summary.append({'pair': pair.name, 'row': rs['before']['row'], 'before': rs['before'], 'after': rs['after'], 'Diff0': raw, 'DiffT': dt, 'DiffB': db})
    for row in ['S31','E02','E03','E04']:
        folder=SESSION/'row-diffs'/row; folder.mkdir(parents=True,exist_ok=True)
        for name in ['Diff0','DiffT','DiffB']:
            (folder/(name+'.diff')).write_text(''.join('PAIR '+p['pair']+'\n'+(SESSION/p['pair']/(name+'.diff')).read_text() for p in summary if p['row']==row))
    (SESSION/'validation.json').write_text(json.dumps({'validated':True,'rows':4,'pairs':14,'executions':28,'timeouts':0,'remaining_tracked_processes':0,'checks':['exact pins','fresh scratch HOME and T3CODE_HOME env check before every execution','shared payload fingerprints','foreground PTY and sleeping process at S31 SIGINT','zero post-signal requests in 1.5s open-server window','S31 staging absent after exit','502 and 504 sequences on exact endpoint','Node v20.11.1 line','real PS7 W/H dispatch','all fixture threads closed']},indent=2))
    (SESSION/'diff-summary.json').write_text(json.dumps(summary,indent=2))
    text = '# Missing CLI evidence for [#32913](https://git.cafe/pingdotgg/t3code/pull/32913)\n\n'
    text += '**4/4 requested rows captured; 14 before/after pairs, 28 final executions. No requested row is unverified.**\n\n'
    text += f"Before: [fa2f4ff2a2](https://git.cafe/pingdotgg/t3code/commit/{PINS['before']}). After: [1af59792b3](https://git.cafe/pingdotgg/t3code/commit/{PINS['after']}). No worktree edits or commits. Synthetic HTTP data only; no T3 database.\n\n"
    text += '**Environment:** '+ENVIRONMENT+'. Kernel 6.8.0-117-generic. Exact version output is in `environment.txt`. Node and PowerShell are the official linux-x64 tarballs downloaded into scratch; download URLs and local hashes are in `tools.json`.\n\n'
    text += '**Artifacts:** ARTIFACT_LINKS\n\n'
    text += 'Archive paths below are relative to the final bundle root. Each pair contains `before/transcript.txt` and `after/transcript.txt` (each records its SHA), independent `server-request.log` files with UTC/relative timestamps, `directory-listing.txt`, `events.json`, `result.json`, executed source/wrappers, and `Diff0.diff`, `DiffT.diff`, `DiffB.diff`. `row-diffs/<row>/` aggregates the pair diffs.\n\n'
    text += '| Row | Before | After |\n|---|---|---|\n'
    text += '| S31 | Both shells exit 1 at first 504; no backoff exists. | Latest and SHA256SUMS backoff Ctrl-C exit 130 in dash and bash --posix. Zero subsequent requests, zero remaining tracked processes. SHA256SUMS staging is present before the signal and removed afterward. Discovery has not created staging. |\n'
    text += '| E02 | Shell piped archive fetch exits 1; interactive archive download exits 22. PS7 R/W/H exit 1 at first 502. | All retry the same endpoint once, receive 200 and exit 0. Shell installs validate the executable and completion marker; PS7 uses real HTTP in verbatim helpers/discovery caller. |\n'
    text += '| E03 | Real built npm launcher on Node 20.11.1 exits 1 at first 504, separately on checksums and archive. | Same endpoint retries after about 0.5s; both installs exit 0, print 1.2.3 and create the completion marker. |\n'
    text += '| E04 | Ctrl-C during the active archive stream exits 130 and leaves one staging directory and partial tarball. | Same outcome and same partial byte count; the PR does not fix active-download SIGINT cleanup. |\n\n'
    text += '## Pair results and one-line diffs\n\n'
    text += 'D0 = raw transcript diff; DT = full transcript diff with timestamps and incidental paths/PIDs masked; DB = grouped behavioural diff. Counts below are bytes. SHA metadata remains in D0/DT.\n\n'
    text += '| Pair / archive directory | Before [fa2f4ff2a2](https://git.cafe/pingdotgg/t3code/commit/'+PINS['before']+') | After [1af59792b3](https://git.cafe/pingdotgg/t3code/commit/'+PINS['after']+') | Diff summary |\n|---|---|---|---|\n'
    for p in summary:
        a,b=p['before'],p['after']
        gaps=[]
        target=b['case']['target']; rr=[q for q in b['requests'] if q['endpoint']==target]
        if len(rr)>1:gaps=[round(rr[i]['request_t']-rr[i-1]['request_t'],3) for i in range(1,len(rr))]
        state='; staging survives' if b['row']=='E04' else ''
        text+=f"| `{p['pair']}` | exit {a['exit_code']}; {reqs(a)} | exit {b['exit_code']}; {reqs(b)}{state} | D0 {p['Diff0']['bytes']} / DT {p['DiffT']['bytes']} / DB {p['DiffB']['bytes']}; "+('same-endpoint request gap '+str(gaps)+' s' if gaps else 'no retry after exit/signal')+' |\n'
    text+='\n## Exact E04 leftovers\n\n'
    for side in PINS:
        p=next(p for p in summary if p['row']=='E04')
        lines=(SESSION/p['pair']/side/'directory-listing.txt').read_text()
        text+=f"**{side.capitalize()} — [{PINS[side][:10]}](https://git.cafe/pingdotgg/t3code/commit/{PINS[side]})**\n\n```text\n{lines}```\n\n"
    text+='## What else changed\n\n'
    text+='- Intended: head makes one additional request to each failed 502/504 endpoint, introduces one backoff, reaches subsequent download/validation steps, and replaces the prior error output with success. Request headers and endpoint order up to the failure are unchanged. Shell noninteractive failure exits 1; its interactive curl failure exits 22; both become 0 on recovery. PS7 exceptions become successful resolved-version or downloaded-byte/hash output.\n'
    text+='- Intended S31 comparison: main exits 1 before a backoff exists, so a backoff Ctrl-C cannot be delivered there. Head reaches an actual `sleep 1`; terminal Ctrl-C exits 130. Latest discovery ends by native SIGINT (wait status 2); checksum download reaches the existing INT trap (wait status 33280). Both report exit code 130.\n'
    text+='- Unchanged limit: latest lookup leaves a 19-byte temporary index in TMPDIR on both pins. Staging is not yet created there. The SHA256SUMS staging directory is cleaned on both the before failure and after interruption.\n'
    text+='- Unchanged limit: launcher mid-body SIGINT leaves an incomplete `1.2.3.staging-*` directory containing only the partial archive on both pins. No completion marker, extraction, published version, or further request is observed. Exact names, modes and bytes are listed above; partial archive bytes are retained in the bundle.\n'
    text+='- Downloader implementation detail visible in the shell traces: head adds `-w %{http_code}` to interactive curl, writes a `.status` sidecar while active, and removes the sidecars after successful completion. Foreground curl command arguments are unchanged.\n'
    text+='- Fixture/metadata differences: UTC and elapsed clocks, loopback ports, PIDs/PGIDs, random temporary/staging names, before/after paths, source hashes and error source locations differ. Terminal read chunks and progress-update flushes can differ; raw transcripts retain them. Final paired payload bytes are identical, as recorded in `artifact-fingerprints.json` and every manifest.\n\n'
    text+='## Normalisation and scope\n\n'
    text+='`Diff0` uses the original terminal transcript bytes. `DiffT` masks UTC/relative clocks (including request_t and transport mono), observation duration, independent run/build paths, loopback ports, tracked PIDs and random mktemp/staging names. It preserves source hashes, SHA/side, all statuses, exceptions, command arguments, signals, exit/wait statuses, byte counts, directory/file presence and event order. Its full timestamp-masked inputs are retained.\n\n'
    text+='`DiffB` additionally groups requests, transport calls, actions, state and concatenated terminal output into stable channels; only terminal read chunk boundaries and cross-channel flush order are removed. Same-endpoint request gaps are retained to 0.1s. All emitted terminal bytes, including ANSI and PowerShell error source frames, remain. SHA/provenance metadata is outside this behavioural view and remains raw/DT. No failed attempt, outcome or leftover is suppressed.\n\n'
    text+='S31 uses a controlling PTY, pipes the full script into shell `-s`, redirects stderr to a pipe to force noninteractive fetch, confirms the foreground process group and sleeping `/usr/bin/sleep 1`, then writes the terminal Ctrl-C byte. The server stays open for 1.5s after exit; the request log confirms zero additional requests. E02 shell pipe cases have no TTY; interactive cases have PTY stderr. PowerShell R/W have redirected stderr; H reports terminal=True and interactive=True. PowerShell executes real PS7/.NET helpers on Linux, without claiming an end-to-end Windows install. Launchers are unpacked outputs of the real `buildNpmLauncher` CLI used by `scripts/build-npm-launcher.test.ts`, with exact source-byte equality checked, and run without fetch hooks on Node 20.11.1.\n\n'
    text+='Earlier same-row captures are preserved in the preliminary bundle. They are superseded by the 28 final executions: S31 gained explicit pre-signal directory snapshots and confirmed sleeping state, and the final complete run shares one fixture payload across all pairs to prevent archive timestamp variation. No preliminary evidence was deleted.\n\n'
    text+='Unverified requested rows: **none**. No UI surface applies. All started servers/processes are stopped, the second checkout is removed, and available disk remains above 5GB; see `cleanup-verification.txt`.\n\n'
    text+='FRICTION: tool | The before checkout could not resolve build dependencies from the root node_modules link alone; scripts/node_modules is a separate workspace dependency directory. Linking both existing dependency directories into the temporary before checkout allowed the real build. | A fixture setup note naming both links would avoid the first failed build attempt.\n'
    text+='FRICTION: model | A report regeneration initially treated its generated row-diffs directory as another capture pair. Filtering for before/result.json fixed reruns; no raw evidence changed. | Make report discovery idempotent from the first run.\n'
    (ROOT/'evidence-sheet.template.md').write_text(text)
    print('VALIDATED 4 rows / 14 pairs / 28 executions')
    for p in summary:print(p['pair'],p['Diff0']['bytes'],p['DiffT']['bytes'],p['DiffB']['bytes'])

if __name__=='__main__':main()