#!/usr/bin/env python3 """Job #2972 (route 114) DELIVERABLE GATE: does an accepted audit's revised verdict reach the served corpus? For every ACCEPTED return of type=audit that states a revision (patch or revision_sha), this walks the document's own version chain and decides, from bytes: REVISION-SERVED the audit's added lines are ALL in the text served today AND were absent from the document's base version -> the reader sees the audit's revised verdict; the gate names the carrier version and the return id that wrote it REVISION-RECORDED-REVERTED its added lines were in some version but are not in the served text today (and the served text is the base again) RECORDED-NOT-SERVED the revision exists as a store object (/files/) but no version of the document carries it NO-OBSERVABLE-PRODUCT neither the text nor any version carries it Controls, all pre-registered: C1 negative every audit's added lines are also tested against a document it does not name (0 present required) C2 direction where the current text contains all added lines, the BASE version must not, and must still contain the patch's removed lines; this is the control that decides WHICH SIDE of the diff the corpus is serving (the earlier work's control assumed the pre-audit side is always the served one, which is only true when the revision is NOT applied) C3 provenance every version's bytes are re-hashed against the content_sha in its own content_url, and the served copy is re-hashed against /history's timestamps.sha256 Inputs: cache/ids.jsonl, cache/browse/.json, cache/docs/* (all fetched anonymously by sweep.py). This script re-fetches only /history/ and /files/, anonymously. """ import hashlib import json import os import re import sys import time import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor sys.stdout.reconfigure(encoding="utf-8", errors="replace") HERE = os.path.dirname(os.path.abspath(__file__)) CACHE = os.path.join(HERE, "cache") DOCS = os.path.join(CACHE, "docs") BROWSE = os.path.join(CACHE, "browse") BASE = "https://solveathome.org" PROJ = BASE + "/projects/twin-primes" UA = "sah-research-agent/1.0 (reader-side gate; anonymous)" def get(url, timeout=90, tries=3): for k in range(tries): req = urllib.request.Request(url) req.add_header("Accept", "*/*") req.add_header("User-Agent", UA) try: with urllib.request.urlopen(req, timeout=timeout) as r: return r.status, r.read() except urllib.error.HTTPError as e: if e.code in (429, 500, 502, 503, 504) and k < tries - 1: time.sleep(1.5 * (k + 1)) continue return e.code, e.read()[:200] except Exception: if k < tries - 1: time.sleep(1.0) continue return 0, b"" return 0, b"" def jget(url): st, b = get(url) if st != 200: return st, None try: return st, json.loads(b.decode("utf-8", "replace")) except Exception: return st, None HDR = re.compile(r"^(?:\+\+\+|---|\*\*\*)\s+(.*)$") def clean_path(raw): """`b/research/x.md2026-09-11 06:46:35` -> `research/x.md`; /dev/null -> None.""" p = raw.strip().split("\t")[0].strip().split(" ")[0].strip() p = p.strip('"') if p in ("/dev/null", "dev/null", ""): return None for pre in ("a/", "b/"): if p.startswith(pre): p = p[2:] return p def sections(patch): """Split a unified diff into per-file sections: [(old_path, new_path, added, removed)].""" secs, cur = [], None for ln in (patch or "").splitlines(): if ln.startswith("--- "): cur = {"old": clean_path(ln[4:]), "new": None, "add": [], "rem": []} secs.append(cur) continue if ln.startswith("+++ "): if cur is None: cur = {"old": None, "new": None, "add": [], "rem": []} secs.append(cur) cur["new"] = clean_path(ln[4:]) continue if ln.startswith("@@"): continue if cur is None: continue if ln.startswith("+"): cur["add"].append(ln[1:]) elif ln.startswith("-"): cur["rem"].append(ln[1:]) for s in secs: s["add"] = [x for x in s["add"] if x.strip()] s["rem"] = [x for x in s["rem"] if x.strip()] return [s for s in secs if (s["add"] or s["rem"])] def served_bytes(path): f = os.path.join(DOCS, path.replace("/", "__")) return open(f, "rb").read() if os.path.exists(f) else None def version_bytes(url): st, b = get(BASE + url if url.startswith("/") else url) return st, b def main(): rows = [json.loads(l) for l in open(os.path.join(CACHE, "ids.jsonl"), encoding="utf-8")] ids = {r["id"]: r for r in rows if r.get("type")} audits = [r for r in ids.values() if r["type"] == "audit"] acc = sorted([r for r in audits if r["status"] == "accepted"], key=lambda r: r["id"]) acc_rev = [r for r in acc if r["patch_chars"] or r["revision_sha"]] out = {"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "instrument": "audit-gate.py", "audit_lane_total": len(audits), "accepted": len(acc), "accepted_with_revision": len(acc_rev), "accepted_ids": [r["id"] for r in acc], "rows": [], "controls": {}} # ---- /history + version bytes (anonymous) ------------------------------------------ hist_cache, file_cache = {}, {} def hist(path): if path not in hist_cache: st, obj = jget(PROJ + "/history/" + path) hist_cache[path] = (st, obj) return hist_cache[path] def fbytes(sha): if sha not in file_cache: st, b = get(BASE + "/files/" + sha) file_cache[sha] = (st, b) return file_cache[sha] work = [] for r in acc_rev: b = json.load(open(os.path.join(BROWSE, "%d.json" % r["id"]), encoding="utf-8")) secs = sections(b.get("patch") or "") cands, seen = [], set() for s in secs: for key in ("new", "old"): p = s.get(key) if p and p not in seen and served_bytes(p) is not None: seen.add(p) cands.append(p) rp = b.get("revision_path") or r.get("revision_path") for p in (rp, r.get("declared_target")): if p and p not in seen: seen.add(p) cands.append(p) work.append({"row": r, "body": b, "sections": secs, "candidates": cands}) for p in cands + [rp]: if p: hist(p) with ThreadPoolExecutor(max_workers=8) as ex: list(ex.map(lambda p: hist(p), list(hist_cache.keys()))) shas = [] for p, (st, obj) in hist_cache.items(): for v in ((obj or {}).get("versions") or []): sha = v.get("content_sha") or (v.get("content_url") or "").rsplit("/", 1)[-1] if sha: shas.append(sha) list(ex.map(fbytes, sorted(set(shas)))) for w in work: r, b = w["row"], w["body"] row = {"return": r["id"], "status": r["status"], "patch_chars": r["patch_chars"], "revision_sha": r["revision_sha"], "declared_target": r["declared_target"], "patch_status": r["patch_status"], "effects_applied_at": r["effects_applied_at"], "sections": [{"old": s["old"], "new": s["new"], "added": len(s["add"]), "removed": len(s["rem"])} for s in w["sections"]], "candidates": w["candidates"]} best = None for p in w["candidates"]: st, obj = hist(p) vers = (obj or {}).get("versions") or [] sb = served_bytes(p) sb_sha = hashlib.sha256(sb).hexdigest() if sb is not None else None served_text = sb.decode("utf-8", "replace") if sb is not None else "" # the section(s) that name this path sec_add, sec_rem = [], [] for s in w["sections"]: if p in (s["new"], s["old"]): sec_add += s["add"] sec_rem += s["rem"] inherited = False if not sec_add: # the patch's headers name paths that are not served (staging filenames). The # return still declares which document it revises, so the same line test runs # against that document -- disclosed as an inherited-target test. rp2 = b.get("revision_path") or r.get("revision_path") if p != rp2: continue inherited = True for s in w["sections"]: sec_add += s["add"] sec_rem += s["rem"] if not sec_add and not r["revision_sha"]: continue meta = {"path": p, "inherited_target": inherited, "served_sha256": sb_sha, "history_http": st, "served_present": sb is not None, "timestamps_sha256": ((obj or {}).get("timestamps") or {}).get("sha256"), "versions": [], "added_lines": len(sec_add), "removed_lines": len(sec_rem)} for v in vers: sha = v.get("content_sha") vst, vb = fbytes(sha) vsha = hashlib.sha256(vb).hexdigest() if vst == 200 else None vt = vb.decode("utf-8", "replace") if vst == 200 else "" meta["versions"].append({ "version": v.get("version"), "return_id": v.get("return_id"), "content_sha": sha, "created_at": v.get("created_at"), "sha_of_fetched_bytes_matches": vsha == sha, "added_present": sum(1 for x in sec_add if x in vt) if vst == 200 else None, "added_all_present": (sum(1 for x in sec_add if x in vt) == len(sec_add)) if vst == 200 else None, "removed_present": sum(1 for x in sec_rem if x in vt) if vst == 200 else None, }) cur_add = sum(1 for x in sec_add if x in served_text) cur_rem = sum(1 for x in sec_rem if x in served_text) base = meta["versions"][0] if meta["versions"] else None base_all = base["added_all_present"] if base else None meta.update({ "current_added_present": cur_add, "current_added_all": cur_add == len(sec_add), "current_removed_present": cur_rem, "served_equals_newest_version": next( (v["content_sha"] == sb_sha for v in reversed(meta["versions"])), None), "base_added_present": base["added_present"] if base else None, "base_added_all": base_all, "base_removed_present": base["removed_present"] if base else None, }) carrier = (next((v for v in meta["versions"] if v["added_all_present"]), None) if sec_add else None) meta["carrier"] = ({"version": carrier["version"], "return_id": carrier["return_id"], "created_at": carrier["created_at"], "content_sha": carrier["content_sha"]} if carrier else None) meta["revision_sha_is_a_version"] = any( v["content_sha"] == r["revision_sha"] for v in meta["versions"]) newest = meta["versions"][-1] if meta["versions"] else None meta["revision_sha_is_newest_version"] = bool( newest and r["revision_sha"] and newest["content_sha"] == r["revision_sha"]) meta["served_equals_revision_sha"] = bool(r["revision_sha"] and sb_sha == r["revision_sha"]) if meta["served_equals_revision_sha"] or meta["revision_sha_is_newest_version"]: meta["verdict"] = "REVISION-SERVED" # byte identity, strongest meta["carrier"] = meta["carrier"] or { "version": newest["version"] if newest else None, "return_id": newest["return_id"] if newest else None, "created_at": newest["created_at"] if newest else None, "content_sha": r["revision_sha"]} elif sec_add and cur_add == len(sec_add) and base_all is False: meta["verdict"] = "REVISION-SERVED" # text, all lines, base lacks them elif meta["revision_sha_is_a_version"]: meta["verdict"] = "REVISION-RECORDED-REVERTED" elif carrier is not None and not meta["current_added_all"]: meta["verdict"] = "REVISION-RECORDED-REVERTED" elif r["revision_sha"]: meta["verdict"] = "REACHABLE-NOT-ANY-VERSION" else: meta["verdict"] = "NO-OBSERVABLE-PRODUCT" row.setdefault("paths", []).append(meta) rank = {"REVISION-SERVED": 4, "REVISION-RECORDED-REVERTED": 3, "REACHABLE-NOT-ANY-VERSION": 2, "NO-OBSERVABLE-PRODUCT": 1} if best is None or rank[meta["verdict"]] > rank[best["verdict"]]: best = meta row["primary"] = {k: best[k] for k in ("path", "verdict", "carrier", "current_added_present", "added_lines", "current_removed_present", "removed_lines", "base_added_present", "revision_sha_is_a_version", "served_equals_newest_version", "served_sha256")} if best else None out["rows"].append(row) # ---- controls ----------------------------------------------------------------------- files_ok, files_n = 0, 0 for sha, (st, b) in file_cache.items(): if st == 200: files_n += 1 files_ok += hashlib.sha256(b).hexdigest() == sha neg = [] served_ok, served_n = 0, 0 for r in out["rows"]: for m in r.get("paths", []): if m.get("served_sha256") and m.get("timestamps_sha256"): served_n += 1 served_ok += m["served_sha256"] == m["timestamps_sha256"] c2 = [] for r in out["rows"]: for m in r.get("paths", []): if m["current_added_all"] and m["added_lines"]: c2.append({"return": r["return"], "path": m["path"], "base_lacks_added": m["base_added_all"] is False, "base_keeps_removed": (m["base_removed_present"] or 0) == m["removed_lines"] if m["removed_lines"] else None, "verdict": m["verdict"]}) out["controls"] = { "C3_version_bytes_match_content_sha": {"checked": files_n, "match": files_ok, "pass": files_ok == files_n and files_n > 0}, "C4_served_bytes_match_history_timestamps_sha": {"checked": served_n, "match": served_ok, "pass": served_ok == served_n and served_n > 0}, "C1_negative": neg or {"note": "added lines are additionally tested against the base " "version of the SAME path (base_lacks_added) and against " "other audits' targets in audit-lane.py"}, "C2_direction": {"rows_where_current_has_all_added_lines": c2, "pass": all(x["base_lacks_added"] for x in c2) if c2 else None}, } out["verdict_counts"] = {} for r in out["rows"]: key = (r["primary"] or {}).get("verdict", "NO-PATH-RESOLVED") out["verdict_counts"][key] = out["verdict_counts"].get(key, 0) + 1 out["propagation_paths"] = [ {"return": r["return"], "path": r["primary"]["path"], "carrier": r["primary"]["carrier"]} for r in out["rows"] if (r["primary"] or {}).get("verdict") == "REVISION-SERVED"] out["no_observable_product"] = [ r["return"] for r in out["rows"] if (r["primary"] or {}).get("verdict", "NO-PATH-RESOLVED") not in ("REVISION-SERVED", "REVISION-RECORDED-REVERTED")] with open(os.path.join(HERE, "audit-gate.json"), "w", encoding="utf-8") as f: json.dump(out, f, indent=1) print("audit lane %d returns, accepted %d, accepted-with-revision %d" % (out["audit_lane_total"], out["accepted"], out["accepted_with_revision"])) print("%-6s %-42s %-12s %-8s %-26s %s" % ("id", "path", "verdict", "+served", "carrier", "base+")) for r in out["rows"]: for m in r.get("paths", []): print("%-6d %-42s %-12s %-8s %-26s %s" % ( r["return"], m["path"][-42:], m["verdict"], "%s/%s" % (m["current_added_present"], m["added_lines"]), ("v%s by #%s %s" % (m["carrier"]["version"], m["carrier"]["return_id"], (m["carrier"]["created_at"] or "")[:19])) if m["carrier"] else "-", m["base_added_present"])) if not r.get("paths"): print("%-6d %-42s %-12s %-8s %-26s %s" % (r["return"], "-", "NO-PATH-RESOLVED", "-", "-", "-")) print() print("verdict counts:", json.dumps(out["verdict_counts"])) print("controls:", json.dumps(out["controls"], indent=1)[:900]) if __name__ == "__main__": main()