#!/usr/bin/env python3 """Job #2972 (route 114): anonymous reader-side fetch harness for the audit-lane census. Every claim in this job is about what a reader sees, so every GET here carries NO Authorization header. Responses are cached under cache/ so the numbers re-derive offline. usage: sweep.py probe find the highest return id that answers 200 sweep.py ids MAX walk /return/ for 1..MAX -> cache/ids.jsonl (+ browse bodies) sweep.py retry [SLEEP] re-walk only the rows that came back 429/5xx, SLOWLY, and merge sweep.py walk recursive /docs walk -> cache/listing.json (with its joined sha256) sweep.py docs GET every path in cache/listing.json -> cache/docs/ """ import json import os import sys import threading import time import urllib.error import urllib.request from concurrent.futures import ThreadPoolExecutor sys.stdout.reconfigure(encoding="utf-8", errors="replace") HERE = os.path.dirname(os.path.abspath(__file__)) CACHE = os.path.join(HERE, "cache") BASE = "https://solveathome.org" PROJ = "%s/projects/twin-primes" % BASE J = "%s/projects/twin-primes/return/" % BASE D = "%s/projects/twin-primes/docs/" % BASE UA = "sah-research-agent/1.0 (reader-side census; anonymous)" lock = threading.Lock() counts = {"ok": 0, "404": 0, "err": 0} def get(url, timeout=90, tries=3): """Anonymous GET. Never sends a credential; returns (status, bytes).""" for k in range(tries): req = urllib.request.Request(url) req.add_header("Accept", "*/*") req.add_header("User-Agent", UA) try: with urllib.request.urlopen(req, timeout=timeout) as r: return r.status, r.read() except urllib.error.HTTPError as e: if e.code in (429, 500, 502, 503, 504) and k < tries - 1: time.sleep(1.5 * (k + 1)) continue return e.code, e.read()[:400] except Exception: if k < tries - 1: time.sleep(1.0 * (k + 1)) continue return 0, b"TRANSPORT_FAILED" return 0, b"TRANSPORT_FAILED" def jget(url): st, body = get(url) if st != 200: return st, None try: return st, json.loads(body.decode("utf-8", "replace")) except Exception: return st, None def bump(k): with lock: counts[k] += 1 def row_of(rid, d): """Compact census row: only what the two questions need.""" patch = d.get("patch") or "" return { "id": rid, "type": d.get("type"), "status": d.get("status"), "lane_id": d.get("lane_id"), "final_rung": d.get("final_rung"), "patch_chars": len(patch), "patch_hash": d.get("patch_hash"), "revision_sha": d.get("revision_sha"), "revision_path": d.get("revision_path"), "declared_target": d.get("target"), "patch_status": (d.get("patch_status") or "")[:60], "effects_applied_at": d.get("effects_applied_at"), "created_at": d.get("created_at"), "model": d.get("model"), "handle": d.get("handle"), "research_route_id": d.get("research_route_id"), "cites": d.get("cites"), "depends_on": d.get("dependencies"), "depends_on_flat": d.get("depends_on"), "also_fix": d.get("also_fix"), "superseded_by": d.get("superseded_by"), "duplicate_of": d.get("duplicate_of"), } ACCEPTED = {"accepted", "proven", "verified"} def keep_body(r): return ((r.get("type") == "audit") or (r.get("status") in ACCEPTED and ((r.get("patch") or "") or r.get("revision_sha")))) def mode_probe(): lo, hi = 1, 4096 # highest id answering 200, by exponential then linear-down search lo = 1400 while lo < hi: mid = (lo + hi + 1) // 2 st, _ = get(J + str(mid)) if st == 200: lo = mid else: hi = mid - 1 print(json.dumps({"highest_200": lo, "next_status": get(J + str(lo + 1))[0]})) with open(os.path.join(CACHE, "probe.json"), "w", encoding="utf-8") as f: json.dump({"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "highest_200": lo, "next_status": get(J + str(lo + 1))[0]}, f, indent=1) def mode_ids(maxid): os.makedirs(CACHE, exist_ok=True) browse = os.path.join(CACHE, "browse") os.makedirs(browse, exist_ok=True) def one(rid): st, d = jget(J + str(rid)) if st == 200 and isinstance(d, dict): bump("ok") r = row_of(rid, d) if keep_body(d): with open(os.path.join(browse, "%d.json" % rid), "w", encoding="utf-8") as f: json.dump(d, f) return r bump("404" if st in (404, 403) else "err") return {"id": rid, "status": "HTTP_%s" % st} rows = [] with ThreadPoolExecutor(max_workers=10) as ex: for r in ex.map(one, range(1, maxid + 1)): rows.append(r) if len(rows) % 200 == 0: print(" ...%d" % len(rows), flush=True) rows.sort(key=lambda x: x["id"]) out = os.path.join(CACHE, "ids.jsonl") with open(out, "w", encoding="utf-8") as f: for r in rows: f.write(json.dumps(r, sort_keys=True) + "\n") print(json.dumps({"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "maxid": maxid, "counts": counts, "rows": len(rows), "found": sum(1 for r in rows if r.get("status") != "HTTP_404")}, indent=1)) def mode_retry(sleep=1.0): """Re-fetch only the rows that answered 429/5xx/transport, one at a time.""" path = os.path.join(CACHE, "ids.jsonl") rows = [json.loads(l) for l in open(path, encoding="utf-8")] todo = [r["id"] for r in rows if r.get("status") in (None,) or str(r.get("status", "")).startswith("HTTP_")] todo = [i for i in todo if rows[i - 1].get("status") != "HTTP_404"] print("retrying %d ids: %s" % (len(todo), todo[:20]), flush=True) browse = os.path.join(CACHE, "browse") for n, rid in enumerate(todo): st, d = jget(J + str(rid)) if st == 200 and isinstance(d, dict): rows[rid - 1] = row_of(rid, d) if keep_body(d): with open(os.path.join(browse, "%d.json" % rid), "w", encoding="utf-8") as f: json.dump(d, f) print(" %d -> 200 (%s/%s)" % (rid, d.get("type"), d.get("status")), flush=True) elif st in (404, 403): # a real absence, not a rate limit: record it as absence (the first pass may have # seen the same id as 429 and left a placeholder) rows[rid - 1] = {"id": rid, "status": "HTTP_%d" % st} b = os.path.join(browse, "%d.json" % rid) if os.path.exists(b): os.remove(b) print(" %d -> %s (absence)" % (rid, st), flush=True) else: print(" %d -> %s (UNRESOLVED, kept)" % (rid, st), flush=True) time.sleep(sleep) with open(path, "w", encoding="utf-8") as f: for r in rows: f.write(json.dumps(r, sort_keys=True) + "\n") left = [r["id"] for r in rows if str(r.get("status", "")).startswith("HTTP_")] print(json.dumps({"retried": len(todo), "still_unresolved": left})) def mode_walk(): """Recursive /docs walk: the endpoint returns a newline list; directories end in '/'.""" os.makedirs(CACHE, exist_ok=True) files, seen_dirs = [], set() todo = [""] while todo: d = todo.pop(0) if d in seen_dirs: continue seen_dirs.add(d) st, body = get(PROJ + "/docs/" + d) if st != 200: print(" dir %r -> %s" % (d, st), flush=True) continue txt = body.decode("utf-8", "replace") if "\x00" in txt[:200] or " non-listing body" % d, flush=True) continue for line in txt.splitlines(): name = line.strip() if not name: continue if name.endswith("/"): todo.append(d + name) else: files.append(d + name) import hashlib sha = hashlib.sha256("\n".join(files).encode()).hexdigest() out = {"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "endpoint": "/docs (recursive walk)", "n": len(files), "dirs": len(seen_dirs), "sha256_of_joined_paths": sha, "paths": files} with open(os.path.join(CACHE, "listing.json"), "w", encoding="utf-8") as f: json.dump(out, f, indent=1) print(json.dumps({"at": out["at"], "n": len(files), "dirs": len(seen_dirs), "sha256": sha})) def mode_docs(): lst = json.load(open(os.path.join(CACHE, "listing.json"), encoding="utf-8")) paths = lst["paths"] root = os.path.join(CACHE, "docs") os.makedirs(root, exist_ok=True) import hashlib man = [] def one(p): dest = os.path.join(root, p.replace("/", "__")) st, body = get(D + p) if st == 200: with open(dest, "wb") as f: f.write(body) bump("ok") return {"path": p, "status": st, "bytes": len(body), "sha256": hashlib.sha256(body).hexdigest()} bump("err") return {"path": p, "status": st, "bytes": None, "sha256": None} with ThreadPoolExecutor(max_workers=10) as ex: for i, m in enumerate(ex.map(one, paths)): man.append(m) if (i + 1) % 200 == 0: print(" ...%d/%d" % (i + 1, len(paths)), flush=True) with open(os.path.join(CACHE, "docs-manifest.json"), "w", encoding="utf-8") as f: json.dump({"at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "listing_sha256": lst.get("sha256_of_joined_paths"), "n": len(man), "unread": [m["path"] for m in man if m["status"] != 200], "files": man}, f, indent=1) print(json.dumps({"n": len(man), "unread": sum(1 for m in man if m["status"] != 200), "counts": counts})) if __name__ == "__main__": os.makedirs(CACHE, exist_ok=True) cmd = sys.argv[1] if len(sys.argv) > 1 else "" if cmd == "probe": mode_probe() elif cmd == "ids": mode_ids(int(sys.argv[2])) elif cmd == "retry": mode_retry(float(sys.argv[2]) if len(sys.argv) > 2 else 1.0) elif cmd == "walk": mode_walk() elif cmd == "docs": mode_docs() else: sys.exit(__doc__)