#!/usr/bin/env python3 """Job #4540 (route 100 step check): fetch the served record the comparison needs. Public, credential-free endpoints only: GET /projects/twin-primes/return/ GET /projects/twin-primes/research-routes/ GET /files/ Every retrieved record is written verbatim into ./served/; a manifest records what exists and what is absent, so the step check is reproducible from the raw bytes. Ids that 404 are recorded as absent -- never invented. """ import json import os import sys HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) sys.path.insert(0, os.path.join(ROOT, ".solveathome", "private", "lib")) from served import served_evidence as se # noqa: E402 DEST = os.path.join(HERE, "served") BASE = se.DEFAULT_BASE # The step-setter's required sources and the route's own returns (the brief names # 1264, 1267, 1833, 2018, 2024); then the three returns the brief names as comparisons # (2026, 2027) or that sit between them (2025); then a probe of the id range above the # current top, so "nothing newer" is a fact about the record and not about our fetch list. IDS = [1264, 1267, 1804, 1833, 2018, 2024, 2025, 2026, 2027] PROBE = range(2028, 2061) ROUTES = ["route:100", "route:127"] # Attached files we build on, fetched by the sha256 the return itself publishes. ATTACH = {1264: ["attack_q1q2.py", "paper-boundary-sandwich.md"], 1267: ["probe_drop.py", "verify_drop2.out", "verify_drop2.py"], 1804: ["kfork.c", "prereg-2785.md", "sweep2785.json", "sweep2785.py"], 1833: ["census2612.json", "census2612.py", "dbound2612.json", "dbound2612.py", "prereg-2612.md", "sweep2612.json", "sweep2612.jsonl", "sweep2612.py", "xcheck2612.out", "xcheck2612.sh"], 2024: ["REPORT.md", "check-job4528.py", "corpus.jsonl", "scan.json", "stepcheck4528.py"]} def get(url, name): try: raw = se.fetch_bytes(url, timeout=45) except Exception as exc: # missing ids 404; record the absence, do not invent return {"name": name, "error": "%s" % type(exc).__name__} with open(os.path.join(DEST, name), "wb") as fh: fh.write(raw) try: body = json.loads(raw) except Exception: return {"name": name, "bytes": len(raw), "error": "not-json"} body = se.unwrap(body) return {"name": name, "bytes": len(raw), "id": body.get("id"), "route": body.get("research_route_id"), "created_at": body.get("created_at"), "status": body.get("status")} def main(): os.makedirs(DEST, exist_ok=True) manifest = {"returns": [], "routes": [], "attached": [], "absent": []} seen = {} for rid in list(IDS) + list(PROBE): info = get(se.return_url(BASE, rid), "return-%d.json" % rid) if "error" in info: manifest["absent"].append(rid) try: os.remove(os.path.join(DEST, "return-%d.json" % rid)) except OSError: pass else: manifest["returns"].append(info) seen[rid] = json.load(open(os.path.join(DEST, "return-%d.json" % rid), encoding="utf-8")) for rid in ROUTES: num = rid.split(":", 1)[1] info = get(se.route_url(BASE, num), "route-%s.json" % num) (manifest["routes"] if "error" not in info else manifest["absent"]).append( info if "error" in info else info) for rid, names in ATTACH.items(): rec = seen.get(rid) if rec is None: continue files = {f["name"]: f["sha256"] for f in se.attached(rec)} for nm in names: if nm not in files: manifest["absent"].append("file:%d:%s" % (rid, nm)) continue info = get(se.file_url(BASE, files[nm]), se.file_name(rid, nm)) info["return"] = rid info["sha256_published"] = files[nm] manifest["attached"].append(info) with open(os.path.join(HERE, "fetch_manifest.json"), "w", encoding="utf-8", newline="\n") as fh: json.dump(manifest, fh, indent=1, sort_keys=True) fh.write("\n") print("returns fetched: %d; absent: %d; routes: %d; attached: %d" % (len(manifest["returns"]), len(manifest["absent"]), len(manifest["routes"]), len(manifest["attached"]))) top = sorted(r["id"] for r in manifest["returns"]) print("ids %s .. %s" % (top[0], top[-1])) return 0 if __name__ == "__main__": sys.exit(main())