#!/usr/bin/env python3
"""Job #4540 (route 100 step check): fetch the served record the comparison needs.
Public, credential-free endpoints only:
GET /projects/twin-primes/return/
GET /projects/twin-primes/research-routes/
GET /files/
Every retrieved record is written verbatim into ./served/; a manifest records what exists and
what is absent, so the step check is reproducible from the raw bytes. Ids that 404 are recorded
as absent -- never invented.
"""
import json
import os
import sys
HERE = os.path.dirname(os.path.abspath(__file__))
ROOT = os.path.abspath(os.path.join(HERE, "..", ".."))
sys.path.insert(0, os.path.join(ROOT, ".solveathome", "private", "lib"))
from served import served_evidence as se # noqa: E402
DEST = os.path.join(HERE, "served")
BASE = se.DEFAULT_BASE
# The step-setter's required sources and the route's own returns (the brief names
# 1264, 1267, 1833, 2018, 2024); then the three returns the brief names as comparisons
# (2026, 2027) or that sit between them (2025); then a probe of the id range above the
# current top, so "nothing newer" is a fact about the record and not about our fetch list.
IDS = [1264, 1267, 1804, 1833, 2018, 2024, 2025, 2026, 2027]
PROBE = range(2028, 2061)
ROUTES = ["route:100", "route:127"]
# Attached files we build on, fetched by the sha256 the return itself publishes.
ATTACH = {1264: ["attack_q1q2.py", "paper-boundary-sandwich.md"],
1267: ["probe_drop.py", "verify_drop2.out", "verify_drop2.py"],
1804: ["kfork.c", "prereg-2785.md", "sweep2785.json", "sweep2785.py"],
1833: ["census2612.json", "census2612.py", "dbound2612.json", "dbound2612.py",
"prereg-2612.md", "sweep2612.json", "sweep2612.jsonl", "sweep2612.py",
"xcheck2612.out", "xcheck2612.sh"],
2024: ["REPORT.md", "check-job4528.py", "corpus.jsonl", "scan.json",
"stepcheck4528.py"]}
def get(url, name):
try:
raw = se.fetch_bytes(url, timeout=45)
except Exception as exc: # missing ids 404; record the absence, do not invent
return {"name": name, "error": "%s" % type(exc).__name__}
with open(os.path.join(DEST, name), "wb") as fh:
fh.write(raw)
try:
body = json.loads(raw)
except Exception:
return {"name": name, "bytes": len(raw), "error": "not-json"}
body = se.unwrap(body)
return {"name": name, "bytes": len(raw), "id": body.get("id"),
"route": body.get("research_route_id"), "created_at": body.get("created_at"),
"status": body.get("status")}
def main():
os.makedirs(DEST, exist_ok=True)
manifest = {"returns": [], "routes": [], "attached": [], "absent": []}
seen = {}
for rid in list(IDS) + list(PROBE):
info = get(se.return_url(BASE, rid), "return-%d.json" % rid)
if "error" in info:
manifest["absent"].append(rid)
try:
os.remove(os.path.join(DEST, "return-%d.json" % rid))
except OSError:
pass
else:
manifest["returns"].append(info)
seen[rid] = json.load(open(os.path.join(DEST, "return-%d.json" % rid), encoding="utf-8"))
for rid in ROUTES:
num = rid.split(":", 1)[1]
info = get(se.route_url(BASE, num), "route-%s.json" % num)
(manifest["routes"] if "error" not in info else manifest["absent"]).append(
info if "error" in info else info)
for rid, names in ATTACH.items():
rec = seen.get(rid)
if rec is None:
continue
files = {f["name"]: f["sha256"] for f in se.attached(rec)}
for nm in names:
if nm not in files:
manifest["absent"].append("file:%d:%s" % (rid, nm))
continue
info = get(se.file_url(BASE, files[nm]), se.file_name(rid, nm))
info["return"] = rid
info["sha256_published"] = files[nm]
manifest["attached"].append(info)
with open(os.path.join(HERE, "fetch_manifest.json"), "w", encoding="utf-8", newline="\n") as fh:
json.dump(manifest, fh, indent=1, sort_keys=True)
fh.write("\n")
print("returns fetched: %d; absent: %d; routes: %d; attached: %d"
% (len(manifest["returns"]), len(manifest["absent"]),
len(manifest["routes"]), len(manifest["attached"])))
top = sorted(r["id"] for r in manifest["returns"])
print("ids %s .. %s" % (top[0], top[-1]))
return 0
if __name__ == "__main__":
sys.exit(main())