#!/usr/bin/env python3 """Download the files attached to a served return, by sha, into a fresh directory. python fetch-served-files.py Anonymous public reads (GET /files/, the contract in protocol/files.md), no credential. Each download is verified against the sha the return declares, so the fresh directory is known to hold the served bytes, not a guess. """ import hashlib import json import os import sys import urllib.request UA = "sah-research-agent/1.0" FILES = "https://solveathome.org/files/" ret = json.load(open(sys.argv[1], encoding="utf-8")) outdir = sys.argv[2] os.makedirs(outdir, exist_ok=True) ok = bad = 0 for f in ret["files"]: sha, name = f["sha256"], f["name"] req = urllib.request.Request(FILES + sha, headers={"User-Agent": UA}) with urllib.request.urlopen(req, timeout=60) as r: body = r.read() got = hashlib.sha256(body).hexdigest() if got != sha: print("MISMATCH %s: got %s" % (name, got)) bad += 1 continue dest = os.path.join(outdir, name) if os.path.exists(dest): dest = os.path.join(outdir, sha[:8] + "-" + name) open(dest, "wb").write(body) print("%-36s %s %d bytes" % (name, sha[:12], len(body))) ok += 1 print("\n%d verified, %d mismatched, out=%s" % (ok, bad, outdir)) sys.exit(1 if bad else 0)