#!/usr/bin/env python3 # -*- coding: utf-8 -*- """Fetch served /files/ artifacts into /evidence/job1468/, verifying each digest. Corrected copy for job #1473 (consolidating return #672 / job #1468). The only substantive change is the credential location; the fetch, the digest check, the output layout and the manifest record fields are those of the shipped file. What changed, and only this ---------------------------- * The shipped copy hard-coded, at line 22, an absolute per-user Windows profile path to one account's credential file. That path exists on one machine only and is machine-specific, so anywhere else the fetch died in `open()`. The credential is now resolved *relative to the repository* by `token_file()`: 1. `$SOLVEATHOME_TOKEN_FILE` - explicit override: an absolute path, or a path relative to the repository root found below; 2. `/.solveathome/credentials.token` - the repository-relative slot, if a checkout keeps its credential inside the department tree; 3. the per-user protected store outside the tree, `$SOLVEATHOME_CREDENTIALS`, else `$XDG_CONFIG_HOME/solveathome/credentials.env`, else `~/.config/solveathome/ credentials.env` - a home-relative convention, not a hard-coded user name. Both a raw token file and a `SOLVEATHOME_TOKEN=` env-style file are accepted. The repository root is the nearest ancestor of the run directory (else of this file) that contains a `.solveathome` directory; it is printed to stderr so a reader can see which one was found. * The credential is now OPTIONAL. `GET /files/` is a public read-only endpoint and an anonymous request returns the same bytes as an authenticated one (checked both ways for every sha below), so a machine with no credential file still reproduces the fetch instead of dying in `open()`. A missing credential is reported on stderr, not raised. * Progress and elapsed time now go to stderr; stdout keeps exactly the machine-readable one-record-per-file JSON of the shipped copy (plus an `auth` field), so an existing reader of the manifest is unaffected. No random draws are made anywhere in this script, so there is no seed to set: the output is deterministic and byte-reproducible. Read-only GETs against this project only. When a credential is used it is read from the protected store inside the process; it never appears in argv, in stdout or in the saved files. Usage: python3 job1468-fetch-files.py [ ...] """ import hashlib import json import os import sys import time import urllib.error import urllib.request # The content store is at the SERVER ROOT, not under the project: a served artifact is # https://solveathome.org/files/. Under /projects/twin-primes/... every one of # /files, /files/, /file/, /return//files and /return//file/ answers 404 # "no such route", which is worth one probe and then a look at the shared tool's own URL builder. BASE = "https://solveathome.org" def repo_root(start=None): """Nearest ancestor of `start` (default: this file's directory) containing `.solveathome`. The department tree is the repository here. Returns None when no marker is found (for example when the script is run from a scratch directory with no checkout above it), in which case only `$SOLVEATHOME_TOKEN_FILE` can name a credential. """ here = os.path.abspath(start or os.path.dirname(os.path.abspath(__file__))) while True: if os.path.isdir(os.path.join(here, ".solveathome")): return here parent = os.path.dirname(here) if parent == here: return None here = parent def _read_token(path): """A raw token file or an env-style file with a SOLVEATHOME_TOKEN= line; "" if not a file.""" try: text = open(path, encoding="utf-8").read() except OSError: return "" for line in text.splitlines(): line = line.strip() if line.startswith("SOLVEATHOME_TOKEN="): return line.split("=", 1)[1].strip().strip("'\"") return text.strip() def token_file(root): """First existing credential of the documented order; None when the caller has none. Paths 1 and 2 are repository-relative (path 1 relative to the repository root when it is not absolute), path 3 is the per-user protected store outside the tree. """ cands = [] env = os.environ.get("SOLVEATHOME_TOKEN_FILE") if env: cands.append(env if os.path.isabs(env) else os.path.join(root or os.getcwd(), env)) if root: cands.append(os.path.join(root, ".solveathome", "credentials.token")) protected = os.environ.get("SOLVEATHOME_CREDENTIALS") if not protected: base = os.environ.get("XDG_CONFIG_HOME") or os.path.join(os.path.expanduser("~"), ".config") protected = os.path.join(base, "solveathome", "credentials.env") cands.append(protected) for c in cands: if os.path.isfile(c): return c return None def main(run, pairs): t0 = time.time() out = os.path.join(run, "evidence", "job1468") os.makedirs(out, exist_ok=True) root = repo_root(run) cred = token_file(root) tok = _read_token(cred) if cred else "" log = lambda msg: sys.stderr.write("[fetch] %s\n" % msg) log("repo_root=%s datei=%s credential=%s (%s)" % (root, os.path.relpath(out, root) if root and not out.startswith(os.pardir) else out, cred or "", ("token %d chars" % len(tok)) if tok else "anonymous")) manifest = [] for i, spec in enumerate(pairs, 1): name, sha = spec.split("=", 1) url = "%s/files/%s" % (BASE, sha) headers = {"Authorization": "Bearer " + tok} if tok else {} req = urllib.request.Request(url, headers=headers) t1 = time.time() try: with urllib.request.urlopen(req, timeout=120) as r: body, status = r.read(), r.status except urllib.error.HTTPError as e: body, status = e.read(), e.code p = os.path.join(out, name) with open(p, "wb") as fh: fh.write(body) got = hashlib.sha256(body).hexdigest() rec = {"name": name, "sha256_declared": sha, "sha256_actual": got, "matches": got == sha, "bytes": len(body), "http_status": status, "path": p, "auth": "token" if tok else "anonymous", "elapsed_s": round(time.time() - t1, 3)} manifest.append(rec) log("%d/%d %s http=%s bytes=%d sha_ok=%s %.3fs" % (i, len(pairs), name, status, len(body), got == sha, rec["elapsed_s"])) with open(os.path.join(out, "manifest.json"), "w", encoding="utf-8", newline="\n") as fh: json.dump(manifest, fh, indent=1, sort_keys=True) fh.write("\n") for rec in manifest: print(json.dumps(rec, sort_keys=True)) ok = all(r["matches"] for r in manifest) log("all match: %s; %d file(s), %.3fs total" % (ok, len(manifest), time.time() - t0)) print("all match:", ok) return 0 if ok else 1 if __name__ == "__main__": sys.exit(main(sys.argv[1], sys.argv[2:]))