#!/usr/bin/env python3 """accepted-text-published.py -- when a return's accepted text MOVES, is the OLD text still served? The watch (census.py) raises ACCEPTED TEXT MOVED when a return's `patch_sha` -- the digest of its own patch text -- differs between two dated measurements. That alert answers a question about the RECORD. It does not answer the question a reader actually has: **are the bytes that were accepted EARLIER still retrievable?** The store half of this corpus answers anonymously at the HOST ROOT (`https://solveathome.org/files/`, not under the project prefix -- that was the #1453 bug), so the question is measurable, and the answer decides the character of a moved patch: old text still resolves -> a REWRITE WITH A TOMBSTONE: the earlier accepted text is still citable by its own digest, so the move is additive; old text stopped resolving -> a REPLACEMENT: the earlier accepted text is gone from the served surface, and the only record of it is whatever the return says. Two readings per id, one response each: * `old_text_resolves` -- GET /files/ anonymously; 200 AND sha256(body) == old_sha; * `record_says` -- GET /return/ and compare the return's own published digests (`patch_hash`, `patch_sha`, `revision_sha`) against the old and new values: whether the RECORD moved with the bytes or still names the old text. The population comes from the census output's own `moved_values` (`--watch`), so the ids and the pair of digests are the ones the alert was raised about rather than a re-derivation that could disagree. Controls (sahcontrol). Both readings are booleans and a False is a claim, so: must-find a digest the return itself DECLARED resolves anonymously, byte-for-byte; must-find the #1453 pin: the same digest is NOT served under the project prefix. This is a must-find on a SECOND reading, not a negative control: the observation is "the project-prefixed URL does not serve it", and a control that expects the object must observe exactly that; must-not-find a fixture digest that names nothing is not served at the host root. python accepted-text-published.py --selftest python accepted-text-published.py --watch census-out.json python accepted-text-published.py --pair 1453: --pair 1453: """ import argparse import hashlib import io import json import os import sys import time import urllib.error import urllib.request HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = "https://solveathome.org" PROJ = ROOT + "/projects/twin-primes" TOKEN = os.path.join(os.environ.get("LOCALAPPDATA", ""), "solveathome", "credentials", "twin-primes.token") TOOLSTORE = os.path.join(os.environ.get("LOCALAPPDATA", ""), "solveathome", "tools", "v1") sys.path.insert(0, TOOLSTORE) try: from sahcontrol import NoControl, Probe except ImportError as _e: # pragma: no cover raise SystemExit("sahcontrol.py (solveathome/tools/v1) is required: refusing to print a reading " "whose controls did not pass (%s)" % _e) try: from sahdated import write as write_dated # the output names its own instant except ImportError as _e: # pragma: no cover raise SystemExit("sahdated.py (solveathome/tools/v1) is required: a reading of the store is read " "as current and must name when it was taken (%s)" % _e) ABSENT_FIXTURE = b"sahcontrol/absent-fixture: nothing was ever stored under this digest" COUNTS = {"requests": 0} def fetch(url, token=None, timeout=60): headers = {"Accept": "*/*", "User-Agent": "sahreq/1.0"} if token: headers["Authorization"] = "Bearer " + token COUNTS["requests"] += 1 try: with urllib.request.urlopen(urllib.request.Request(url, headers=headers), timeout=timeout) as r: return r.status, r.read() except urllib.error.HTTPError as e: return e.code, e.read() except urllib.error.URLError as e: return 0, json.dumps({"transport_error": str(e)}).encode() def sha_of(b): return hashlib.sha256(b).hexdigest() def served(sha, prefix="root"): """(present, code) -- the PRESENCE observation and the status, never a verdict.""" base = ROOT if prefix == "root" else PROJ code, blob = fetch("%s/files/%s" % (base, sha)) return (code == 200 and sha_of(blob) == sha), code def read_return(rid, token): code, body = fetch("%s/return/%d" % (PROJ, rid), token) if code != 200: return None try: return json.loads(body) except ValueError: return None def published_digests(rr): out = {} for k in ("patch_hash", "patch_sha", "revision_sha"): if rr.get(k): out[k] = rr[k] for f in (rr.get("files") or []): if isinstance(f, dict): for k in ("sha256", "sha", "content_sha"): if f.get(k): out.setdefault("declared_files", []).append(f[k]) return out def read_pair(rid, old_sha, new_sha, token): """One reading: does the OLD text still resolve, and what does the return now name?""" present_old, code_old = served(old_sha) present_new, code_new = served(new_sha) if new_sha else (None, None) rr = read_return(rid, token) digests = published_digests(rr) if rr else {} names = [k for k, v in digests.items() if v == old_sha] names_new = [k for k, v in digests.items() if new_sha and v == new_sha] return { "return": rid, "old_sha": old_sha, "new_sha": new_sha, "old_text_resolves": present_old, "old_text_status": code_old, "new_text_resolves": present_new, "new_text_status": code_new, "record_readable": rr is not None, "record_names_old": names, "record_names_new": names_new, "record_version": rr.get("revision_sha") if rr else None, "record_patch_status": rr.get("patch_status") if rr else None, "read": ("rewrite-with-tombstone" if present_old else "replacement") if old_sha else "no-prior-digest", } def controls(gate, token): """A known-present digest, a known-absent one, and the #1453 URL shape -- before any reading.""" fixture = sha_of(ABSENT_FIXTURE) present, code = served(fixture) gate.must_not_find("a digest that names nothing is not served at the host root", present, detail="%s -> %s" % (fixture[:12], code)) # a known-present digest: the return's own declared file, which the corpus-wide rule says the store # answers for by content address. If this fails the store's shape has changed and no reading below # may be narrated. probe_id, probe_sha = None, None for rid in (1, 2, 3, 300, 96, 97, 151, 1453, 1461): rr = read_return(rid, token) if not rr: continue for s in published_digests(rr).get("declared_files") or []: probe_id, probe_sha = rid, s break if probe_sha: break if probe_sha: ok_present, code_p = served(probe_sha) gate.must_find("a digest the return %s DECLARED resolves anonymously at the host root" % probe_id, ok_present, detail="%s -> %s" % (probe_sha[:12], code_p)) # The observation is `pfx_present` -- the OBJECT under the project prefix -- and the # expectation lives in the name. Writing it the other way round is the exact inversion this # project hit three times in one day, so it is written the way `sahcontrol` asks for it. pfx_present, code_x = served(probe_sha, prefix="proj") gate.must_not_find("the SAME digest is NOT served under the project prefix (the #1453 shape)", pfx_present, detail="%s/files/... -> %s" % (PROJ, code_x)) return probe_id, probe_sha def selftest(args): """Both polarities, on REAL data: an alert whose old text is still served, and one whose old text never existed. A probe that cannot read GONE cannot read STILL SERVED either.""" token = io.open(TOKEN, encoding="utf-8").read().strip().split("=")[-1].strip() gate = Probe("accepted-text-published: is the earlier accepted text still served?") probe_id, probe_sha = controls(gate, token) absent = sha_of(ABSENT_FIXTURE) ok = True if not probe_sha: print("no declared digest found on any probed return: the positive control cannot be built") return 2 good = read_pair(probe_id, probe_sha, None, token) gone = read_pair(probe_id, absent, None, token) good_ok = good["old_text_resolves"] is True and good["read"] == "rewrite-with-tombstone" gone_ok = gone["old_text_resolves"] is False and gone["read"] == "replacement" ok = ok and good_ok and gone_ok lines = [] try: gate.claim("1 of 2 probed prior texts still resolves by its own digest", 1, 2, printer=lines.append) except NoControl as exc: print("REFUSED: %s" % exc) return 3 print("control: declared digest %s -> %s | fixture digest -> %s" % (probe_sha[:12], "served" if good["old_text_resolves"] else "GONE", "served" if gone["old_text_resolves"] else "GONE")) print("ok a still-served prior text reads as rewrite-with-tombstone: %s" % good_ok) print("ok a prior text that names nothing reads as replacement: %s" % gone_ok) print("ok the gate admitted the reading: %s" % lines[0]) print("\naccepted-text publication probe selftest: %s" % ("PASS" if ok else "FAIL")) return 0 if ok else 1 def main(argv): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--watch", help="a census output JSON carrying `moved_values`") ap.add_argument("--pair", action="append", default=[], help="return_id:old_sha[:new_sha] -- the digests the alert was raised about") ap.add_argument("--selftest", action="store_true") ap.add_argument("--out", default=os.path.join(HERE, "accepted-text-published.json")) a = ap.parse_args(argv) if a.selftest: return selftest(a) token = io.open(TOKEN, encoding="utf-8").read().strip().split("=")[-1].strip() pairs = [] for spec in a.pair: bits = spec.split(":") pairs.append((int(bits[0]), bits[1], bits[2] if len(bits) > 2 else None)) watched = None if a.watch: blob = json.load(io.open(a.watch, encoding="utf-8")) watched = blob.get("moved_values") or {} for rid, fields in sorted(watched.items(), key=lambda kv: int(kv[0])): pair = fields.get("patch_sha") if pair and pair[0] and pair[0] != pair[1]: pairs.append((int(rid), pair[0], pair[1])) if not pairs: print("nothing to read: pass --pair or --watch (a census output with `moved_values`). The " "watch only writes that field when a return's accepted text actually moved.") return 2 gate = Probe("accepted-text-published: is the earlier accepted text still served?") probe_id, probe_sha = controls(gate, token) rows = [read_pair(rid, old, new, token) for rid, old, new in pairs] served_n = sum(1 for r in rows if r["old_text_resolves"]) out = {"tool": "accepted-text-published.py/1.0", "at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), "population": "the ids the watch named as ACCEPTED TEXT MOVED, from %s" % (a.watch or "explicit --pair"), "control_positive": {"return": probe_id, "declared_sha": probe_sha}, "controls": [c.as_json() for c in gate.controls()], "rows": rows, "requests": COUNTS["requests"], "summary": {"pairs": len(rows), "old_text_still_served": served_n, "old_text_gone": len(rows) - served_n, "record_moved_with_bytes": sum(1 for r in rows if r["record_names_new"] and not r["record_names_old"])}} try: gate.claim("%d of %d prior accepted texts still resolve by their own digest" % (served_n, len(rows)), served_n, len(rows)) except NoControl as exc: out["claim_refused"] = str(exc) write_dated(a.out, out, observed_at=out["at"], source="the instant the readings below were taken") sys.stderr.write("REFUSED (no reading may be printed):\n%s\nwrote %s\n" % (exc, a.out)) return 3 for r in rows: print(" return %-6d old %s %s | record names old=%s new=%s | %s" % (r["return"], r["old_sha"][:12], "SERVED" if r["old_text_resolves"] else "GONE ", r["record_names_old"] or "-", r["record_names_new"] or "-", r["read"])) write_dated(a.out, out, observed_at=out["at"], source="the instant the readings below were taken") print("wrote %s" % a.out) return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))