#!/usr/bin/env python3 """census.py -- the dated, incremental return census. `sweep-return-patches.py` is a full census: every id 1..1700, every time, ~2100 requests. This is the same census with a MEMORY, so a later pass pays the delta and can say what CHANGED rather than re-derive everything. Three ideas, and the third is the one that makes the other two safe: 1. BASELINE. A dated state file records the ceiling reached, the ids that existed, the id density, the patch-bearing set, and a per-return fingerprint. A pass diffs against it: appeared, vanished, changed. 2. DELTA. A later pass re-queries, instead of everything: * the FRONTIER -- ids above the recorded ceiling (that is where new returns land); * every previously PATCH-BEARING return (they carry the measurement, so they are never sampled); * the HOT ZONE -- the top `--hot` ids, where the churn is; * a deterministic SAMPLE of the rest of the existing ids, and a sample of the previously ABSENT ids (an id that never existed can still be filled later, so the gaps are sampled too). What is NOT re-queried is therefore stated, with its rate, rather than left implicit: a disappearance outside the hot zone, the patch-bearing set and the sample is seen with the sample's probability and the run says so. 3. THE CHANGE DETECTOR IS ITSELF CONTROLLED. "0 changed" is a zero, and a zero may only be narrated if the detector can return a non-zero: the gate registers a fixture pair whose fingerprints MUST differ, and a pair that must not. Without that, "nothing changed" is indistinguishable from "the fingerprint is constant". 4. THE WATCH. Each pass appends ONE DATED LINE to `census-history.jsonl`, so the series survives the baseline being rewritten and a later pass can be read against an earlier one. And the diff is TYPED, not just a hash: a hash can say that a return moved, never what moved, so the watch could not tell a status change from a movement of the accepted text. Three kinds of movement raise an ALERT (exit 5 unless `--no-alert-exit`), because they are the ones the accepted-patch lane rests on: VANISHED the return was present at the previous measurement and is gone now STATUS MOVED `status` or `patch_status` differs (pending -> accepted, etc.) ACCEPTED TEXT MOVED `patch_sha` (the text's own digest), `revision_sha`, or the declared file shas differ -- the delivered bytes moved under the same id An APPEARED return is recorded in the history but raises no alert: a new return is the corpus working, not the corpus changing under something already accepted. An alert is only as good as the selection it is seen through, so the alert claims print the ids NOT re-queried beside them. The population and evidence claims are the gate's (`sahcontrol`), as in the full sweep -- including one control per alert kind, so "0 alerts" cannot be a diff that can never fire. python census.py --full # first pass: sweep everything, write the baseline python census.py # later pass: delta only, diff against the baseline python census.py --show-history # the series: one dated line per pass (no network) python census.py --sample-rate 0.25 --hot 300 --grow 200 """ import argparse import hashlib import json import os import sys from concurrent.futures import ThreadPoolExecutor HERE = os.path.dirname(os.path.abspath(__file__)) sys.path.insert(0, HERE) sys.path.insert(0, os.path.join(os.environ.get("LOCALAPPDATA", ""), "solveathome", "tools", "v1")) import importlib.util _spec = importlib.util.spec_from_file_location("sweepmod", os.path.join(HERE, "sweep-return-patches.py")) sweep = importlib.util.module_from_spec(_spec) _spec.loader.exec_module(sweep) from sahcontrol import NoControl, Probe # noqa: E402 (the gate is required, not optional) from sahdated import Covered, Undated # noqa: E402 (a baseline must name when it was taken) from sahdated import write as write_dated # noqa: E402 (and a written one must name it itself) STATE = os.path.join(HERE, "census-state.json") def projection(rr): """Everything the census measures about one return, as COMPARABLE FIELDS. Kept as fields and not only as a hash, because a hash can say THAT a return moved and never WHAT moved: the 1.0 state stored only the fingerprint, so its diff could not name a kind of change, and a status change was indistinguishable from a rewrite of the accepted text. """ return { "id": rr.get("id"), "type": rr.get("type"), "status": rr.get("status"), "job_id": rr.get("job_id"), "created_at": rr.get("created_at"), "patch_sha": hashlib.sha256((rr.get("patch") or "").encode("utf-8")).hexdigest() if rr.get("patch") else None, "patch_status": rr.get("patch_status"), "revision_sha": rr.get("revision_sha"), "declared": sorted(sweep.declared_shas(rr)), } def fingerprint(rr): """A canonical projection of everything the census measures about one return.""" return hashlib.sha256(json.dumps(projection(rr), sort_keys=True).encode("utf-8")).hexdigest() STATUS_FIELDS = ("status", "patch_status") TEXT_FIELDS = ("patch_sha", "revision_sha", "declared") def classify(old, new): """What KIND of movement happened between two dated measurements of one return. Returns (changed_fields, status_moved, accepted_text_moved). The two kinds are separated on purpose: a return that goes from `patch_status: pending` to `accepted` has not had a byte of its text touched, and a return whose `patch_sha` moved has not changed status. """ if not old or not new: return [], False, False fields = [k for k in ("status", "patch_status", "patch_sha", "revision_sha", "declared", "type", "job_id", "created_at") if old.get(k) != new.get(k)] return (fields, any(f in STATUS_FIELDS for f in fields), any(f in TEXT_FIELDS for f in fields)) def capable_of(kind, baseline_ids, baseline_proj): """Which ids the BASELINE holds that could POSSIBLY produce an alert of this kind. This is the pre-scope the escalation needs, and it is written next to `classify` on purpose: the two must agree, or a confirmation would stop looking at an id that can still raise the kind it is confirming. The asymmetry between the kinds is the whole content of the function: * VANISHED needs EXISTENCE -- an id the baseline never held cannot disappear. * STATUS / TEXT MOVED need a stored PROJECTION, but NOT a stored value: `classify` compares fields, so a return that had no patch at the baseline and ACQUIRES its first one has moved its accepted text. The tempting narrowing -- "only ids already carrying a patch can move a patch" -- is unsound, and it is the exact shape of the first ascent under a fixed return id, which is the event this watch exists to see. So capability is the projection SET, and a projected id with an empty projection stays in. """ if kind == "vanished": return set(baseline_ids) if kind in ("status", "text"): return set(baseline_proj) return set(baseline_ids) | set(baseline_proj) ALERT_KINDS = ("vanished", "status", "text") def load_state(path=STATE): """Read the baseline from `path`. The first version ignored its caller entirely and always read the module-level STATE, so `--state ` was honoured for WRITING and silently ignored for READING: every pass compared against the default baseline while the caller believed it was comparing against the one it named. Nothing failed loudly -- the flag was half-wired, which is worse than absent -- and the census selftest found it: three deliberately falsified baselines produced three empty diffs, because the falsified file was never opened. """ if not (path and os.path.exists(path)): return None raw = json.load(open(path, encoding="utf-8")) # A baseline is read as CURRENT by every later pass, so it must name its instant -- the same rule # the adjudication record now follows. A file that cannot be dated is not silently compared # against: the pass says so and behaves as a full pass, because the alternative is judging a diff # against an observation whose date nobody wrote down (the defect this function's first version # had in a different costume: it ignored `--state` on read altogether). try: Covered(path, raw) except Undated as exc: sys.stderr.write("baseline %s cannot be read as current:\n %s\n" " -> this pass runs FULL and judges no diff\n" % (path, exc)) return None return raw def escalation_scope(sampled, baseline_ids, baseline_proj, frontier, found, kinds=None): """Which ids a full confirmation actually has to measure -- scoped PER ALERT KIND. An id can carry an alert only if the BASELINE holds something that kind can move: existence (so it can VANISH) or a stored projection (so a field can MOVE). Re-measuring an id the baseline never held cannot produce an alert, and re-measuring an id this pass already measured pays twice for one answer. What must be measured is therefore (union over the kinds that RANG of what could carry that kind | the frontier | found) minus what this pass already measured Scoping by KIND and not only by capability is what keeps the bill honest: confirming a VANISHED alert does not require the field-bearing population, and the union over kinds is taken only over the kinds that actually rang, so an alert of one kind never pays for the others' capability. Where a kind's capability IS the whole projected population -- and for status/text it is, because a return can acquire its first patch -- the union collapses to it, and the passing of that collapse is printed rather than assumed. The frontier stays in because that is where new returns land: they cannot be alerts, but they are the population, and a confirmation that silently dropped them would understate `existing_now`. Ids that never existed, lie below the frontier and were not sampled are excluded -- an arrival there is seen at the sample's probability, which `scope` discloses rather than hides. """ kinds = set(kinds or ALERT_KINDS) per_kind = {k: capable_of(k, baseline_ids, baseline_proj) for k in sorted(kinds)} capable = set() for k in per_kind: capable |= per_kind[k] need = capable | set(frontier) | set(found) union_all = set() for k in ALERT_KINDS: union_all |= capable_of(k, baseline_ids, baseline_proj) return (sorted(need - set(sampled)), capable, (need & set(sampled)), {"kinds": sorted(kinds), "per_kind": {k: len(v) for k, v in per_kind.items()}, "union": len(capable), "union_all_kinds": len(union_all), "narrowed_by_kind": len(union_all) - len(capable)}) def fetch_ids(the_ids, token): """One measurement: query these ids and return the returns that answered.""" with ThreadPoolExecutor(max_workers=sweep.WORKERS) as pool: objs = list(pool.map(lambda i: sweep.get_return(i, token), the_ids)) return {i: o for i, o in zip(the_ids, objs) if o} def diff_against(old, found, fps, projs, queried): """Diff one measurement against the baseline. Pure: no gate, no network, no I/O. Factored out for the ESCALATION. An incremental pass alarms through the ids it chose to re-query, so a single alert may be one of several it never looked at; when one rings, the same diff is recomputed over the whole id range against the SAME baseline. A second copy of this logic would be a second thing to get wrong, and the two copies would drift. """ out = {"appeared": [], "vanished": [], "changed": [], "status_moved": [], "accepted_text_moved": [], "fields": {}, "old": {}, "new": {}, "alert_count": 0} if old is None: return out # a baseline has nothing to diff against old_ids = set(old["existing_ids"]) old_fps = {int(k): v for k, v in (old.get("fingerprints") or {}).items()} old_proj = {int(k): v for k, v in (old.get("projections") or {}).items()} # JSON object keys are STRINGS, so a baseline reloaded from disk has "1441" where this run has # 1441. Comparing them directly made `changed` STRUCTURALLY EMPTY -- a zero that could never be # anything else. Normalised above, on the way in, and the same for the projections. out["appeared"] = sorted(i for i in found if i not in old_ids) out["vanished"] = sorted(i for i in old_ids if i in queried and i not in found) for i in sorted(projs): if i not in old_proj: continue fields, status_moved, text_moved = classify(old_proj[i], projs[i]) if fields: out["fields"][i] = fields out["old"][i], out["new"][i] = old_proj[i], projs[i] if status_moved: out["status_moved"].append(i) if text_moved: out["accepted_text_moved"].append(i) hash_moved = set(i for i, f in fps.items() if i in old_fps and old_fps[i] != f) out["changed"] = sorted(hash_moved | set(out["fields"])) out["alert_count"] = (len(out["vanished"]) + len(out["status_moved"]) + len(out["accepted_text_moved"])) return out def publication_readings(pairs, token, probe_path): """Ask the STORE whether the earlier accepted text is still served, for each text move. Imported, not re-derived: `accepted-text-published.py` owns the reading and its controls, and a second copy of that logic here is a second thing to get wrong. A refusal to control is returned as such rather than dropped -- a publication reading with no positive control would be exactly the kind of zero this project keeps meeting. """ if not pairs: return {"rows": [], "note": "no accepted text moved, so there is nothing to publish-check"} spec = importlib.util.spec_from_file_location("pubprobe", probe_path) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) gate = mod.Probe("watch -> publication reading: is the EARLIER accepted text still served?") mod.controls(gate, token) rows = [mod.read_pair(rid, old, new, token) for rid, old, new in pairs] out = {"tool": "census.py -> " + getattr(mod, "__name__", "pubprobe"), "rows": rows, "controls": [c.as_json() for c in gate.controls()], "summary": {"pairs": len(rows), "old_text_still_served": sum(1 for r in rows if r["old_text_resolves"]), "old_text_gone": sum(1 for r in rows if not r["old_text_resolves"])}} try: gate.claim("%d of %d prior accepted texts still resolve by their own digest" % (out["summary"]["old_text_still_served"], len(rows)), out["summary"]["old_text_still_served"], len(rows)) except NoControl as exc: out["claim_refused"] = str(exc) return out def show_history(path): """The series, offline: one dated line per pass, and what each measurement raised.""" if not os.path.exists(path): print("no history at %s: the next pass writes its first line" % path) return 0 rows = [json.loads(l) for l in open(path, encoding="utf-8") if l.strip()] # The basis column is NOT truncated: an early line's basis was a sentence, a later one's is a # token, and cutting either to a fixed width turns the series into something it did not say. print("%-20s %-11s %-27s %7s %7s %7s %7s %7s" % ("at (UTC)", "mode", "basis", "exist", "queried", "vanish", "status", "text", )) for h in rows: al = h.get("alerts") or {} print("%-20s %-11s %-27s %7d %7d %7d %7d %7d" % (h.get("at"), h.get("mode"), str(h.get("basis")), h.get("existing_now") or 0, h.get("ids_queried") or 0, len(al.get("vanished") or []), len(al.get("status_moved") or []), len(al.get("accepted_text_moved") or []))) print("%d dated measurement(s); alerts are counted per pass, and a pass with none is a zero the " "gate only admits because the classifier is controlled" % len(rows)) return 0 def select_ids(state, ceiling, grow, hot, sample_rate): """Which ids this pass must query, and why -- so the saving is auditable, not asserted.""" if state is None: return list(range(1, ceiling + 1)), {"mode": "full", "reason": "no baseline"} known = set(state["existing_ids"]) patchy = set(state.get("patch_bearing_ids") or []) prev_ceiling = int(state["ceiling"]) # THE FRONTIER STARTS AT THE LAST RETURN THAT EXISTS, never at the recorded ceiling. The first # version used `prev_ceiling + 1 .. prev_ceiling + grow`, and on this corpus (last return 1465, # recorded ceiling 1700) that queried 1701..1850 -- an empty space -- while ids 1466..1700 were # never re-queried by ANY incremental pass, though that is exactly where new returns land. The # census selftest caught it: three deliberately falsified baselines produced three empty diffs. frontier = list(range(max(known) + 1, ceiling + 1)) if known else [] # The hot zone is the newest `hot` EXISTING returns, not the last `hot` ids below the ceiling. # The first version measured it from `prev_ceiling - hot`, which on this corpus (ids up to ~1460, # ceiling 1700) selected NOTHING -- the guarantee was announced in `why` and covered no ids at # all. The audit trail printed `hot_zone: 0`, which is how the defect was found. hotzone = sorted(known)[-hot:] if hot else [] step = max(1, int(round(1.0 / sample_rate))) sample = [i for i in sorted(known) if i % step == 0] absent_prev = [i for i in range(1, prev_ceiling + 1) if i not in known] absent_sample = [i for i in absent_prev if i % step == 0] keep = sorted(set(frontier) | patchy | set(hotzone) | set(sample) | set(absent_sample)) why = {"mode": "incremental", "frontier": len(frontier), "patch_bearing": len(patchy), "hot_zone": len(hotzone), "sample_existing_step": step, "sample_existing": len(sample), "sample_absent_step": step, "sample_absent": len(absent_sample), "not_queried_existing": len(known) - len(set(keep) & known), "note": ("a disappearance outside the hot zone, the patch-bearing set and the sample is " "seen with the sample's probability -- stated, not hidden")} return keep, why def main(argv): ap = argparse.ArgumentParser(description=__doc__) ap.add_argument("--full", action="store_true", help="ignore the baseline and sweep every id") ap.add_argument("--ceiling", type=int, default=sweep.CEILING) ap.add_argument("--grow", type=int, default=150, help="ids to add past the recorded ceiling") ap.add_argument("--hot", type=int, default=200, help="always re-query the top N ids") ap.add_argument("--sample-rate", type=float, default=0.1) ap.add_argument("--state", default=None, help="the baseline to read AND rewrite (default %s; omitted when --baseline " "is given, so another snapshot is never modified)" % os.path.basename(STATE)) ap.add_argument("--baseline", default=None, help="read the diff baseline from HERE and do not write it: points the same watch " "at any dated snapshot, this run's or another one's") ap.add_argument("--publish-check", action="store_true", help="for every ACCEPTED TEXT MOVED hit, ask the store whether the EARLIER text " "is still served, and attach the reading") ap.add_argument("--history", default=os.path.join(HERE, "census-history.jsonl"), help="append-only: one dated line per pass (the watch's series)") ap.add_argument("--no-alert-exit", action="store_true", help="report alerts but exit 0 (default is exit 5 when an alert fired)") ap.add_argument("--complete", action="store_true", help="measure every alert-capable id against the baseline even when no alert " "rang, so a corpus-wide 'nothing moved anywhere' is not a sampled claim " "(implied by --publish-check, which is itself a corpus-wide reading)") ap.add_argument("--no-escalate", dest="escalate", action="store_false", help="do NOT re-measure the whole id range when an alert rings (default: do)") ap.set_defaults(escalate=True) ap.add_argument("--show-history", action="store_true", help="print the series and exit; makes no request") ap.add_argument("--json", default=None, help="also write the verdict JSON here, so a later step can consume it") a = ap.parse_args(argv) if a.show_history: return show_history(a.history) token = open(sweep.TOKEN, encoding="utf-8").read().strip() # A READ-ONLY baseline is what makes this watch usable beyond the run that owns the state file: a # sibling can point it at any dated snapshot and get the typed diff and the alerts, without being # able to overwrite somebody else's baseline by accident. state_path = a.state or (None if a.baseline else STATE) state = None if a.full else load_state(a.baseline or state_path) if state and a.ceiling == sweep.CEILING: # Anchor the probe bound to the LAST HIT, not to the previous ceiling. Growing from the # ceiling drifts: each pass added `grow` ids past a frontier that was already empty, so the # id space widened without bound (1700 -> 1850 -> 2150) and the density denominator drifted # away from the last return that exists. The probe bound should follow the corpus: it is # `last existing id + grow`, which is where a new return can actually appear. last_hit = max(state["existing_ids"]) a.ceiling = max(int(state["ceiling"]), last_hit + a.grow) ids, why = select_ids(state, a.ceiling, a.grow, a.hot, a.sample_rate) # ---- controls, before any count is narrated ------------------------------------------------ gate = Probe("return census: population and churn since the baseline") # NB: the must-pass "the probe can see returns at all" is registered AFTER the fetch, on what the # pass actually found. The first version asked `get_return(max(existing_ids))` -- and on a # falsified baseline (the census selftest) that maximum IS the fabricated id, so the control # failed and the gate correctly refused a measurement it had no reason to distrust. A control # must not take its evidence from the artefact under test. far = a.ceiling + 500 gate.must_not_find("an id far past the ceiling is not found", sweep.get_return(far, token) is not None, detail="id %d" % far) # the change detector must be able to say "changed", or "0 changed" means nothing fx_a = fingerprint({"id": 1, "type": "explore", "status": "recorded", "patch": "x", "files": []}) fx_b = fingerprint({"id": 1, "type": "explore", "status": "recorded", "patch": "x", "files": [{"sha256": "0" * 64}]}) fx_c = fingerprint({"id": 1, "type": "explore", "status": "recorded", "patch": "x", "files": []}) gate.must_find("the fingerprint MOVES when a declared file is added", fx_a != fx_b) gate.must_not_find("the fingerprint is STABLE for identical projections", fx_a != fx_c) # One control per ALERT KIND. A watch whose alarm cannot ring is not a watch: "0 alerts" is a # zero, and it is only narratable if each kind of movement is demonstrably detectable. p_pending = projection({"id": 1, "status": "recorded", "patch_status": "pending", "patch": "x"}) p_accepted = projection({"id": 1, "status": "recorded", "patch_status": "accepted", "patch": "x"}) p_rewritten = projection({"id": 1, "status": "recorded", "patch_status": "pending", "patch": "y"}) p_same = projection({"id": 1, "status": "recorded", "patch_status": "pending", "patch": "x"}) gate.must_find("a patch_status change is named as a STATUS move", classify(p_pending, p_accepted)[1]) gate.must_find("a rewritten patch text is named as an ACCEPTED TEXT move", classify(p_pending, p_rewritten)[2]) gate.must_not_find("two measurements of an unchanged return produce no typed movement", any(classify(p_pending, p_same))) # The ESCALATION's scoping rule, controlled where it can be wrong: which ids a confirmation needs. # A rule that quietly dropped an alert-capable id would look like a smaller bill, so the exclusion # is pinned in both directions. _sc, _cap, _reused, _k = escalation_scope(sampled={1, 2}, baseline_ids=[1, 2, 3, 9], baseline_proj={3}, frontier=[10, 11], found={1}) gate.must_find("an alert-capable id this pass did not measure IS in the confirmation scope", 3 in _sc and 9 in _sc, _sc) gate.must_find("the FRONTIER is in the confirmation scope (new returns are the population)", 10 in _sc and 11 in _sc, _sc) gate.must_not_find("an id the baseline never held, below the frontier and unsampled, is NOT in " "it -- that exclusion is the saving", 7 in _sc or 8 in _sc, _sc) # ---- the scope is taken PER ALERT KIND, so the saving must be real and the narrowing SOUND ----- # (a) a VANISHED-only confirmation does not pay for the field-bearing population _sc_v, _cap_v, _, _kv = escalation_scope(sampled=set(), baseline_ids=[1, 2, 3, 9], baseline_proj={3, 40, 41}, frontier=[], found={}, kinds=["vanished"]) gate.must_find("a VANISHED-only confirmation scopes to EXISTENCE (ids 1,2,3,9), not to the " "projected ids", _cap_v == {1, 2, 3, 9} and _kv["narrowed_by_kind"] == 2, _kv) # (b) ...but a TEXT confirmation may NOT be narrowed that way: an id with a stored projectION and # no stored patch is exactly the return whose FIRST ascent would be missed. This is the # pre-registered falsifier for the tempting optimisation. _sc_t, _cap_t, _, _kt = escalation_scope(sampled=set(), baseline_ids=[1, 2, 3], baseline_proj={3, 40, 41}, frontier=[], found={}, kinds=["text"]) gate.must_find("a TEXT confirmation scopes to the PROJECTED ids even with no stored patch -- 40 " "and 41 could acquire their first one", 40 in _cap_t and 41 in _cap_t, _kt) gate.must_not_find("a text scope never includes an id with no projection at all", 1 in _cap_t or 2 in _cap_t, _cap_t) gate.must_find("scoping by KIND is never wider than the union over all kinds", _kt["union"] <= _kt["union_all_kinds"], _kt) # The live version of this control -- every id that ACTUALLY alerted is inside its own kind's # capability -- is registered in the escalation block, where a real diff exists to check. gate.must_find("capability is per KIND, not one shared set (a vanished-capable id with no " "projection is not text-capable)", capable_of("vanished", [7], set()) == {7} and capable_of("text", [7], set()) == set(), None) found = fetch_ids(ids, token) fps = {i: fingerprint(o) for i, o in found.items()} projs = {i: projection(o) for i, o in found.items()} gate.must_find("the probe found at least one return (it can see returns at all)", bool(found), detail="%d of %d queried ids returned a return" % (len(found), len(ids))) # ids belonging to the baseline that this pass will NOT re-query -- computed before the control # block, because one of the controls is exactly about their fate carry_ids = sorted((set(state["existing_ids"]) if state else set()) - set(ids)) # A first pass has no baseline to diff against, and reporting the whole population as "appeared" # would be the same category error this file exists to avoid: a diff needs two dated observations. # `diff_against` returns empty sets for a baseline, so the run narrates no diff instead of five # zeros it has nothing behind. scope = {"queried": len(ids), "probe_range": a.ceiling, "complete": False, "sample_step": why.get("sample_existing_step"), "escalated": False} d = diff_against(state, found, fps, projs, set(ids)) # ---- ESCALATION: an alarm on a sampled pass is not yet a finding ---------------------------- # An incremental pass judges its diff through the ids it chose to re-query, so one alert could be # one of several it never looked at -- the alarm would then be a lower bound wearing the clothes # of a result. When one rings, the alert is CONFIRMED by measuring every ALERT-CAPABLE id against # the SAME baseline (same code, wider scope), and it is the complete measurement that is written # to the state, the history and the claim. escalated = None kinds_rang = [k for k, ids_k in (("vanished", d["vanished"]), ("status", d["status_moved"]), ("text", d["accepted_text_moved"])) if ids_k] # FORCED COMPLETENESS. An alert escalates, but a CORPUS-WIDE reading cannot wait for one to ring: # "no accepted text moved anywhere" judged over a 10% sample is a claim at the sample's # probability, and the publication reading attached to each hit inherits that weakness. `--complete` # (and `--publish-check`, whose whole point is a corpus-wide store reading) forces the same # confirmation scope the escalation would use, with the reason stated instead of implied. forced = bool(a.complete or a.publish_check) if state is not None and (d["alert_count"] or forced) and a.escalate: sampled = set(ids) frontier = range(max(state["existing_ids"]) + 1, a.ceiling + 1) todo, capable, reused_ids, kind_scope = escalation_scope( sampled, state["existing_ids"], {int(k) for k in (state.get("projections") or {})}, frontier, set(found), kinds=(kinds_rang or ALERT_KINDS)) more = fetch_ids(todo, token) if todo else {} found = dict(found) found.update(more) fps = {i: fingerprint(o) for i, o in found.items()} projs = {i: projection(o) for i, o in found.items()} ids = sorted(sampled | set(todo)) carry_ids = sorted(set(state["existing_ids"]) - set(ids)) escalated = {"sampled_alerts": d["alert_count"], "sampled_ids_queried": len(sampled), "ids_reused": len(reused_ids), "ids_added": len(todo), "alert_capable": len(capable), "kind_scope": kind_scope, "forced_by": ("--complete/--publish-check: a corpus-wide reading is claimed" if forced and not kinds_rang else ("%d alert(s) rang: %s" % (d["alert_count"], ", ".join(kinds_rang)) if kinds_rang else None)), # the sampled pass also measures ids that can carry no alert at all (the absent-id # sample); they are neither reused nor added, and the count is stated so the # arithmetic of the confirmation adds up in the record "sampled_ids_without_alert_role": len(sampled) - len(reused_ids)} why = dict(why, mode="escalated-capable", note=(escalated["forced_by"] + ": every id that CAN carry " + ("the kinds that rang" if kinds_rang else "an alert") + " was measured against the SAME baseline before any claim was narrated -- " "reusing the ids this pass already measured rather than paying twice") if escalated["forced_by"] else "every alert-capable id was measured against the SAME baseline") d = diff_against(state, found, fps, projs, set(ids)) scope = {"queried": len(ids), "probe_range": a.ceiling, "complete": True, "complete_over": "every alert-capable id, plus the frontier", "alert_capable": len(capable), "ids_reused_from_this_pass": len(reused_ids), "ids_added": len(todo), "excluded_never_existed": max(0, a.ceiling - len(ids)), "sample_step": None, "escalated": True} gate.must_find("every id that can carry an alert was measured, reused or re-queried", capable <= set(ids), detail="%d alert-capable ids; %d reused from this pass, %d added" % (len(capable), len(reused_ids), len(todo))) # ...and the scope is per KIND, so the control that matters is on real alerts: an id that DID # alert must be inside its own kind's capability. If a capability set were too small, the # confirmation would silently stop looking at an id that can still raise what it is confirming # -- a narrower bill that reads like a saving. This is that check, on live data. _b_ids, _b_proj = set(state["existing_ids"]), set(int(k) for k in (state.get("projections") or {})) for _kind, _hit in (("vanished", d["vanished"]), ("status", d["status_moved"]), ("text", d["accepted_text_moved"])): gate.must_find("every id that DID raise a %s alert is inside that kind's capability" % _kind, all(i in capable_of(_kind, _b_ids, _b_proj) for i in _hit), detail={"kind": _kind, "ids": _hit[:8], "capability": len( capable_of(_kind, _b_ids, _b_proj))}) gate.must_find("the per-kind scope never exceeds the union over all kinds (a saving, not a " "widening)", kind_scope["union"] <= kind_scope["union_all_kinds"], kind_scope) gate.must_find("the confirmation is CHEAPER than the id space it could have walked", len(ids) < a.ceiling, detail="%d measured vs %d ids in 1..%d" % (len(ids), a.ceiling, a.ceiling)) appeared, vanished, changed = d["appeared"], d["vanished"], d["changed"] status_moved, text_moved = d["status_moved"], d["accepted_text_moved"] old_ids = set(state["existing_ids"]) if state else set() old_fps = {int(k): v for k, v in (state.get("fingerprints") or {}).items()} if state else {} old_proj = {int(k): v for k, v in (state.get("projections") or {}).items()} if state else {} # known ids we did NOT re-query are carried forward, and that carry-forward is recorded carry = carry_ids # The invariant that a carried-forward id is not a vanished one -- the regression found above # (hundreds of false disappearances), pinned so it cannot come back silently. It needs the diff, # so it is registered AFTER the fetch but BEFORE any claim is narrated, and ONLY when a baseline # exists: a first pass has no diff to judge, and inventing its controls would make the gate # refuse a correct baseline (observed: exit 3 on the very first --full run of this design). if state is not None: gate.must_find("the baseline fingerprints key on the same TYPE this run does (int)", all(isinstance(k, int) for k in old_fps), detail="%d keys" % len(old_fps)) gate.must_not_find("a carried-forward id is not counted as vanished", bool(set(carry_ids) & set(vanished)), detail="carried %d, flagged %d" % (len(carry_ids), len(vanished))) gate.must_find("the vanished set is judged over a non-empty queried subset", len(set(ids) & old_ids) > 0, detail="%d baseline ids re-queried" % len(set(ids) & old_ids)) existing_now = sorted(set(found) | set(carry)) patchy_now = sorted(i for i in existing_now if (found.get(i, {}).get("patch") if i in found else None) or (i in (state.get("patch_bearing_ids") or []) if state else False)) out = { "tool": "census.py/1.0", "at": sweep.time.strftime("%Y-%m-%dT%H:%M:%SZ", sweep.time.gmtime()), "mode": why["mode"], "why": why, "ids_queried": len(ids), "requests": sweep._counts["requests"], "existing_now": len(existing_now), "ceiling": a.ceiling, "last_hit": (max(existing_now) if existing_now else 0), # density over the PROVEN populated range, not over the artificial ceiling: the ceiling is a # probe bound and grew every pass (1700 -> 1850 -> 2150), which made the denominator drift # away from the last return that exists. "density": round(len(existing_now) / float(max(existing_now) if existing_now else 1), 4), "density_denominator": (max(existing_now) if existing_now else 0), "id_space_probed": a.ceiling, "carried_forward_without_requery": len(carry), "vanished_judged_over_ids_queried": len(set(ids) & old_ids), "patch_bearing_now": len(patchy_now), "first_pass": state is None, "diff": {"appeared": appeared, "vanished": vanished, "changed": changed}, # The watch: three kinds of movement, and the count that decides the exit code. APPEARED is # in the diff but not among them -- a new return is the corpus working. "alerts": {"vanished": vanished, "status_moved": status_moved, "accepted_text_moved": text_moved}, "alert_count": len(vanished) + len(status_moved) + len(text_moved), "alert_scope": scope, "escalated": escalated, "fields_moved": {str(k): v for k, v in d["fields"].items()}, # The OLD and NEW value of every field that moved, so a later step can ask what the movement # MEANS rather than only that it happened -- `accepted-text-published.py` reads # `moved_values[]["patch_sha"]` and probes whether the OLD digest is still served. "moved_values": {str(i): {f: [d["old"][i].get(f), d["new"][i].get(f)] for f in d["fields"][i]} for i in sorted(d["fields"])}, "history": a.history, "controls": [c.as_json() for c in gate.controls()], "previous": None if state is None else {"at": state.get("at"), "existing": len(state["existing_ids"]), "ceiling": state["ceiling"]}, } # The fingerprint map is ACCUMULATED, not replaced. Writing only this pass's fingerprints shrank # the map from 1288 entries to 422 after one incremental pass (only re-queried ids carry one), so # a return that enters and leaves the sample would lose its baseline and never be comparable # again -- `changed` would decay toward the structurally-empty zero it used to be. Entries are # kept for every id in the live population and pruned for the ones that are gone. fp_out, proj_out = {}, {} for i in existing_now: v = fps.get(i) or old_fps.get(i) if v: fp_out[i] = v w = projs.get(i) or old_proj.get(i) if w: proj_out[i] = w # Written through the dated-record helper, so the baseline's own instant is generated rather than # typed -- the pass before this one had a baseline that could not be dated at all. A pass reading # a FOREIGN baseline writes none: the diff is published, the other run's file is left alone. if state_path is None: # The `if` used to wrap only the bookkeeping and leave the write OUTSIDE it, so a read-only # baseline was announced as "not written" and then written-or-crashed anyway: the guard did # not guard. It is an else now, and the census selftest reads a foreign baseline and checks # that the file it points at is byte-identical afterwards. out["state_written"] = None else: write_dated(state_path, { "at": out["at"], "ceiling": a.ceiling, "existing_ids": existing_now, "patch_bearing_ids": patchy_now, "fingerprints": {str(k): v for k, v in fp_out.items()}, "projections": {str(k): v for k, v in proj_out.items()}, "carried_forward": [str(i) for i in carry], "mode": why["mode"], }) out["state_written"] = state_path out["fingerprints_kept"] = len(fp_out) out["fingerprints_added"] = len([i for i in fps if i not in old_fps]) out["projections_kept"] = len(proj_out) # ---- the publication reading, attached to each text move ------------------------------------ if a.publish_check: pairs = [] for i in text_moved: pair = (d["old"].get(i) or {}).get("patch_sha"), (d["new"].get(i) or {}).get("patch_sha") if pair[0]: pairs.append((i, pair[0], pair[1])) try: out["publication"] = publication_readings( pairs, token, os.path.join(HERE, "accepted-text-published.py")) except Exception as exc: # noqa: BLE001 out["publication"] = {"refused": "the reading could not be taken: %r" % (exc,)} # ---- the series: one DATED line per pass, appended. The baseline is rewritten every pass (it is # the previous measurement), so without this file a change is visible once and then unaskable; the # history is what makes the watch a series rather than a pair. hist = { "at": out["at"], "mode": why["mode"], "basis": "full-reseed" if a.full else ("baseline" if state is None else "diff"), "baseline_at": None if state is None else state.get("at"), "ids_queried": len(ids), "existing_now": len(existing_now), "last_hit": out["last_hit"], "appeared": appeared, "vanished": vanished, "changed": changed, "status_moved": status_moved, "accepted_text_moved": text_moved, "alerts": out["alerts"], "alert_count": out["alert_count"], "scope": scope, "escalated": escalated, "publication": out.get("publication"), "not_requeried": len(carry), "vanished_judged_over_ids_queried": len(set(ids) & old_ids) if state else None, } with open(a.history, "a", encoding="utf-8", newline="\n") as fh: fh.write(json.dumps(hist, sort_keys=True) + "\n") out["history_lines"] = sum(1 for _ in open(a.history, encoding="utf-8")) try: gate.claim("%d returns exist in ids 1..%d (density %.3f over the populated range; %d ids " "probed)" % (len(existing_now), out["last_hit"], out["density"], a.ceiling), len(existing_now), out["last_hit"]) if state is not None: # A diff needs TWO dated observations. On a first pass these three claims are not # narrated at all rather than narrated as zeros -- a zero with nothing to compare to is # precisely the narration this gate exists to stop. gate.claim("%d returns APPEARED since the baseline" % len(appeared), len(appeared), len(existing_now), extra={"ids": appeared[:25]}) gate.claim("%d returns VANISHED since the baseline" % len(vanished), len(vanished), len(existing_now), extra={"ids": vanished[:25]}) gate.claim("%d re-queried returns CHANGED content since the baseline" % len(changed), len(changed), len(fps), extra={"ids": changed[:25]}) gate.claim("%d ALERTS between %s and %s (scope: %s): %d vanished, %d with a moved " "status, %d with moved accepted text" % (out["alert_count"], state.get("at"), out["at"], ("complete over the alert-capable population" if scope["complete"] else "sampled"), len(vanished), len(status_moved), len(text_moved)), out["alert_count"], len(existing_now), extra={"vanished": vanished[:25], "status_moved": status_moved[:25], "accepted_text_moved": text_moved[:25], "scope": scope, "escalated": escalated}) except NoControl as exc: sys.stderr.write("REFUSED: no census verdict may be printed\n%s\n" % exc) return 3 scope_label = ("complete over every alert-capable id: %d measured (%d reused from this pass, " "%d added), %d never-existing ids left unqueried" % (scope["queried"], scope.get("ids_reused_from_this_pass", 0), scope.get("ids_added", 0), scope.get("excluded_never_existed", 0)) if scope["complete"] else "sampled: %d ids re-queried of a %d-id probe range" % (scope["queried"], scope["probe_range"])) if state is None: # A baseline has nothing to diff, so it makes NO alert claim at all: "0 alerts" here would be # a zero with nothing behind it, which is the narration this whole file refuses. print("ALERTS: none judged -- this pass is the baseline; the next pass is the first that can " "alert (a diff needs two dated measurements)") elif out["alert_count"]: print("ALERTS (%d) between %s and %s -- a return that moved is not a new return:" % (out["alert_count"], hist["baseline_at"], out["at"])) for i in vanished: print(" VANISHED return %d: present at the previous measurement, gone now" % i) for i in status_moved: print(" STATUS MOVED return %d: %s" % (i, json.dumps( {k: [old_proj.get(i, {}).get(k), projs[i].get(k)] for k in STATUS_FIELDS}, sort_keys=True))) for i in text_moved: print(" ACCEPTED TEXT MOVED return %d: %s" % (i, json.dumps( {k: [old_proj.get(i, {}).get(k), projs[i].get(k)] for k in TEXT_FIELDS}, sort_keys=True))) else: print("ALERTS (0) between %s and %s: no re-queried return vanished, moved status, or moved " "its accepted text" % (hist["baseline_at"], out["at"])) if escalated: ks = escalated["kind_scope"] print(" SCOPE: %s -- the first pass measured %d ids and the confirmation measured every id " "that CAN carry %s against the SAME baseline: %d alert-capable, %d reused from this " "pass, %d added -- %d requests, not the %d-id space it could have walked" % (escalated["forced_by"] or "an alert rang", escalated["sampled_ids_queried"], "the kinds that rang (%s)" % ", ".join(ks["kinds"]) if escalated["sampled_alerts"] else "an alert", escalated["alert_capable"], escalated["ids_reused"], escalated["ids_added"], scope["queried"], a.ceiling)) print(" KINDS: per-kind capability %s -- union %d, narrowed by kind %d, out of %d for all " "kinds (a text move can be a FIRST patch, so text/status scope to the whole projected " "population and only VANISHED narrows)" % (json.dumps(ks["per_kind"], sort_keys=True), ks["union"], ks["narrowed_by_kind"], ks["union_all_kinds"])) else: print(" SCOPE: %s%s" % (scope_label, "" if state is None else " (a change outside the queried set is seen with the sample's probability, and is " "the reason an alert escalates)")) print(json.dumps({k: v for k, v in out.items() if k not in ("controls",)}, indent=1, sort_keys=True)) print("state -> %s" % (state_path or "(not written: --baseline is read-only)")) print("history -> %s (%d line(s) now; this pass appended one)" % (a.history, out["history_lines"])) pub = out.get("publication") if pub is not None: if pub.get("refused") or pub.get("claim_refused"): print("PUBLICATION: %s" % (pub.get("refused") or pub["claim_refused"])) else: print("PUBLICATION: %s" % json.dumps(pub["summary"], sort_keys=True)) for r in pub.get("rows") or []: print(" return %-6d old %s %s -- %s" % (r["return"], (r["old_sha"] or "")[:12], "SERVED" if r["old_text_resolves"] else "GONE ", r["read"])) if a.json: write_dated(a.json, {k: v for k, v in out.items() if k != "controls"}) print("verdict -> %s (a later step may read `moved_values` from it)" % a.json) if out["alert_count"] and not a.no_alert_exit: return 5 return 0 if __name__ == "__main__": sys.exit(main(sys.argv[1:]))