#!/usr/bin/env python3 """Job #4528 -- route 100 step check (record comparison only; no K* is computed for a new shape, no sweep is run). It produces the two artefacts the return rests on: corpus.jsonl the searchable text (report_md + recipe_md + finding + human_md + the research object) of every served return fetched for the check, verbatim, one JSON object per line; scan.json the comparison: which returns were recorded after the step-setter #1833, which of them carry the step's vocabulary, whether any of them reports the step's success shape at all, the served #1833 table and sweep it rests on, and the exact price of the step's own `Mp <= 1e10` clause against the population and a measured rate. The instruments used here are the promoted ones: private/lib/served/step_scan.py (the routeless-aware comparison set) private/lib/covering/base_extension.py (population, price, phase bound D) served_evidence.py fetched and hashed the served records and attached files. Nothing here claims a bound on G2, beta_2 or twin-prime infinitude. """ from __future__ import annotations import hashlib import itertools import json import os import re import sys HERE = os.path.dirname(os.path.abspath(__file__)) ROOT = os.path.abspath(os.path.join(HERE, "..", "..")) sys.path.insert(0, os.path.join(ROOT, ".solveathome", "private", "lib")) sys.path.insert(0, os.path.join(ROOT, ".solveathome", "private", "lib", "covering")) sys.path.insert(0, os.path.join(ROOT, ".solveathome", "private", "lib", "served")) from served import step_scan # noqa: E402 import base_extension as be # noqa: E402 SERVED = os.path.join(HERE, "served") BRIEF = os.path.join(ROOT, ".solveathome", "private", "runs", "launch-sv37hvl1lkqlgqdny8jx821m", "receipts", "next_assignment.json") RUN = ".solveathome/private/runs/launch-sv37hvl1lkqlgqdny8jx821m" # The step's own vocabulary, fixed here before the comparison is read. LOCATORS = ["1267", "1804", "1833", "1812", "1246", "kfork", "sweep2612", "census2612", "dbound", "prereg-2612", "xcheck2612", "D(P,p,A)", "D(P,p,L)", "D(30,", "floor(2A/p)", "floor(2L/p)", "phase bound", "phase gate", "merge gain", "merge-gain", "base extension", "base-extension", "A-run", "A_P", "K*(Pp", "boundary-transfer", "boundary transfer", "1264"] OBJECT_WORDS = ["drop", "attain", "fork", "merge", "covering run", "maximal drop", "K*("] # The step's success shape, as literal patterns (pre-registered before the read). # "kfork" alone is the *instrument* the cross-check names, not a success claim, so # it is recorded under instrument_mentions and not as a success shape. SUCCESS_PATTERNS = { "row_drop_eq_D": r"drop\s*=\s*D(?![\w])", "merge_gain_invariant": r"merge[\s-]gain", "drop_at_least_3": r"drop\s*(?:>=|>|\u2265)\s*=?\s*3\b", } INSTRUMENT_WORDS = ["kfork", "kstar", "sweep2612", "dbound2612"] # A verbatim quotation of the step (raw JSON or JSON-escaped quotes); a phrase # inside one of these regions is the step's own wording, not a report of it. STEP_BLOCK = re.compile(r'\{\s*"method"\s*:\s*"Reuse sweep2612.*?' r'"required_sources"\s*:\s*\[[^\]]*\][^}]*\}', re.S) # The served return whose closing question is the step; the step-setter. STEP_SETTER = 1833 STEP_CHECK_2018 = 2018 # Measured instrument timings: the served kfork.c of #1804, compiled with the # workspace toolchain, run on rows already recorded in sweep2612.jsonl (their A, B # are on the record; nothing new is computed). (P, p, R, Mp, B_pass_seconds). TIMINGS = [ [6, 5, [11, 13, 17, 23, 29, 41], 1994416710, 7.48], [2, 7, [5, 19, 23, 31, 37], 35086730, 0.16], [6, 5, [11, 19, 23, 31], 4470510, 0.01], ] # The rate the step's own setter implies is not recoverable (sweep2612.jsonl is # appended across several chunked invocations), so the price is quoted at the # measured single-thread rate and at a 4-thread reading of the same instrument. RATE_SINGLE = 2.4e8 # 2.19e8 .. 2.67e8 from TIMINGS; 2.4e8 is the midpoint RATE_OPTIMISTIC = 1.0e9 # 4 threads at the measured per-thread rate STEP_BUDGET_HOURS = 2.0 def sha256_file(p): with open(p, "rb") as fh: return hashlib.sha256(fh.read()).hexdigest() def step_from_brief(): body = json.load(open(BRIEF, encoding="utf-8"))["body"]["brief_md"] for line in body.splitlines(): s = line.strip() if s.startswith('{"method"') and s.endswith("}"): return json.loads(s) raise SystemExit("step JSON not found in the brief") def build_corpus(records): lines = [] for rid in sorted(records, key=int): rec = records[rid] text = step_scan.report_text(rec) lines.append(json.dumps({ "id": int(rid), "route": rec.get("research_route_id"), "created_at": rec.get("created_at"), "type": rec.get("type"), "status": rec.get("status"), "n_chars": len(text), "text": text, }, separators=(",", ":"))) with open(os.path.join(HERE, "corpus.jsonl"), "w", encoding="utf-8", newline="\n") as fh: fh.write("\n".join(lines) + "\n") def phase_D(P, p, L): """A second exact D(P,p,L), written here from #1833's definition (the driver's copy; the checker embeds its own).""" cls = be.alive_classes(P) m = len(cls) best = None for phi in range(m): o = [((phi + i) // m) * P + cls[(phi + i) % m] - cls[phi] for i in range(L)] cnt = [0] * p for x in o: cnt[x % p] += 1 k = min(cnt[(-s) % p] + cnt[(-s - 2) % p] for s in range(p)) best = k if best is None else max(best, k) return best def main(): recs = {} for name in os.listdir(SERVED): if name.startswith("return-") and name.endswith(".json"): recs[name[7:-5]] = json.load(open(os.path.join(SERVED, name), encoding="utf-8")) step = step_from_brief() build_corpus(recs) rows = step_scan.scan_rows(recs.values()) t_setter = [r for r in rows if str(r["id"]) == str(STEP_SETTER)][0]["created_at"] after = step_scan.later_than(rows, t_setter) # the step's vocabulary, per return strong = {} for r in sorted(after, key=lambda x: int(x["id"])): loc = step_scan.keyword_hits(r["text"], LOCATORS) obj = step_scan.keyword_hits(r["text"], OBJECT_WORDS) if loc and obj: strong[str(r["id"])] = {"route": r["route"], "created_at": r["created_at"], "locators": loc, "object_words": obj} # the step's success shape, per return, with the quotation rule shape_hits, instr_hits, occurrences = {}, {}, [] for r in sorted(after, key=lambda x: int(x["id"])): norm = r["text"].replace('\\"', '"') blocks = [(m.start(), m.end()) for m in STEP_BLOCK.finditer(norm)] hits = [k for k, pat in SUCCESS_PATTERNS.items() if re.search(pat, norm, re.I)] inst = step_scan.keyword_hits(norm, INSTRUMENT_WORDS) if inst: instr_hits[str(r["id"])] = inst if not hits: continue shape_hits[str(r["id"])] = hits for k in hits: for m in re.finditer(SUCCESS_PATTERNS[k], norm, re.I): occurrences.append({ "return": int(r["id"]), "pattern": k, "offset": m.start(), "in_step_block": any(s <= m.start() < e for s, e in blocks), "sentence": " ".join(norm[max(0, norm.rfind("\n", 0, m.start()) + 1): norm.find("\n", m.end()) if norm.find("\n", m.end()) > 0 else m.end() + 120].split())[:240], }) # A return that itself declares the step still open (outcome `promising` with the # step copied as its next_step) is quoting the shape, not reporting it. still_open = {} for rid, rec in recs.items(): res = rec.get("research") or {} still_open[str(rid)] = (res.get("outcome") == "promising" and json.dumps(res.get("next_step"), sort_keys=True) == json.dumps(step, sort_keys=True)) reports = [o for o in occurrences if not o["in_step_block"] and not still_open.get(str(o["return"]))] for o in occurrences: o["declared_step_open_by_its_own_return"] = bool(still_open.get(str(o["return"]))) # served #1833 table db = json.load(open(os.path.join(SERVED, "f1833_dbound2612.json"), encoding="utf-8")) strict = db["strict"] served_d = {(str(r[0]), r[1], r[2]): r[4] for r in strict} thr = {str(p): be.min_L_for_drop(30, p, 3) for p in (7, 11, 13, 17, 19)} d30 = {} for p in (7, 11, 13): for L in (4, 7, 8, 11, 12, 14, 17, 18, 19, 20, 21): d, _dmin = be.phase_bound_D(30, p, L) k = ("30", p, L) d30[k] = {"D": d, "floor": (2 * L) // p, "served": served_d.get(k)} # the served `strict` list carries only the shapes where D < floor, so a shape # absent from it must have D == floor to agree agree = all((row["served"] if row["served"] is not None else row["floor"]) == row["D"] for row in d30.values()) served_d = {(str(r[0]), r[1], r[2]): r[4] for r in strict} # served #1833 sweep sw = json.load(open(os.path.join(SERVED, "f1833_sweep2612.json"), encoding="utf-8")) srows = [json.loads(l) for l in open(os.path.join(SERVED, "f1833_sweep2612.jsonl"), encoding="utf-8") if l.strip()] def Mp(P, p, R): m = P for q in R: m *= q return m * p per_P = {} maxdrop = {} for P, p, R, a, b, d in srows: s = per_P.setdefault(str(P), {"n": 0, "sum_Mp": 0, "max_Mp": 0, "nD_ge3": 0, "n_drop_ge3": 0, "n_drop_gt_D": 0, "n_drop_eq_D": 0}) s["n"] += 1 s["sum_Mp"] += Mp(P, p, R) s["max_Mp"] = max(s["max_Mp"], Mp(P, p, R)) s["nD_ge3"] += (d >= 3) s["n_drop_ge3"] += (a - b >= 3) s["n_drop_gt_D"] += (a - b > d) s["n_drop_eq_D"] += (a - b == d) maxdrop[str((P, p))] = max(maxdrop.get(str((P, p)), -10 ** 9), a - b) attained = sorted(([P, p, a, d] for P, p, R, a, b, d in srows if a - b == d), key=lambda x: (x[0], x[1], x[2], x[3])) attained_counts = {} for P, p, a, d in attained: key = "%d/%d/%d/%d" % (P, p, a, d) attained_counts[key] = attained_counts.get(key, 0) + 1 # the step's own population, re-enumerated from the setter's prereg pop = {} for P, pool, mmax in ((2, [q for q in be.primes_upto(43) if q >= 3], 2e8), (6, [q for q in be.primes_upto(47) if q >= 5], 5e8)): sets = [R for n in range(2, len(pool) + 1) for R in itertools.combinations(pool, n) if P * be.period_M(1, R) <= mmax] rows_ub, sum_ub, band_rows, band_sum = 0, 0, 0, 0 for R in sets: M = P * be.period_M(1, R) for p in be.primes_upto(61): if P % p == 0 or p in R: continue mp = M * p if mp <= 1e10: rows_ub += 1 sum_ub += mp if mp > 2e9: band_rows += 1 band_sum += mp pop[str(P)] = {"sets": len(sets), "rows_ub_1e10": rows_ub, "sum_Mp_ub_1e10": sum_ub, "band_rows": band_rows, "band_sum_Mp": band_sum} band_slots = sum(v["band_sum_Mp"] for v in pop.values()) # how far up the ascending-Mp band order a budget reaches band_mp = [] for P, pool, mmax in ((2, [q for q in be.primes_upto(43) if q >= 3], 2e8), (6, [q for q in be.primes_upto(47) if q >= 5], 5e8)): for n in range(2, len(pool) + 1): for R in itertools.combinations(pool, n): M = P * be.period_M(1, R) if M > mmax: continue for p in be.primes_upto(61): if P % p == 0 or p in R: continue mp = M * p if 2e9 < mp <= 1e10: band_mp.append(mp) band_mp.sort() reach = {} for name, rate in (("single_thread_2.4e8", RATE_SINGLE), ("four_thread_1.0e9", RATE_OPTIMISTIC)): budget = STEP_BUDGET_HOURS * 3600 * rate acc, k = 0, 0 for mp in band_mp: if acc + mp > budget: break acc += mp k += 1 reach[name] = {"rows_reachable": k, "rows_band": len(band_mp), "slots": acc, "budget_slots": budget, "fraction_of_band": round(k / len(band_mp), 4)} d3frac = sum(v["nD_ge3"] for v in per_P.values()) / float(sum(v["n"] for v in per_P.values())) recs_after_2018 = sorted(int(r["id"]) for r in step_scan.later_than(rows, [x for x in rows if str(x["id"]) == str(STEP_CHECK_2018)][0]["created_at"])) out = { "job": 4528, "route_id": 100, "attempt_id": "103635d032cfad0a353ad59cdfdf509a", "step_setter": {"id": STEP_SETTER, "created_at": t_setter}, "previous_step_check": {"id": STEP_CHECK_2018}, "step": step, "vocabulary": {"locators": LOCATORS, "object_words": OBJECT_WORDS}, "success_patterns": SUCCESS_PATTERNS, "corpus": {"file": "corpus.jsonl", "n_records": len(recs), "sha256": sha256_file(os.path.join(HERE, "corpus.jsonl")), "n_after_step_setter": len(after), "n_after_previous_step_check": len(recs_after_2018), "ids_after_previous_step_check": recs_after_2018}, "comparison": {"strong_hits": strong, "success_shape_candidates": shape_hits, "success_shape_occurrences": occurrences, "instrument_mentions": instr_hits, "reports_success_shape": reports}, "served_1833_dbound": { "rows": db["rows"], "n_strict": db["n_strict"], "min_L_for_D_ge3_at_P30": {k: v for k, v in thr.items()}, "D_check_at_P30": {"%s/%d/%d" % k: v for k, v in d30.items()}, "served_values_agree": agree, }, "served_1833_sweep": {"n_rows": len(srows), "per_P": per_P, "max_drop_per_Pp": maxdrop, "attained_drop_eq_D_counts": attained_counts, "max_attained_D": max(d for *_x, d in attained), "drop_eq_D_ge3_rows": [r for r in attained if r[3] >= 3], "summary_file": {k: {"sets": v["sets"], "rows_done": v["rows_done"], "Mp_reached": v["Mp_reached"], "drop_eq_D_ge3": v["drop_eq_D_ge3"], "guard_drop_gt_D": v["guard_drop_gt_D"]} for k, v in sw["bases"].items()}, "D_ge3_fraction": round(d3frac, 4)}, "price": {"population_ub": pop, "band_slots_ub": band_slots, "rate_slots_per_sec": {"measured_single_thread": RATE_SINGLE, "four_thread_reading": RATE_OPTIMISTIC}, "timings": TIMINGS, "hours_upper_bound_population": { "at_single_thread": round(band_slots / RATE_SINGLE / 3600, 2), "at_four_thread": round(band_slots / RATE_OPTIMISTIC / 3600, 2)}, "hours_at_D_ge3_fraction": { "at_single_thread": round(band_slots * d3frac / RATE_SINGLE / 3600, 2), "at_four_thread": round(band_slots * d3frac / RATE_OPTIMISTIC / 3600, 2)}, "step_budget_hours": STEP_BUDGET_HOURS, "reach_within_step_budget": reach}, "instrument_check": { "kfork_rows": [{"P": P, "p": p, "R": R, "Mp": mp} for P, p, R, mp, _s in TIMINGS], "recorded_A_B": [[P, p, R, a, b] for P, p, R, a, b, d in srows if [P, p, R] in [[t[0], t[1], t[2]] for t in TIMINGS]], }, } # the three timed rows must be on the record with the values the binary printed timed = {(P, p, tuple(R)) for P, p, R, _mp, _s in TIMINGS} out["instrument_check"]["timed_rows_on_record"] = [ [P, p, list(R), a, b] for P, p, R, a, b, d in srows if (P, p, tuple(R)) in timed] with open(os.path.join(HERE, "scan.json"), "w", encoding="utf-8", newline="\n") as fh: json.dump(out, fh, indent=1) print("corpus records", len(recs), "after setter", len(after), "strong hits", len(strong), "success-shape candidates", sorted(shape_hits), "reports", len(reports)) print("band slots (upper bound)", band_slots, "hours at 1e9/s", out["price"]["hours_upper_bound_population"]["at_four_thread"]) print("scan.json", sha256_file(os.path.join(HERE, "scan.json"))) if __name__ == "__main__": main()