"""Job #1936 (route 35 rev 10, pursue): spread-gated density-partial block-grain test at finer scales, plus a gap-local occupancy test, at x = 19 and x = 23 (and, optionally, x = 29 by a chunked tile sieve). Derived from return #1029's job1934-blockgrain.py (design of job #1450; repairs R1, R2 of route 35 rev 9); the changes, all fixed here BEFORE the first run: S1 SCALES = {8, 16, 32, 64, 128} (the registered coarse scales are kept so that the old cells reproduce #1029). S2 Spread gate BEFORE reading: a cell whose grain statistic has < 5 distinct values is declared VACUOUS and its rho / p are recorded under "vacuous" (not as a test); the verdict counts only non-vacuous cells. S3 Grain statistics: gmax (largest admissible gap in the block), nlong (count of admissible gaps > 2x) as before, and topdec = the block's count of admissible gaps >= the tile's 90th-percentile gap value (threshold fixed from the whole period's gap distribution, so it is a census statistic and period-invariant). S4 Gap-local test (the "gap-local scale" the route names): for every admissible slot r the context c(r) = max of the two adjacent admissible gaps (a census statistic). Slots are binned into c-deciles (bin edges from the tile); the response is the occupancy rate of the bin, twins in the bin's slots over the exposure divided by the bin's slots x periods. Statistic: Spearman rho of occupancy rate against decile rank (10 points); null: 400 draws permuting the per-slot twin indicator within each period (exchangeable under "arrangement carries no occupancy information beyond the slot count"); rank-p. No density control is needed: every bin is a set of slots and the control lambda*slots is identical across bins up to the bin size, which the rate divides out. R1, R2 unchanged: per-period lambda_k, blocks inside one period, tbar_b = mean_k t_kb; beta refit inside every draw. Pre-registered falsifiers (fixed here before the first run): F-block: in a level, for a grain statistic, |z| >= 2 AND rank-p <= 0.01 with the same sign at >= 2 non-vacuous scales among {M/64, M/128} (the two new scales), evaluated separately from the registered coarse scales. F-local: |z| >= 2 AND rank-p <= 0.01 at BOTH levels with the same sign of rho(occupancy rate, context group). Gates: G0 pi_2(10^6) = 8169; G1 slots per period = D(T_x); G1c largest marking prime < M; G2 block sums reproduce the period slot count and the sieved twin total; G3 spread (>= 5 distinct values) per cell, reported, not required. Stdout is the gate ledger and the verdict (deterministic); progress goes to stderr. RUN: python job1936-blockgrain.py --levels 19,23 --out job1936-blockgrain.json """ import argparse, json, math, sys, time import numpy as np SCALES = [8, 16, 32, 64, 128] NEW_SCALES = ["L=64", "L=128"] PERIODS = {19: 8, 23: 2, 29: 2} DRAWS, SEED = 400, 1936 TWIN_BELOW_1E6 = 8169 D_T = {19: 378675, 23: 7952175, 29: 214708725} CHUNK = 1 << 24 def primes_to(n): b = np.ones(n + 1, dtype=bool); b[:2] = False for q in range(2, int(n ** 0.5) + 1): if b[q]: b[q * q:: q] = False return np.flatnonzero(b) def tile_word(x): """Admissible twin-slot offsets r in [0, M): r, r+2 coprime to x# (M = x#). Direct lift; fine up to x = 23.""" res = np.array([1], dtype=np.int64); P = 2 for q in [int(q) for q in primes_to(x) if q > 2]: v = (res[:, None] + np.arange(q, dtype=np.int64) * P).ravel() v = v[(v % q != 0) & (v % q != (q - 2) % q)] res, P = v, P * q return P, np.sort(res) def twins_in(small_primes, lo, hi, log=None): """Positions n in [lo, hi) with n and n+2 both prime, by a chunked segmented sieve (as in #1029).""" out = []; c0 = lo; t0 = time.time(); nchunk = 0 while c0 < hi: c1 = min(c0 + CHUNK, hi) arr = np.ones(c1 - c0 + 2, dtype=bool) arr[(-c0) % 2:: 2] = False for q in small_primes[1:]: q = int(q) start = max(q * q, ((c0 + q - 1) // q) * q) if start < c1 + 2: arr[start - c0:: q] = False if c0 <= 2: arr[:2 - c0] = False pos = np.flatnonzero(arr[:-2] & arr[2:]) out.append(pos.astype(np.int64) + c0) c0 = c1; nchunk += 1 if log and nchunk % 50 == 0: print(f"sieve {c0}/{hi} {time.time() - t0:.1f}s", file=log, flush=True) return np.concatenate(out) if out else np.zeros(0, dtype=np.int64) def rank_avg(v): v = np.asarray(v, dtype=float); n = len(v) order = np.argsort(v, kind="stable") r = np.empty(n); sv = v[order]; i = 0 while i < n: j = i while j + 1 < n and sv[j + 1] == sv[i]: j += 1 r[order[i:j + 1]] = (i + j) / 2.0 + 1.0 i = j + 1 return r def pearson(a, b): a = np.asarray(a, float); b = np.asarray(b, float) a = a - a.mean(); b = b - b.mean() da, db = math.sqrt(float(a @ a)), math.sqrt(float(b @ b)) return 0.0 if da == 0 or db == 0 else float(a @ b) / (da * db) def spearman(a, b): return pearson(rank_avg(a), rank_avg(b)) class LCG: def __init__(self, s): self.s = s def next(self): self.s = (self.s * 6364136223846793005 + 1442695040888963407) % (2 ** 64) return self.s >> 33 def shuffle(self, n): idx = list(range(n)) for i in range(n - 1, 0, -1): j = self.next() % (i + 1) idx[i], idx[j] = idx[j], idx[i] return idx def perm_partial(tbar, a_b, grain, rng, draws=DRAWS): """Signed rho of the refit residual against grain, and its two-sided permutation rank-p (beta refit per draw).""" tbar = np.asarray(tbar, float); a_b = np.asarray(a_b, float); grain = np.asarray(grain, float) def resid_of(v): am = a_b - a_b.mean(); den = float(am @ am) beta = 0.0 if den == 0 else float(am @ (v - v.mean())) / den return v - beta * a_b obs = spearman(resid_of(tbar), grain) ge, n = 0, len(tbar) for _ in range(draws): idx = rng.shuffle(n) if abs(spearman(resid_of(tbar[idx]), grain)) >= abs(obs) - 1e-15: ge += 1 return obs, (ge + 1) / (draws + 1) def fisher_z(rho, n): if n <= 3 or abs(rho) >= 1.0: return 0.0 return rho * math.sqrt(max(n - 3, 1)) def gap_local(word, M, tpts, periods, rng, draws=DRAWS, ngroups=10): """S4 (binning repaired after run 1, see the report): occupancy rate by the slot's adjacent-gap context c(r) = max(gap before, gap after) (cyclic). Run 1 cut c into quantile deciles, which for a statistic with 23 (x = 19) or 33 (x = 23) distinct values leaves empty bins and undefined rates; here the distinct context values, in increasing order, are merged greedily into at most `ngroups` contiguous groups of roughly equal slot mass (no group empty), the response is each group's occupancy rate, the statistic is Spearman rho of rate against group order, and the null is the exact permutation null in closed form: under 'twins are a uniformly random subset of the period's slots of the observed size', the per-group twin counts are multivariate hypergeometric, sampled `draws` times per period.""" gaps = np.diff(np.concatenate([word, [word[0] + M]])) before = np.roll(gaps, 1) c = np.maximum(gaps, before) vals, inv, cnt = np.unique(c, return_inverse=True, return_counts=True) target = len(word) / ngroups; groups = np.zeros(len(vals), dtype=int); g = 0; acc = 0 for i in range(len(vals)): if acc >= target and g < ngroups - 1: g += 1; acc = 0 groups[i] = g; acc += cnt[i] grp = groups[inv]; ng = g + 1 slots_g = np.bincount(grp, minlength=ng).astype(float) tw_g = np.zeros(ng); tw_k = [] for k in range(1, periods + 1): sel = (tpts >= k * M) & (tpts < (k + 1) * M) off = tpts[sel] - k * M pos = np.searchsorted(word, off) ok = (pos < len(word)) & (word[np.minimum(pos, len(word) - 1)] == off) assert ok.all(), "a twin fell outside the tile word" tk = np.bincount(grp[pos[ok]], minlength=ng).astype(float); tw_g += tk; tw_k.append(int(ok.sum())) rate = tw_g / (slots_g * periods) order = np.arange(ng) obs = spearman(rate, order) nrng = np.random.default_rng(rng.next()) ge = 0 for _ in range(draws): tot = np.zeros(ng) for k in range(periods): tot += nrng.multivariate_hypergeometric(slots_g.astype(np.int64), tw_k[k]) if abs(spearman(tot / (slots_g * periods), order)) >= abs(obs) - 1e-15: ge += 1 p = (ge + 1) / (draws + 1) return {"groups": ng, "group_context_ranges": [[int(vals[groups == i].min()), int(vals[groups == i].max())] for i in range(ng)], "slots_per_group": slots_g.tolist(), "twins_per_group": tw_g.tolist(), "occupancy_rate_per_group": rate.tolist(), "rho_rate_vs_group": obs, "perm_p": p, "z_fisher": fisher_z(obs, ng), "draws": draws, "null": "multivariate hypergeometric per period (exact label-permutation null)", "distinct_context_values": int(len(vals)), "gap_value_histogram": {str(int(v)): int(n) for v, n in zip(*np.unique(gaps, return_counts=True))}} def main(): ap = argparse.ArgumentParser() ap.add_argument("--out", required=True) ap.add_argument("--levels", default="19,23") ap.add_argument("--draws", type=int, default=DRAWS) a = ap.parse_args() log = sys.stderr levels = [int(v) for v in a.levels.split(",")] out = {"instrument": "job1936-blockgrain.py", "job": 1936, "route": 35, "derived_from": "return #1029 job1934-blockgrain.py", "preregistered": {"scales": SCALES, "new_scales": NEW_SCALES, "periods": {str(k): PERIODS[k] for k in levels}, "draws": a.draws, "seed": SEED, "spread_gate": ">= 5 distinct values, applied before reading; vacuous cells are not tests", "grain_statistics": ["gmax", "nlong (gaps > 2x)", "topdec (gaps >= tile 90th percentile)"], "falsifier_block": "|z|>=2 AND rank-p<=0.01, same sign, >=2 non-vacuous scales among L=64,128, per statistic per level", "falsifier_local": "|z|>=2 AND rank-p<=0.01 at both levels, same sign of rho(occupancy rate, context decile)", "lambda": "per-period control (R1)", "permutation": "labels permuted against fixed a_b, beta refit per draw (R2)"}, "gates": {}, "levels": {}, "checks": [], "ledger": []} def check(tag, ok, detail): out["checks"].append({"check": tag, "pass": bool(ok), "detail": detail}) out["ledger"].append(f"{'PASS' if ok else 'FAIL'} {tag} :: {detail}") sp0 = primes_to(int(math.isqrt(10 ** 6)) + 1) twins0 = twins_in(sp0, 3, 10 ** 6 - 1) check("G0_pi2_1e6", len(twins0) == TWIN_BELOW_1E6, f"observed {len(twins0)} published {TWIN_BELOW_1E6}") out["gates"]["G0"] = {"observed": int(len(twins0)), "published": TWIN_BELOW_1E6} if len(twins0) != TWIN_BELOW_1E6: json.dump(out, open(a.out, "w"), indent=1); print("\n".join(out["ledger"])); return 2 for x in levels: t0 = time.time() M, word = tile_word(x) P_periods = PERIODS[x] slots = int(len(word)); hi = (P_periods + 1) * M check(f"G1_slots_x{x}", slots == D_T[x], f"offset list {slots} vs D(T_{x}) {D_T[x]}") check(f"G1b_period_x{x}", int(word[-1]) == M - 1 and int(word[0]) == 29, f"word span [{int(word[0])},{int(word[-1])}] mod M={M}") if slots != D_T[x]: json.dump(out, open(a.out, "w"), indent=1); print("\n".join(out["ledger"])); return 3 sp = primes_to(int(math.isqrt(hi)) + 1) check(f"G1c_sieve_primes_x{x}", int(sp[-1]) < M, f"largest marking prime {int(sp[-1])} < first period start M={M}") tpts = twins_in(sp, M, hi, log=log) print(f"x={x}: word {slots}, twins {len(tpts)} in [{M},{hi}) {time.time() - t0:.1f}s", file=log, flush=True) per_k = {} for k in range(1, P_periods + 1): sel = (tpts >= k * M) & (tpts < (k + 1) * M) per_k[k] = int(sel.sum()) tot_tw, tot_sl = int(sum(per_k.values())), P_periods * slots lam_bar = tot_tw / tot_sl lam_k = {k: per_k[k] / slots for k in per_k} allgaps = np.diff(np.concatenate([word, [word[0] + M]])) q90 = float(np.quantile(allgaps, 0.9)) lv = {"M": int(M), "periods": P_periods, "slots_per_period": slots, "twins": tot_tw, "twins_per_k": {str(k): per_k[k] for k in per_k}, "lambda_hat": lam_bar, "lambda_k_min": min(lam_k.values()), "lambda_k_max": max(lam_k.values()), "slope_frac": (per_k[1] - per_k[P_periods]) / per_k[1] if P_periods > 1 else 0.0, "tile_gap_max": int(allgaps.max()), "tile_gap_q90": q90, "tile_gap_distinct": int(len(np.unique(allgaps))), "scales": {}} for div in SCALES: L = M // div; nb = div b_of = np.minimum(word // L, nb - 1) a_b = np.bincount(b_of, minlength=nb).astype(float) t_kb = np.zeros((P_periods, nb)) for k in range(1, P_periods + 1): lo = k * M sel = (tpts >= lo) & (tpts < lo + M) t_kb[k - 1] = np.bincount(np.minimum((tpts[sel] - lo) // L, nb - 1).astype(int), minlength=nb) g_b = np.zeros(nb); nlong_b = np.zeros(nb); topdec_b = np.zeros(nb) for bidx in range(nb): rs = word[b_of == bidx] if len(rs) >= 2: gaps = np.diff(rs) g_b[bidx] = float(gaps.max()); nlong_b[bidx] = float((gaps > 2 * x).sum()); topdec_b[bidx] = float((gaps >= q90).sum()) tbar = t_kb.mean(axis=0); both = t_kb.astype(int) check(f"G2_sums_x{x}_L{L}", int(a_b.sum()) == slots and int(both.sum()) == tot_tw, f"a_b sum {int(a_b.sum())} vs slots {slots}; t_b sum {int(both.sum())} vs twins {tot_tw}") if int(a_b.sum()) != slots or int(both.sum()) != tot_tw: json.dump(out, open(a.out, "w"), indent=1); print("\n".join(out["ledger"])); return 4 cell = {} for name, grain in (("gmax", g_b), ("nlong", nlong_b), ("topdec", topdec_b)): nd = int(len(np.unique(grain))) vac = nd < 5 check(f"G3_spread_x{x}_L{L}_{name}", not vac, f"{nd} distinct values{' VACUOUS' if vac else ''}") r, p = perm_partial(tbar, a_b, grain, LCG(SEED), draws=a.draws) rec = {"rho_partial": r, "perm_p": p, "z_fisher": fisher_z(r, nb), "rho_density_control": spearman(lam_bar * a_b, grain), "rho_naive_tbar": spearman(tbar, grain), "distinct": nd, "vacuous": vac} cell[name] = rec lv["scales"][f"L={div}"] = {"L": int(L), "nb": nb, "a_b": a_b.tolist(), "tbar_b": tbar.tolist(), "t_kb": both.tolist(), "g_b": g_b.tolist(), "nlong_b": nlong_b.tolist(), "topdec_b": topdec_b.tolist(), "stats": cell} print(f"x={x} L=M/{div} done {time.time() - t0:.1f}s", file=log, flush=True) json.dump(out, open(a.out, "w"), indent=1) lv["gap_local"] = gap_local(word, M, tpts, P_periods, LCG(SEED + 1), draws=a.draws) print(f"x={x} gap-local done {time.time() - t0:.1f}s", file=log, flush=True) out["levels"][str(x)] = lv json.dump(out, open(a.out, "w"), indent=1) # ---- pre-registered verdicts verdict = {} for x in levels: lv = out["levels"][str(x)] for name in ("gmax", "nlong", "topdec"): hits = 0; sign = 0; nonvac = 0 for sc in NEW_SCALES: s = lv["scales"][sc]["stats"][name] if s["vacuous"]: continue nonvac += 1 fired = abs(s["z_fisher"]) >= 2 and s["perm_p"] <= 0.01 hits += int(fired) if fired: sign += 1 if s["rho_partial"] > 0 else -1 verdict[f"x{x}_{name}_new_scales"] = {"non_vacuous_cells": nonvac, "scales_fired": hits, "falsifier_fired": hits >= 2 and abs(sign) == hits} gl = [out["levels"][str(x)]["gap_local"] for x in levels] fired_each = [abs(g["z_fisher"]) >= 2 and g["perm_p"] <= 0.01 for g in gl] signs = [1 if g["rho_rate_vs_group"] > 0 else -1 for g in gl] verdict["gap_local"] = {"fired_per_level": dict(zip([str(x) for x in levels], fired_each)), "falsifier_fired": all(fired_each) and len(set(signs)) == 1 and len(levels) >= 2} out["verdict"] = verdict json.dump(out, open(a.out, "w"), indent=1) print("\n".join(out["ledger"])) print("VERDICT " + json.dumps(verdict, sort_keys=True)) return 0 if __name__ == "__main__": sys.exit(main())