"""Job #1936 (route 35 rev 10): the same block-grain and gap-local tests at x = 29 (M = 29# = 6 469 693 230, 2 periods, exposure [M, 3M)), by streaming: the tile word (214 708 725 slots per period) is never held in memory. The period is cut into 128 fine blocks of L = M // 128 (the last one takes the remainder); each fine block's admissible offsets are produced by a residue sieve (strike r = 0 and r = -2 mod q for q = 3..29 in the block), its in-block gaps are histogrammed by value, and the gap across the fine-block boundary is kept separately, so every coarser block (M/8 .. M/64) is a union of fine blocks whose gap statistics are exact sums/maxima of the fine histograms plus the interior boundary gaps. Twin positions come from the chunked segmented sieve of #1029 over [M, 3M). The gap-local context c(r) = max(gap before, gap after) is accumulated per value (slots and twins per context value) across the stream, and its null is the exact multivariate hypergeometric of the main instrument. Statistics, gates (G1 slot count = D(T_29) = 214 708 725, G2 sums, G3 spread), falsifiers and seed are those of job1936-blockgrain.py. Stdout: ledger + verdict; stderr: progress. RUN: python job1936-x29.py --out job1936-x29.json """ import argparse, json, math, sys, time import numpy as np from importlib.machinery import SourceFileLoader bg = SourceFileLoader("bg", "job1936-blockgrain.py").load_module() X = 29; PERIODS = 2; DRAWS, SEED = 400, 1936; D_T29 = 214708725; FINE = 128 SCALES = [8, 16, 32, 64, 128]; NEW_SCALES = ["L=64", "L=128"] def block_offsets(lo, hi, qs): """Admissible offsets r in [lo, hi): r and r + 2 coprime to 29# (r odd handled by q = 2: r must be odd).""" n = hi - lo ok = np.ones(n, dtype=bool) ok[(-lo) % 2:: 2] = False # even r are out (2 | r) for q in qs: for kill in (0, (q - 2) % q): s = (kill - lo) % q ok[s:: q] = False return np.flatnonzero(ok).astype(np.int64) + lo def main(): ap = argparse.ArgumentParser(); ap.add_argument("--out", required=True); ap.add_argument("--draws", type=int, default=DRAWS) a = ap.parse_args(); log = sys.stderr; t0 = time.time() qs = [int(q) for q in bg.primes_to(X) if q > 2] M = 2 for q in qs: M *= q hi = (PERIODS + 1) * M out = {"instrument": "job1936-x29.py", "job": 1936, "route": 35, "level": X, "M": int(M), "periods": PERIODS, "preregistered": "as job1936-blockgrain.py (scales 8..128, spread gate, gmax/nlong/topdec, gap-local groups, F-block on L=64,128, F-local)", "checks": [], "ledger": []} def check(tag, ok, detail): out["checks"].append({"check": tag, "pass": bool(ok), "detail": detail}); out["ledger"].append(f"{'PASS' if ok else 'FAIL'} {tag} :: {detail}") # ---- twins over the exposure sp = bg.primes_to(int(math.isqrt(hi)) + 1) check("G1c_sieve_primes_x29", int(sp[-1]) < M, f"largest marking prime {int(sp[-1])} < M={M}") tpts = bg.twins_in(sp, M, hi, log=log) print(f"twins {len(tpts)} in [{M},{hi}) {time.time() - t0:.1f}s", file=log, flush=True) per_k = {k: int(((tpts >= k * M) & (tpts < (k + 1) * M)).sum()) for k in range(1, PERIODS + 1)} tot_tw = sum(per_k.values()) # ---- stream the tile word in fine blocks L = M // FINE edges = [b * L for b in range(FINE)] + [M] GMAX = 4096 # gap values are far below this (T_29 max gap is a few hundred) ghist = np.zeros((FINE, GMAX), dtype=np.int64) # in-block gap histograms bgap = np.zeros(FINE, dtype=np.int64) # gap from the block's last slot to the next block's first slot a_fine = np.zeros(FINE, dtype=np.int64) ctx_slots = np.zeros(GMAX, dtype=np.int64); ctx_tw = np.zeros((PERIODS, GMAX), dtype=np.int64) prev_last = None; first_slot = None; pending = None # pending = (offset, gap_before) of the previous block's last slot slots_total = 0 for b in range(FINE): lo, hi_b = edges[b], edges[b + 1] offs = block_offsets(lo, hi_b, qs) a_fine[b] = len(offs); slots_total += len(offs) if first_slot is None: first_slot = int(offs[0]) gaps = np.diff(offs) if len(gaps): ghist[b] += np.bincount(gaps, minlength=GMAX)[:GMAX] if prev_last is not None: bgap[b - 1] = int(offs[0]) - prev_last # contexts: for interior slots max(gap before, gap after); the block's first slot needs the boundary gap before # it (bgap[b-1]) and its last slot the boundary gap after it (known only at the next block) -> carry as pending gb = np.empty(len(offs), dtype=np.int64); ga = np.empty(len(offs), dtype=np.int64) gb[1:] = gaps; ga[:-1] = gaps gb[0] = bgap[b - 1] if b > 0 else -1; ga[-1] = -1 c = np.maximum(gb, ga) # twins in this block for each period, mapped to slot indices tw_idx = [] for k in range(1, PERIODS + 1): sel = (tpts >= k * M + lo) & (tpts < k * M + hi_b) off = tpts[sel] - k * M pos = np.searchsorted(offs, off) assert (offs[np.minimum(pos, len(offs) - 1)] == off).all(), "twin outside the tile word" tw_idx.append(pos) # resolve the pending last slot of the previous block (its gap after = bgap[b-1]) if pending is not None: poff, pgb, ptw = pending pc = max(pgb, int(bgap[b - 1])) ctx_slots[pc] += 1 for k in range(PERIODS): if ptw[k]: ctx_tw[k, pc] += 1 # accumulate contexts for all but the last slot (and the first slot of block 0, whose gap-before wraps: handled at the end) keep = np.ones(len(offs), dtype=bool); keep[-1] = False if b == 0: keep[0] = False ctx_slots += np.bincount(c[keep], minlength=GMAX)[:GMAX] for k in range(PERIODS): twk = np.zeros(len(offs), dtype=bool); twk[tw_idx[k]] = True ctx_tw[k] += np.bincount(c[keep & twk], minlength=GMAX)[:GMAX] pending = (int(offs[-1]), int(gb[-1]), [bool(np.isin(len(offs) - 1, tw_idx[k])) for k in range(PERIODS)]) if b == 0: first_info = (int(offs[0]), int(ga[0]), [bool(np.isin(0, tw_idx[k])) for k in range(PERIODS)]) prev_last = int(offs[-1]) if b % 8 == 7: print(f"block {b + 1}/{FINE} slots so far {slots_total} {time.time() - t0:.1f}s", file=log, flush=True) # wrap: last slot of the period -> first slot + M wrap_gap = first_slot + M - prev_last poff, pgb, ptw = pending; pc = max(pgb, wrap_gap); ctx_slots[pc] += 1 for k in range(PERIODS): if ptw[k]: ctx_tw[k, pc] += 1 foff, fga, ftw = first_info; fc = max(wrap_gap, fga); ctx_slots[fc] += 1 for k in range(PERIODS): if ftw[k]: ctx_tw[k, fc] += 1 check("G1_slots_x29", slots_total == D_T29, f"streamed {slots_total} vs D(T_29) {D_T29}") check("G1b_period_x29", first_slot == 29 and prev_last == M - 1, f"first slot {first_slot}, last slot {prev_last}, M-1 {M - 1}") check("G1d_context_total_x29", int(ctx_slots.sum()) == slots_total, f"context slots {int(ctx_slots.sum())} vs {slots_total}") check("G1e_context_twins_x29", int(ctx_tw.sum()) == tot_tw, f"context twins {int(ctx_tw.sum())} vs sieved {tot_tw}") allhist = ghist.sum(axis=0) + np.bincount(bgap[:-1], minlength=GMAX)[:GMAX] + np.bincount([wrap_gap], minlength=GMAX)[:GMAX] gvals = np.flatnonzero(allhist); cum = np.cumsum(allhist[gvals]); q90 = int(gvals[np.searchsorted(cum, 0.9 * cum[-1])]) lam_bar = tot_tw / (PERIODS * slots_total) lv = {"M": int(M), "periods": PERIODS, "slots_per_period": int(slots_total), "twins": int(tot_tw), "twins_per_k": {str(k): per_k[k] for k in per_k}, "lambda_hat": lam_bar, "lambda_k_min": min(per_k.values()) / slots_total, "lambda_k_max": max(per_k.values()) / slots_total, "tile_gap_max": int(gvals.max()), "tile_gap_q90": q90, "tile_gap_distinct": int(len(gvals)), "gap_value_histogram": {str(int(v)): int(allhist[v]) for v in gvals}, "scales": {}} # ---- block statistics at every scale from the fine histograms t_fine = np.zeros((PERIODS, FINE)) for k in range(1, PERIODS + 1): sel = (tpts >= k * M) & (tpts < (k + 1) * M) t_fine[k - 1] = np.bincount(np.minimum((tpts[sel] - k * M) // L, FINE - 1), minlength=FINE) for div in SCALES: nb = div; per = FINE // div a_b = np.array([a_fine[i * per:(i + 1) * per].sum() for i in range(nb)], dtype=float) t_kb = np.array([[t_fine[k, i * per:(i + 1) * per].sum() for i in range(nb)] for k in range(PERIODS)]) g_b = np.zeros(nb); nlong_b = np.zeros(nb); topdec_b = np.zeros(nb) for i in range(nb): h = ghist[i * per:(i + 1) * per].sum(axis=0).copy() if per > 1: h += np.bincount(bgap[i * per:(i + 1) * per - 1], minlength=GMAX)[:GMAX] # interior boundary gaps nz = np.flatnonzero(h) g_b[i] = float(nz.max()) if len(nz) else 0.0 nlong_b[i] = float(h[2 * X + 1:].sum()); topdec_b[i] = float(h[q90:].sum()) tbar = t_kb.mean(axis=0) check(f"G2_sums_x29_L{M // div}", int(a_b.sum()) == slots_total and int(t_kb.sum()) == tot_tw, f"a_b sum {int(a_b.sum())} vs {slots_total}; t sum {int(t_kb.sum())} vs {tot_tw}") cell = {} for name, grain in (("gmax", g_b), ("nlong", nlong_b), ("topdec", topdec_b)): nd = int(len(np.unique(grain))); vac = nd < 5 check(f"G3_spread_x29_L{M // div}_{name}", not vac, f"{nd} distinct values{' VACUOUS' if vac else ''}") r, p = bg.perm_partial(tbar, a_b, grain, bg.LCG(SEED), draws=a.draws) cell[name] = {"rho_partial": r, "perm_p": p, "z_fisher": bg.fisher_z(r, nb), "rho_density_control": bg.spearman(lam_bar * a_b, grain), "rho_naive_tbar": bg.spearman(tbar, grain), "distinct": nd, "vacuous": vac} lv["scales"][f"L={div}"] = {"L": int(M // div), "nb": nb, "a_b": a_b.tolist(), "tbar_b": tbar.tolist(), "t_kb": t_kb.astype(int).tolist(), "g_b": g_b.tolist(), "nlong_b": nlong_b.tolist(), "topdec_b": topdec_b.tolist(), "stats": cell} print(f"L=M/{div} done {time.time() - t0:.1f}s", file=log, flush=True) # ---- gap-local from the context accumulators (same grouping rule and null as the main instrument) vals = np.flatnonzero(ctx_slots); cnt = ctx_slots[vals]; ngroups = 10 target = slots_total / ngroups; groups = np.zeros(len(vals), dtype=int); g = 0; acc = 0 for i in range(len(vals)): if acc >= target and g < ngroups - 1: g += 1; acc = 0 groups[i] = g; acc += cnt[i] ng = g + 1 slots_g = np.array([cnt[groups == i].sum() for i in range(ng)], dtype=float) tw_kg = np.array([[ctx_tw[k, vals[groups == i]].sum() for i in range(ng)] for k in range(PERIODS)], dtype=float) rate = tw_kg.sum(axis=0) / (slots_g * PERIODS); order = np.arange(ng) obs = bg.spearman(rate, order) rng = bg.LCG(SEED + 1); nrng = np.random.default_rng(rng.next()); ge = 0 for _ in range(a.draws): tot = np.zeros(ng) for k in range(PERIODS): tot += nrng.multivariate_hypergeometric(slots_g.astype(np.int64), int(tw_kg[k].sum())) if abs(bg.spearman(tot / (slots_g * PERIODS), order)) >= abs(obs) - 1e-15: ge += 1 lv["gap_local"] = {"groups": ng, "group_context_ranges": [[int(vals[groups == i].min()), int(vals[groups == i].max())] for i in range(ng)], "slots_per_group": slots_g.tolist(), "twins_per_group": tw_kg.sum(axis=0).tolist(), "occupancy_rate_per_group": rate.tolist(), "rho_rate_vs_group": obs, "perm_p": (ge + 1) / (a.draws + 1), "z_fisher": bg.fisher_z(obs, ng), "draws": a.draws, "null": "multivariate hypergeometric per period", "distinct_context_values": int(len(vals))} out["levels"] = {"29": lv} verdict = {} for name in ("gmax", "nlong", "topdec"): hits = 0; sign = 0; nonvac = 0 for sc in NEW_SCALES: s = lv["scales"][sc]["stats"][name] if s["vacuous"]: continue nonvac += 1; fired = abs(s["z_fisher"]) >= 2 and s["perm_p"] <= 0.01; hits += int(fired) if fired: sign += 1 if s["rho_partial"] > 0 else -1 verdict[f"x29_{name}_new_scales"] = {"non_vacuous_cells": nonvac, "scales_fired": hits, "falsifier_fired": hits >= 2 and abs(sign) == hits} gl = lv["gap_local"]; verdict["gap_local_x29"] = {"fired": abs(gl["z_fisher"]) >= 2 and gl["perm_p"] <= 0.01, "rho": gl["rho_rate_vs_group"]} out["verdict"] = verdict json.dump(out, open(a.out, "w"), indent=1) print("\n".join(out["ledger"])); print("VERDICT " + json.dumps(verdict, sort_keys=True)) return 0 if __name__ == "__main__": sys.exit(main())