# power1936.py -- the smallest detectable |rho| per cell, as the route's success branch asks ("a null with the cell's # spread, block count and the smallest detectable rho recorded"). For every non-vacuous cell of a job1936 output file the # permutation null of job1936-blockgrain.perm_partial is regenerated from the stored a_b, tbar_b and grain arrays with the # same LCG seed (so the observed p is reproduced as a check), and the null's 0.99 quantile of |rho_partial| is the # smallest |rho| that would have reached rank-p <= 0.01 in that cell; alongside, the Fisher |rho| for |z| = 2 (2/sqrt(nb-3)). # Also the gap-local cells: the hypergeometric null's 0.99 quantile of |rho| over the groups. Deterministic. Prints JSON. import sys, json, math import numpy as np from importlib.machinery import SourceFileLoader bg = SourceFileLoader("bg", "job1936-blockgrain.py").load_module() rows = [] for f in sys.argv[1:]: d = json.load(open(f)) for x, lv in d["levels"].items(): for sc, v in lv["scales"].items(): a_b = np.array(v["a_b"], float); tbar = np.array(v["tbar_b"], float); nb = v["nb"] for name, key in (("gmax", "g_b"), ("nlong", "nlong_b"), ("topdec", "topdec_b")): s = v["stats"][name] if s["vacuous"]: continue grain = np.array(v[key], float) am = a_b - a_b.mean(); den = float(am @ am) def resid(vv): beta = 0.0 if den == 0 else float(am @ (vv - vv.mean())) / den return vv - beta * a_b rng = bg.LCG(bg.SEED); obs = bg.spearman(resid(tbar), grain); null = [] for _ in range(s.get("draws", 400) if "draws" in s else 400): idx = rng.shuffle(nb); null.append(abs(bg.spearman(resid(tbar[idx]), grain))) null = np.sort(np.array(null)); p = (int((null >= abs(obs) - 1e-15).sum()) + 1) / (len(null) + 1) rows.append(dict(file=f, x=int(x), scale=sc, nb=nb, statistic=name, distinct=s["distinct"], rho_obs=obs, p_reproduced=p, p_stored=s["perm_p"], min_detectable_abs_rho_p01=float(np.quantile(null, 0.99)), abs_rho_for_z2=2 / math.sqrt(max(nb - 3, 1)), null_abs_rho_median=float(np.median(null)))) g = lv.get("gap_local") if g and "slots_per_group" in g: slots_g = np.array(g["slots_per_group"], float); ng = len(slots_g); order = np.arange(ng); periods = lv["periods"] tw_total = sum(g["twins_per_group"]); per_k = [int(round(tw_total / periods))] * periods # equal split is enough for the null quantile rng = bg.LCG(bg.SEED + 1); nrng = np.random.default_rng(rng.next()); null = [] for _ in range(g["draws"]): tot = np.zeros(ng) for k in range(periods): tot += nrng.multivariate_hypergeometric(slots_g.astype(np.int64), per_k[k]) null.append(abs(bg.spearman(tot / (slots_g * periods), order))) null = np.sort(np.array(null)) rows.append(dict(file=f, x=int(x), scale="gap-local", nb=ng, statistic="occupancy rate vs context group", distinct=g["distinct_context_values"], rho_obs=g["rho_rate_vs_group"], p_stored=g["perm_p"], min_detectable_abs_rho_p01=float(np.quantile(null, 0.99)), abs_rho_for_z2=2 / math.sqrt(max(ng - 3, 1)), null_abs_rho_median=float(np.median(null)))) json.dump(rows, open("power1936.json", "w"), indent=1) for r in rows: print(f"x={r['x']} {r['scale']:>9} nb={r['nb']:>3} {r['statistic']:<32} nd={r['distinct']:>3} rho={r['rho_obs']:+.3f} p={r['p_stored']:.3f}" + (f" (repro {r['p_reproduced']:.3f})" if 'p_reproduced' in r else "") + f" min|rho|@p.01={r['min_detectable_abs_rho_p01']:.3f} |rho|@z2={r['abs_rho_for_z2']:.3f}")