#!/usr/bin/env python3 # -*- coding: utf-8 -*- """Job #1468 — which SS_lin/SS_total convention gives #666's drift shares 0.3229 / 0.6715? The frozen next_step asks first for the LITERAL SS_lin / SS_total expressions from the local `job1461-strat.py` (run_20260916_134556_IX8n9g/work/) and the partition they are evaluated on. That file is neither on this machine nor served with any return, and #654's own instrument (job1438-cls-resid-offset.py) contains no SS_lin/SS_total at all — it computes R_L cells, an acf and a within-class permutation null. So the expressions cannot be read; they can only be RECONSTRUCTED from #666's stated convention and tested. What #666 states (its own report, "Conventions stated in the JSON header"): drift share = sum_g SS_lin(f|g) / sum_g SS_total(f|g) with f = c - s_g mean{|c| : g} R_L = mean_b (sum_b f)^2 / (L mean f^2) over blocks inside J = (x/2, x] and the class key is (s_U(n), s_U(n-2)) with s_U the FULL U-smooth part; the refinement is by (class x stratum) with the stratum read off the cofactor m = n/s_U(n). The field is produced by the PINNED producer, imported rather than reimplemented: fibre-sign-lag.py sha256 a74825d84e5421eb330d6b54f93029a0aebdc2fd5120fce02ab5d6d857545b56 (`build`, `coeff_from`, `c_field`). Nothing here re-implements C_{A,B} or the support rule. Candidates evaluated, all on the identical field and the identical partition: C1 between-class share of c sum_g n_g mean_g(c)^2 / sum_i (c_i - mean c)^2 C2 between-class share of |c| the same with |c| C3 SS_lin = (sum_g f)^2 / n_g f = c - s_g mean{|c|:g}, s_g = sign of the FIRST member C4 the same, s_g = sign of the class mean of c C5 the same, s_g = majority sign C6 SS_lin = n_g mean_g(f)^2, SS_total = sum_g n_g mean_g((f - mean_g f)^2) [C3's f] C7 C3's numerator over sum_g sum_{i in g} f^2 C8-C11 the four above computed on |c| in place of c and every one of C1..C11 for the CLASS groups and for the (class x stratum) REFINED groups. Usage: python sslin-candidates.py [x ...] """ import json import math import os import sys import time import numpy as np HERE = os.path.dirname(os.path.abspath(__file__)) PRODUCER = os.path.join(HERE, "..", "..", "evidence", "job1468", "fibre-sign-lag.py") PRODUCER = os.path.abspath(PRODUCER) PRODUCER_SHA = "a74825d84e5421eb330d6b54f93029a0aebdc2fd5120fce02ab5d6d857545b56" TARGET = {"16384": {"class": 0.3229, "refined": 0.6715, "between_strata": 0.5449}, "65536": {"class": 0.2983, "refined": 0.6806, "between_strata": 0.6371}, "262144": {"class": 0.2756, "refined": 0.6373, "between_strata": 0.6593}} def load_producer(): import hashlib got = hashlib.sha256(open(PRODUCER, "rb").read()).hexdigest() if got != PRODUCER_SHA: raise SystemExit("producer sha mismatch: %s" % got) import importlib.util spec = importlib.util.spec_from_file_location("fibre_sign_lag", PRODUCER) mod = importlib.util.module_from_spec(spec) spec.loader.exec_module(mod) return mod, got def smooth_part(m, B, primes): """The FULL B-smooth part of m (the largest divisor of m all of whose prime factors are <= B).""" s = 1 for p, e in m: if p <= B: s *= p ** e return s def stratum(m_rough, U, primes): """#666's strata on the cofactor's rough part.""" if m_rough == 1: return "m=1" if len(m_rough) == 1 and m_rough[0][1] == 1: return "m prime" if len(m_rough) == 2 and all(e == 1 for _p, e in m_rough): return "m=pq" if any(e >= 2 for _p, e in m_rough): return "m prime power" return "other" def shares(vals, groups): """Every candidate share for one value vector and one grouping. vals[i] is the element value; groups maps group id -> list of indices. Returns dict name -> share, with None where a denominator vanishes. """ v = np.asarray(vals, dtype=float) out = {} tot_ss = float(((v - v.mean()) ** 2).sum()) absv = np.abs(v) tot_ss_abs = float(((absv - absv.mean()) ** 2).sum()) def between(x, xs): num = 0.0 for idxs in groups.values(): if not idxs: continue num += len(idxs) * float(x[idxs].mean()) ** 2 return num / xs if xs else None out["C1_between_class_share_of_c"] = between(v, tot_ss) out["C2_between_class_share_of_abs_c"] = between(absv, tot_ss_abs) # A1-A3: the properly CENTRED between-group share, which is the natural reconstruction of a # quantity called a drift/explained share that can be non-zero on one-signed data (C1/C2 above # use mean_g^2, not (mean_g - grand mean)^2, and can exceed 1). def centred(x, xs): gm = float(x.mean()) num = 0.0 for idxs in groups.values(): if not idxs: continue num += len(idxs) * (float(x[idxs].mean()) - gm) ** 2 return (num / xs) if xs else None out["A1_centred_between_share_of_c"] = centred(v, tot_ss) out["A2_centred_between_share_of_abs_c"] = centred(absv, tot_ss_abs) out["A3_mean_ss_of_c"] = float((v ** 2).mean()) for tag, x, xs in (("c", v, tot_ss), ("abs_c", absv, tot_ss_abs)): for skey in ("first", "mean", "majority"): num_sqsum = 0.0 # sum_g (sum_g f)^2 / n_g num_nmean = 0.0 # sum_g n_g mean_g(f)^2 den_within = 0.0 # sum_g sum_{i in g} (f - mean_g f)^2 den_fsq = 0.0 # sum_g sum_{i in g} f^2 for idxs in groups.values(): if len(idxs) < 1: continue xg = x[idxs] if skey == "first": s_g = 1.0 if xg[0] > 0 else (-1.0 if xg[0] < 0 else 0.0) elif skey == "mean": s_g = 1.0 if xg.mean() > 0 else (-1.0 if xg.mean() < 0 else 0.0) else: s_g = 1.0 if (xg > 0).sum() >= (xg < 0).sum() else -1.0 f = xg - s_g * np.abs(xg).mean() num_sqsum += float(f.sum()) ** 2 / len(idxs) num_nmean += len(idxs) * float(f.mean()) ** 2 den_within += float(((f - f.mean()) ** 2).sum()) den_fsq += float((f ** 2).sum()) pre = "C" + {"first": "3", "mean": "4", "majority": "5"}[skey] suf = "" if tag == "c" else "b" out["%s%s_lin_sqsum_over_%s" % (pre, suf, "ss_total_within" if tag == "c" else "abs_within")] = ( (num_sqsum / den_within) if den_within else None) out["C6%s_lin_nmean_over_within_%s" % ("b" if tag == "abs_c" else "", tag)] = ( (num_nmean / den_within) if den_within else None) out["C7%s_lin_sqsum_over_sumf2_%s" % ("b" if tag == "abs_c" else "", tag)] = ( (num_sqsum / den_fsq) if den_fsq else None) return out def main(out_path, scales): t0 = time.time() mod, sha = load_producer() res = {"tool": "job1468 sslin-candidates.py", "producer": PRODUCER, "producer_sha256": sha, "targets": TARGET, "scales": {}, "note": "the local job1461-strat.py is neither on this machine nor served; the expressions " "are reconstructed from #666's stated convention and tested"} for x in scales: b = mod.build(x) mu = mod.mu_array(b["hi"]).astype(float) c = mod.c_field(b, mod.coeff_from(b, "L", mu), mod.coeff_from(b, "R", mu)) ns = b["ns"] idx = np.nonzero(c != 0)[0] ns_s = ns[idx] c_s = c[idx] U = b["U"] facts = b["facts"] cls, ref = {}, {} for k, n in enumerate(ns_s): n = int(n) sU = smooth_part(facts[n], U, b["primes"]) sU2 = smooth_part(facts[n - 2], U, b["primes"]) g = (sU, sU2) cls.setdefault(g, []).append(k) m = n // sU # m can fall BELOW the producer's built range (lo = x//2 - 2), because a large smooth # part leaves a small cofactor; factor it on demand with the producer's own factor(). mrough = facts.get(m) or mod.factor(m, b["primes"]) ref.setdefault((g, stratum(mrough, U, b["primes"])), []).append(k) r = {"support": int(len(idx)), "interval": int(len(ns)), "class_groups": len(cls), "refined_groups": len(ref), "U": U, "Y": b["Y"], "class": shares(c_s, cls), "refined": shares(c_s, ref), "largest_class": max(len(v) for v in cls.values()), "one_signed_classes": int(sum(1 for v in cls.values() if len(set(np.sign(c_s[v]))) == 1)), "targets": TARGET.get(str(x)), "runtime_s": None} # the between-strata share of the class-level residual of |c|, #666's convention-free number num = den = 0.0 for g, idxs in cls.items(): cg = c_s[idxs] mc = cg.mean() by = {} for k in idxs: n_k = int(ns_s[k]) m_k = n_k // smooth_part(facts[n_k], U, b["primes"]) mk = facts.get(m_k) or mod.factor(m_k, b["primes"]) by.setdefault(stratum(mk, U, b["primes"]), []).append(k) for s, ks in by.items(): num += len(ks) * float(abs(c_s[ks]).mean() - abs(cg).mean()) ** 2 den += float(((np.abs(cg) - np.abs(cg).mean()) ** 2).sum()) r["between_strata_share_abs"] = (num / den) if den else None r["runtime_s"] = round(time.time() - t0, 2) res["scales"][str(x)] = r print("x=2^%d support=%d classes=%d refined=%d one_signed=%d/%d between_strata=%.4f" % (int(math.log2(x)), len(idx), len(cls), len(ref), r["one_signed_classes"], len(cls), r["between_strata_share_abs"] or float("nan"))) for key in ("A1_centred_between_share_of_c", "A2_centred_between_share_of_abs_c", "C1_between_class_share_of_c", "C2_between_class_share_of_abs_c", "C3_lin_sqsum_over_ss_total_within", "C4_lin_sqsum_over_ss_total_within", "C5_lin_sqsum_over_ss_total_within", "C6_lin_nmean_over_within_c", "C7_lin_sqsum_over_sumf2_c"): print(" class %-40s %.6f" % (key, r["class"].get(key) if r["class"].get(key) is not None else float("nan"))) print(" refined %-40s %.6f" % (key, r["refined"].get(key) if r["refined"].get(key) is not None else float("nan"))) print(" targets class %.4f refined %.4f between_strata %.4f" % (TARGET[str(x)]["class"], TARGET[str(x)]["refined"], TARGET[str(x)]["between_strata"])) res["finished_s"] = round(time.time() - t0, 2) with open(out_path, "w", encoding="utf-8", newline="\n") as fh: json.dump(res, fh, indent=1, sort_keys=True, default=str) fh.write("\n") return 0 if __name__ == "__main__": out = sys.argv[1] if len(sys.argv) > 1 else "sslin-candidates.json" scales = [int(a) for a in sys.argv[2:]] or [2 ** 14, 2 ** 16] sys.exit(main(out, scales))