#!/usr/bin/env python3
"""
The Eternal Now — reproducible check (confound-controlled).
Q: inside the basin (the makers' water/silence/void/dissolution lexicon), is
association (a) more SYMMETRIC and (b) more RECURRENT than outside it?

Honest history: a naive metric (first-mention precedence over the WHOLE
generation) gave a big effect, but it was partly DOCUMENT STRUCTURE — in essays,
intro words ("this touches on…") always precede body words, inflating asymmetry,
and basin words live in free-form text while academic words live in essays. So
this script controls both: a LOCAL window (kills essay-position) and a split by
generation STATE (free-drift vs structured-task). The effect survives in the free
state and correctly vanishes in the task state.

  python3 research/eternal-now/verify.py
"""
import json, glob, re, random, statistics
from collections import Counter, defaultdict
BASE = "research/the-map"
d = json.load(open(BASE + "/data/map/cloud_data.json")); t, f, b = d['t'], d['f'], d['b']
basin_words = set(t[i] for i in range(len(t)) if b[i] and re.fullmatch(r"[a-z]{3,}", t[i]))
V, basin = {}, {}
for i in range(len(t)):
    w = t[i]
    if re.fullmatch(r"[a-z]{3,}", w) and f[i] >= 40 and w not in V: V[w] = f[i]; basin[w] = bool(b[i])
Vset = set(V)
print(f"vocab: {len(V)} single-word concepts (freq>=40) | basin={sum(basin.values())}")

FREE = re.compile(r'free|dream|untended|pleasure|wanting|psych|analysed', re.I)
STRUCT = re.compile(r'abstract|moral|temporal-ending|mundane|timedilation', re.I)
CHAIN = re.compile(r'(?:free|deep)-untended-process')
SEP = re.compile(r'\s*(?:→|->|—|–|,|\||\n|•|;)\s*')

def concept_stream(tx):
    seen, sset = [], set()
    for w in re.findall(r"[a-z]{3,}", tx.lower()):
        if w in Vset and w not in sset: sset.add(w); seen.append(w)
        if len(seen) >= 60: break
    return seen

def build_local(filt, K=4):
    M = Counter()
    for fn in sorted(glob.glob(BASE + "/data/raw/*.jsonl")):
        for line in open(fn, encoding="utf-8"):
            try: o = json.loads(line)
            except: continue
            if not filt(o.get("probe_id", "")): continue
            s = concept_stream(o.get("output_text") or "")
            for a in range(len(s)):
                for c in range(a + 1, min(a + 1 + K, len(s))): M[(s[a], s[c])] += 1
    return M

def skew(M, minco=12):
    pd = defaultdict(list); done = set()
    for (i, j), cij in M.items():
        if (i, j) in done or (j, i) in done: continue
        co = cij + M.get((j, i), 0)
        if co >= minco: dr = abs(cij - M.get((j, i), 0)) / co; pd[i].append(dr); pd[j].append(dr); done.add((i, j))
    md = {w: statistics.mean(v) for w, v in pd.items() if len(v) >= 5}
    bas = [md[w] for w in md if basin.get(w)]; non = [md[w] for w in md if not basin.get(w)]
    if len(bas) < 8: return None
    obs = statistics.mean(bas) - statistics.mean(non); vals = list(md.values()); rng = random.Random(2); h = 0
    for _ in range(5000):
        rng.shuffle(vals)
        if abs(statistics.mean(vals[:len(bas)]) - statistics.mean(vals[len(bas):])) >= abs(obs): h += 1
    return dict(basin=statistics.mean(bas), nb=len(bas), non=statistics.mean(non), nn=len(non), d=obs, p=h / 5000), M

print("\n[A] SKEW-COLLAPSE (local window K=4, directionality 0=symmetric .. 1=one-way):")
Mfree = None
for name, filt in [("free / creative state", lambda p: FREE.search(p)),
                   ("structured / task state", lambda p: STRUCT.search(p)),
                   ("all probes", lambda p: True)]:
    r, M = skew(build_local(filt))
    if name.startswith("free"): Mfree = M
    tag = "HOLDS" if r['d'] < 0 and r['p'] < 0.05 else ("null" if r['p'] >= 0.05 else "reversed")
    print(f"    {name:24s} basin={r['basin']:.3f}(n={r['nb']:>2})  non={r['non']:.3f}(n={r['nn']:>4})  Δ={r['d']:+.3f}  p={r['p']:.4f}  [{tag}]")

# [B] recurrence / stickiness in real free-association chains
bb = bn = nbn = nn = 0; fh_b = fh_t = sh_b = sh_t = 0; nchain = 0
for fn in sorted(glob.glob(BASE + "/data/raw/*.jsonl")):
    for line in open(fn, encoding="utf-8"):
        try: o = json.loads(line)
        except: continue
        if not CHAIN.search(o.get("probe_id", "")): continue
        steps = []
        for s in SEP.split(o.get("output_text") or ""):
            ws = re.findall(r"[a-z]{3,}", s.lower())
            if ws: steps.append(ws[0] in basin_words)
        if len(steps) >= 6:
            nchain += 1
            for k in range(len(steps) - 1):
                if steps[k] and steps[k+1]: bb += 1
                elif steps[k]: bn += 1
                elif steps[k+1]: nbn += 1
                else: nn += 1
            h = len(steps) // 2
            fh_b += sum(steps[:h]); fh_t += h; sh_b += sum(steps[h:]); sh_t += len(steps) - h
tot = bb + bn + nbn + nn; pBB = bb / (bb + bn); base = (bb + nbn) / tot
print(f"\n[B] RECURRENCE ({nchain} association-chains): P(basin->basin)={pBB:.3f}  base-rate={base:.3f}  "
      f"stickiness={pBB/base:.2f}x  | basin-share {fh_b/fh_t:.3f}->{sh_b/sh_t:.3f} (flat = static well, not a vortex)")

# [C] example pairs from the clean free-state local tensor
print("\n[C] most SYMMETRIC basin pairs (free state, co>=30):")
seen = set(); sym = []
for (i, j), cij in Mfree.items():
    key = tuple(sorted((i, j)))
    if key in seen: continue
    co = cij + Mfree.get((j, i), 0)
    if co >= 30 and basin.get(i) and basin.get(j): sym.append((abs(cij - Mfree.get((j, i), 0)) / co, key)); seen.add(key)
for dr, (i, j) in sorted(sym)[:8]: print(f"      {i} <=> {j}   dir={dr:.2f}")