# Round numbers in gold, v2 analysis (24 Oct 2026). Usage: python3 -I rn2.py XAU_1d_data.jsonl OUTDIR
import sys, json, math, random, csv, os, datetime as dt
from fractions import Fraction
src, out = sys.argv[1:3]; os.makedirs(out, exist_ok=True)
B = []
for l in open(src):
    if l.strip():
        r = json.loads(l); B.append((r["Date"][:10].replace(".", "-"), r["Open"], r["High"], r["Low"], r["Close"]))
B.sort()
# window and hole rule: same as notes 12-16 (skip a day whose previous bar is > 4 calendar days back)
D = []
for i, b in enumerate(B):
    if not ("2005-01-01" <= b[0] <= "2024-12-31"): continue
    if (dt.date.fromisoformat(b[0]) - dt.date.fromisoformat(B[i-1][0])).days > 4: continue
    D.append(b)
N = len(D); assert N == 5138, N
idx = {b[0]: i for i, b in enumerate(D)}

def events(step, off, look):
    """touch events for grid {off + k*step}; returns list of dicts"""
    ev = []
    for i, (d, o, h, l, c) in enumerate(D):
        k0 = math.ceil((l - off) / step); k1 = math.floor((h - off) / step)
        for k in range(k0, k1 + 1):
            L = off + k * step
            if o == L: continue                      # open exactly on level
            side = 1 if o > L else -1
            first = all(not (D[j][3] <= L <= D[j][2]) for j in range(max(0, i - look), i)) and i >= look
            res = {}
            for hz, j in (("d0", i), ("d1", i + 1), ("d5", i + 5)):
                if j >= N: res[hz] = None; continue
                cj = D[j][4]
                res[hz] = None if cj == L else (1 if (cj - L) * side > 0 else 0)
            ev.append({"i": i, "date": d, "L": L, "first": first, **res})
    return ev

def share(ev, hz, first):
    v = [e[hz] for e in ev if (e["first"] or not first) and e[hz] is not None]
    return sum(v), len(v)

def binom_p(k, n, p):  # exact two-sided, sum of outcomes with prob <= prob(k)
    lp = lambda i: math.lgamma(n+1)-math.lgamma(i+1)-math.lgamma(n-i+1)+i*math.log(p)+(n-i)*math.log(1-p)
    pk = lp(k); return min(1.0, sum(math.exp(lp(i)) for i in range(n+1) if lp(i) <= pk + 1e-9))

def wilson(k, n, z=1.96):
    p = k/n; d = 1+z*z/n; c = p+z*z/(2*n); r = z*math.sqrt(p*(1-p)/n+z*z/(4*n*n)); return (c-r)/d, (c+r)/d

random.seed(20261024)
rows = []; flow = []; boot_rows = []; look_rows = []; grid_rows = []
for step, offs in ((100, [o for o in range(1, 100) if o % 5]), (50, [o for o in range(1, 50) if o % 5])):
    grids = {0: events(step, 0, 5)}; grids.update({o: events(step, o, 5) for o in offs})
    R = grids[0]
    nall = len(R); nfirst = sum(e["first"] for e in R)
    flow.append({"grid": step, "touches_all": nall, "first_touches": nfirst,
                 "first_d0_used": share(R, "d0", True)[1], "first_d1_used": share(R, "d1", True)[1], "first_d5_used": share(R, "d5", True)[1],
                 "all_d0_used": share(R, "d0", False)[1], "all_d1_used": share(R, "d1", False)[1], "all_d5_used": share(R, "d5", False)[1]})
    for first in (True, False):
        for hz in ("d0", "d1", "d5"):
            k, n = share(R, hz, first)
            pk = [share(grids[o], hz, first) for o in offs]
            ps = sum(a for a, _ in pk) / sum(b for _, b in pk)
            rates = [a/b for a, b in pk]; r0 = k/n
            grid_rows.append({"grid": step, "sample": "first" if first else "all", "horizon": hz, "offset": 0, "held": k, "n": n, "rate_pct": round(100*r0, 2)})
            for o, (a, b) in zip(offs, pk): grid_rows.append({"grid": step, "sample": "first" if first else "all", "horizon": hz, "offset": o, "held": a, "n": b, "rate_pct": round(100*a/b, 2)})
            below = sum(1 for x in rates if x <= r0) + 1; above = sum(1 for x in rates if x >= r0) + 1   # include observed grid
            n_g = len(rates) + 1
            p_rank = min(1.0, 2 * min(below, above) / n_g)
            rank = 1 + sum(1 for x in rates if x > r0)   # 1 = held most
            lo, hi = wilson(k, n)
            rows.append({"grid": step, "sample": "first" if first else "all", "horizon": hz, "held": k, "n": n,
                         "round_pct": round(100*r0, 1), "ci_lo": round(100*lo, 1), "ci_hi": round(100*hi, 1),
                         "placebo_pct": round(100*ps, 1), "diff_pts": round(100*(r0-ps), 1),
                         "binom_p": binom_p(k, n, ps), "rank_p": round(p_rank, 3), "rank": rank, "grids": n_g})
    # month-block bootstrap of round-minus-placebo difference (first touches) for the 100 grid
    if step == 100:
        def blk(d, kind): return d[:7] if kind == "month" else d[:4] + "Q" + str((int(d[5:7]) - 1)//3 + 1)
        for kind in ("month", "quarter"):
          months = sorted({blk(b[0], kind) for b in D})
          for hz in ("d0", "d1", "d5"):
              def bym(ev):
                  m = {}
                  for e in ev:
                      if e["first"] and e[hz] is not None:
                          a = m.setdefault(blk(e["date"], kind), [0, 0]); a[0] += e[hz]; a[1] += 1
                  return m
              Rm = bym(R); Pm = {}
              for o in offs:
                  for mm, (a, b) in bym(grids[o]).items():
                      x = Pm.setdefault(mm, [0, 0]); x[0] += a; x[1] += b
              obs = sum(v[0] for v in Rm.values())/sum(v[1] for v in Rm.values()) - sum(v[0] for v in Pm.values())/sum(v[1] for v in Pm.values())
              diffs = []
              for _ in range(2000):
                  s = [random.choice(months) for _ in months]
                  rk = sum(Rm.get(m, [0, 0])[0] for m in s); rn = sum(Rm.get(m, [0, 0])[1] for m in s)
                  pk_ = sum(Pm.get(m, [0, 0])[0] for m in s); pn = sum(Pm.get(m, [0, 0])[1] for m in s)
                  if rn and pn: diffs.append(rk/rn - pk_/pn)
              diffs.sort(); lo, hi = diffs[int(.025*len(diffs))], diffs[int(.975*len(diffs))-1]
              pb = min(1.0, 2*min(sum(1 for x in diffs if x <= 0), sum(1 for x in diffs if x >= 0))/len(diffs))
              boot_rows.append({"block": kind, "horizon": hz, "diff_pts": round(100*obs, 1), "ci_lo": round(100*lo, 1), "ci_hi": round(100*hi, 1), "boot_p": round(pb, 3), "se_pts": round(100*(sum((x-sum(diffs)/len(diffs))**2 for x in diffs)/(len(diffs)-1))**0.5, 1), "blocks": len(months), "reps": len(diffs)})
        # lookback sensitivity, round grid vs placebo, d0 and d1
        for look in (1, 5, 10):
            Rl = events(100, 0, look); Pl = [events(100, o, look) for o in offs]
            for hz in ("d0", "d1"):
                k, n = share(Rl, hz, True); pk = [share(g, hz, True) for g in Pl]
                ps = sum(a for a, _ in pk)/sum(b for _, b in pk); rates = sorted(a/b for a, b in pk); r0 = k/n
                p_rank = min(1.0, 2*min(sum(1 for x in rates if x <= r0)+1, sum(1 for x in rates if x >= r0)+1)/(len(rates)+1))
                look_rows.append({"lookback_days": look, "horizon": hz, "held": k, "n": n, "round_pct": round(100*r0, 1), "placebo_pct": round(100*ps, 1), "rank_p": round(p_rank, 3)})

# Holm on the primary family: grid 100, first touches, rank-based p, 3 horizons
prim = [r for r in rows if r["grid"] == 100 and r["sample"] == "first"]
order = sorted(prim, key=lambda r: r["rank_p"])
for j, r in enumerate(order):
    r["holm_rank_p"] = round(min(1.0, max(min(1.0, (len(order)-i)*order[i]["rank_p"]) for i in range(j+1))), 3)
def wcsv(name, rr):
    with open(os.path.join(out, name), "w", newline="") as f:
        w = csv.DictWriter(f, fieldnames=list({k: 0 for r in rr for k in r}.keys())); w.writeheader(); w.writerows(rr)
for r in rows: r["binom_p"] = float("%.3g" % r["binom_p"])
wcsv("grid_rates.csv", grid_rows); wcsv("tests.csv", rows); wcsv("flow.csv", flow); wcsv("boot.csv", boot_rows); wcsv("lookback.csv", look_rows)
print(json.dumps({"flow": flow, "rows": rows, "boot": boot_rows, "look": look_rows}, indent=0))