#!/usr/bin/env python3 from __future__ import annotations import json import sqlite3 from collections import Counter from datetime import timezone from pathlib import Path import matplotlib.dates as mdates import matplotlib.pyplot as plt import numpy as np import pandas as pd from matplotlib.patches import Rectangle ROOT = Path(__file__).resolve().parent evidence = json.loads((ROOT / "evidence.json").read_text()) START = pd.Timestamp(evidence["window"]["start"]) END = pd.Timestamp(evidence["window"]["end"]) def iso(value) -> str | None: if value is None: return None value = pd.Timestamp(value) if value.tzinfo is None: value = value.tz_localize("UTC") return value.isoformat().replace("+00:00", "Z") frame = pd.DataFrame(evidence["candles"]) frame["time"] = pd.to_datetime(frame["time"], utc=True) for column in ("open", "high", "low", "close", "volume", "atr", "adx"): frame[column] = pd.to_numeric(frame[column], errors="coerce") frames = {pair: rows.sort_values("time").set_index("time") for pair, rows in frame.groupby("pair")} db = sqlite3.connect(f"file:{(ROOT / 'trades.sqlite').resolve()}?mode=ro", uri=True) db.row_factory = sqlite3.Row trades = {row["id"]: dict(row) for row in db.execute("SELECT * FROM trades")} def entry_order(trade: dict) -> dict: side = "sell" if trade["is_short"] else "buy" return dict(db.execute( "SELECT * FROM orders WHERE ft_trade_id=? AND ft_order_side=? AND order_type='limit' ORDER BY order_date LIMIT 1", (trade["id"], side), ).fetchone()) def path_metric(pair: str, when, side: str, hours: int) -> dict | None: when = pd.Timestamp(when) if when.tzinfo is None: when = when.tz_localize("UTC") data = frames[pair] eligible = data[data.index >= when.floor("5min")] if eligible.empty: return None start = eligible.index[0] stop = start + pd.Timedelta(hours=hours) if data.index.max() < stop: return None rows = data[(data.index >= start) & (data.index <= stop)] entry = float(rows.iloc[0].close) high, low, last = float(rows.high.max()), float(rows.low.min()), float(rows.iloc[-1].close) if side == "short": mfe, mae, end_ret = (entry-low)/entry, (high-entry)/entry, (entry-last)/entry else: mfe, mae, end_ret = (high-entry)/entry, (entry-low)/entry, (last-entry)/entry return {"entry": entry, "mfe": mfe, "mae": mae, "endReturn": end_ret, "trailArm": mfe >= 0.025, "hardStop": mae >= 0.025} def aggregate(items) -> dict: rows = [row for row in items if row is not None] if not rows: return {"n": 0} return { "n": len(rows), "mfeMean": float(np.mean([x["mfe"] for x in rows])), "maeMean": float(np.mean([x["mae"] for x in rows])), "endReturnMean": float(np.mean([x["endReturn"] for x in rows])), "endReturnMedian": float(np.median([x["endReturn"] for x in rows])), "positiveEndRate": float(np.mean([x["endReturn"] > 0 for x in rows])), "trailArmRate": float(np.mean([x["trailArm"] for x in rows])), "hardStopRate": float(np.mean([x["hardStop"] for x in rows])), } def cluster(events, gap_minutes=15): groups = [] for event in sorted(events, key=lambda x: (x["pair"], x["side"], pd.Timestamp(x["time"]))): when = pd.Timestamp(event["time"]) if when.tzinfo is None: when = when.tz_localize("UTC") if groups and groups[-1]["pair"] == event["pair"] and groups[-1]["side"] == event["side"] and when - pd.Timestamp(groups[-1]["last"]) <= pd.Timedelta(minutes=gap_minutes): groups[-1]["last"] = iso(when) groups[-1]["events"] += 1 groups[-1]["shadowEligible"] |= event.get("shadowEligible", False) continue groups.append({"pair": event["pair"], "side": event["side"], "time": iso(when), "last": iso(when), "events": 1, "reason": event.get("reason"), "shadowEligible": event.get("shadowEligible", False)}) return groups gate_events = [] for item in evidence["gates"]: details = json.loads(item["details_json"]) gate_events.append({"pair": item["pair"], "side": item["side"], "time": item["occurred_at"], "reason": item["reason"], "shadowEligible": bool(details.get("replacementShadow", {}).get("eligible"))}) gate_clusters = cluster(gate_events) reference_gap_events = [] for row in evidence["candles"]: expected = "enter_long" if row["referenceSide"] == "long" else "enter_short" if (row["referenceKind"] == "reference_signal" and row["referenceAvailable"] is True and row["referencePresent"] is True and row["referenceSide"] in {"long", "short"} and row["strategyAction"] != expected and row["verdict"] == "question" and row["firstDivergence"] in {"signal_detection", "risk_gate"}): reference_gap_events.append({"pair": row["pair"], "side": row["referenceSide"], "time": row["time"], "reason": "reference_strategy_gap", "btcBull1d": row["btcBull1d"]}) reference_gap_clusters = cluster(reference_gap_events) new_ids = sorted(tid for tid,t in trades.items() if START <= pd.Timestamp(t["open_date"], tz="UTC") < END) closed_ids = sorted(tid for tid,t in trades.items() if t["close_date"] and START <= pd.Timestamp(t["close_date"], tz="UTC") < END) carried_ids = sorted(tid for tid,t in trades.items() if pd.Timestamp(t["open_date"], tz="UTC") < START and (t["close_date"] is None or pd.Timestamp(t["close_date"], tz="UTC") >= START)) new_rows = [] for tid in new_ids: trade = trades[tid]; order = entry_order(trade); latest = float(frames[trade["pair"]].iloc[-1].close) side = "short" if trade["is_short"] else "long" gross_mark = ((trade["open_rate"]-latest) if trade["is_short"] else (latest-trade["open_rate"])) * trade["amount"] new_rows.append({"id": tid, "pair": trade["pair"], "side": side, "orderTime": iso(order["order_date"]), "fillTime": iso(order["order_filled_date"]), "fillLatencySeconds": (pd.Timestamp(order["order_filled_date"])-pd.Timestamp(order["order_date"])).total_seconds(), "openRate": trade["open_rate"], "isOpen": bool(trade["is_open"]), "closeTime": iso(trade["close_date"]), "closeRate": trade["close_rate"], "profitRatio": trade["close_profit"], "realizedPnl": trade["close_profit_abs"], "exitReason": trade["exit_reason"], "initialStop": trade["initial_stop_loss"], "latestMark": latest, "grossOpenPnl": gross_mark if trade["is_open"] else None}) cancelled = [{"caseId": f"timeout-{i+1}", "pair": x["pair"], "side": x["side"], "cancelTime": iso(x["time"]), "orderTime": "2026-08-12T08:50:08.724Z" if x["pair"] == "ZEC/USDT:USDT" and x["time"] == "2026-08-12 09:00:11" else iso(pd.Timestamp(x["time"], tz="UTC")-pd.Timedelta(minutes=10)), "price": float(x["rate"])} for i,x in enumerate(evidence["deletedZeroFillEntries"])] replacement_gaps = [] for tid in carried_ids + new_ids: orders = [dict(x) for x in db.execute("SELECT * FROM orders WHERE ft_trade_id=? AND ft_order_side='stoploss' ORDER BY order_date", (tid,))] for before, after in zip(orders, orders[1:]): if before["status"] == "canceled" and before["order_update_date"]: replacement_gaps.append((pd.Timestamp(after["order_date"])-pd.Timestamp(before["order_update_date"])).total_seconds()) paths = {} filled = [{"pair": x["pair"], "side": x["side"], "time": x["orderTime"]} for x in new_rows] for hours in (6,24): paths[f"filled{hours}h"] = aggregate(path_metric(x["pair"],x["time"],x["side"],hours) for x in filled) paths[f"gateBlocked{hours}h"] = aggregate(path_metric(x["pair"],x["time"],x["side"],hours) for x in gate_clusters) paths[f"referenceGap{hours}h"] = aggregate(path_metric(x["pair"],x["time"],x["side"],hours) for x in reference_gap_clusters) paths[f"shadowEligible{hours}h"] = aggregate(path_metric(x["pair"],x["time"],x["side"],hours) for x in gate_clusters if x["shadowEligible"]) paths[f"shadowIneligible{hours}h"] = aggregate(path_metric(x["pair"],x["time"],x["side"],hours) for x in gate_clusters if not x["shadowEligible"]) paths[f"cancelled{hours}h"] = aggregate(path_metric(x["pair"],x["orderTime"],x["side"],hours) for x in cancelled) closed = [trades[x] for x in closed_ids] event_total = sum(x["n"] for x in evidence["events"]) success = sum(x["n"] for x in evidence["events"] if x["status"] == "succeeded") verdicts = Counter(); divergences = Counter(); schemas = Counter() for row in evidence["results"]: verdicts[row["verdict"]] += row["n"] if row["verdict"] == "question": divergences[row["firstDivergence"]] += row["n"] for row in evidence["versions"]: schemas[f"{row['schemaVersion']} / {row['promptVersion']}"] += row["n"] metrics = { "window": evidence["window"], "runtime": {"strategy": "VolatilityBreakoutRiskCap", "actualVersion": "5.4.1-d48-riskcap-observe", "auditorPayloadVersions": dict(Counter(str(x["payloadStrategyVersion"]) for x in evidence["candles"])), "timeframe": "5m", "mode": "dry-run", "pairsObserved": len(frames), "candles": len(frame)}, "startingExposure": [{"id": tid, "pair": trades[tid]["pair"], "side": "short" if trades[tid]["is_short"] else "long", "openDate": iso(trades[tid]["open_date"]), "closeDate": iso(trades[tid]["close_date"])} for tid in carried_ids], "trades": {"newEntries": len(new_ids), "newClosed": sum(not trades[x]["is_open"] for x in new_ids), "newOpen": sum(bool(trades[x]["is_open"]) for x in new_ids), "newRealizedPnl": sum((trades[x]["close_profit_abs"] or 0) for x in new_ids), "newOpenGrossMarkPnl": sum((x["grossOpenPnl"] or 0) for x in new_rows), "closedInWeek": len(closed), "closedPnl": sum(x["close_profit_abs"] for x in closed), "wins": sum(x["close_profit_abs"]>0 for x in closed), "losses": sum(x["close_profit_abs"]<0 for x in closed), "entryAttempts": len(new_ids)+len(cancelled), "entryFills": len(new_ids), "entryFillRate": len(new_ids)/(len(new_ids)+len(cancelled)), "fillLatencyMedianSeconds": float(np.median([x["fillLatencySeconds"] for x in new_rows])), "fillLatencyMaxSeconds": max(x["fillLatencySeconds"] for x in new_rows), "newTradeRows": new_rows, "cancelledEntries": cancelled}, "auditor": {"events": event_total, "successful": success, "coverage": success/event_total, "verdicts": dict(verdicts), "questionFirstDivergence": dict(divergences), "versions": dict(schemas), "deadLetters": event_total-success, "queueErrors": evidence["queueErrors"], "entrySubmissions": len(new_ids)+len(cancelled), "entrySubmissionReviewed": len(new_ids)+len(cancelled), "stopReplacementGapSeconds": {"n":len(replacement_gaps),"median":float(np.median(replacement_gaps)), "p95":float(np.percentile(replacement_gaps,95)),"max":max(replacement_gaps)}}, "cohorts": {"gateRawObservations":len(gate_events),"gateClusters":len(gate_clusters), "shadowEligibleRaw":sum(x["shadowEligible"] for x in gate_events), "shadowEligibleClusters":sum(x["shadowEligible"] for x in gate_clusters), "referenceGapRaw":len(reference_gap_events),"referenceGapClusters":len(reference_gap_clusters), "referenceGapBtcBullFalse":sum(x.get("btcBull1d") is False for x in reference_gap_events), "referenceGapSides":dict(Counter(x["side"] for x in reference_gap_events)),"paths":paths}} (ROOT/"metrics.json").write_text(json.dumps(metrics,ensure_ascii=False,indent=2)) cases=[] for row in new_rows: cases.append({"case_id":f"trade-{row['id']}","pair":row["pair"],"side":row["side"], "reference":{"kind":"reference_signal","present":True,"time":row["orderTime"],"rule_id":"d48-core-breakout-v1","defined_before_outcome":True}, "strategy_signal":{"present":True,"time":row["orderTime"]},"gate":{"status":"passed","reason":None}, "order":{"submitted":True,"time":row["orderTime"],"intended_price":row["openRate"]}, "fill":{"filled":True,"time":row["fillTime"],"price":row["openRate"]}, "exit":{"present":not row["isOpen"],"time":row["closeTime"],"reason":row["exitReason"],"stop_price":row["initialStop"]}, "outcome":{"pnl_ratio":row["profitRatio"],"mfe_ratio":None,"mae_ratio":None},"evidence":[f"sqlite:trades:{row['id']}","auditor:candle_close","auditor:order_created"]}) for item in cancelled: cases.append({"case_id":item["caseId"],"pair":item["pair"],"side":item["side"], "reference":{"kind":"reference_signal","present":True,"time":item["orderTime"],"rule_id":"d48-core-breakout-v1","defined_before_outcome":True}, "strategy_signal":{"present":True,"time":item["orderTime"]},"gate":{"status":"passed","reason":None}, "order":{"submitted":True,"time":item["orderTime"],"intended_price":item["price"]}, "fill":{"filled":False,"time":None,"price":None},"exit":{"present":False,"time":None,"reason":None,"stop_price":None}, "outcome":{"pnl_ratio":None,"mfe_ratio":None,"mae_ratio":None},"evidence":["freqtrade.log:fully_cancelled","auditor:order_created"]}) for i,item in enumerate(gate_clusters,1): cases.append({"case_id":f"gate-{i}","pair":item["pair"],"side":item["side"], "reference":{"kind":"hindsight_annotation","present":False,"time":None}, "strategy_signal":{"present":True,"time":item["time"]},"gate":{"status":"blocked","reason":item["reason"]}, "order":{"submitted":False,"time":None,"intended_price":None},"fill":{"filled":False,"time":None,"price":None}, "exit":{"present":False,"time":None,"reason":None,"stop_price":None},"outcome":{"pnl_ratio":None,"mfe_ratio":None,"mae_ratio":None}, "evidence":[f"gate_observations:{item['time']}"]}) for i,item in enumerate(reference_gap_clusters,1): cases.append({"case_id":f"reference-gap-{i}","pair":item["pair"],"side":item["side"], "reference":{"kind":"reference_signal","present":True,"time":item["time"],"rule_id":"d48-core-breakout-v1","defined_before_outcome":True}, "strategy_signal":{"present":False,"time":None},"gate":{"status":"not_reached","reason":None}, "order":{"submitted":False,"time":None,"intended_price":None},"fill":{"filled":False,"time":None,"price":None}, "exit":{"present":False,"time":None,"reason":None,"stop_price":None},"outcome":{"pnl_ratio":None,"mfe_ratio":None,"mae_ratio":None}, "evidence":[f"auditor:candle_close:{item['time']}"]}) review={"review_id":"2026-08-10-week-to-date-bot1","as_of":iso(END),"strategy":{"bot_id":"bot1","name":"VolatilityBreakoutRiskCap","version":"5.4.1-d48-riskcap-observe"}, "window":{"start":iso(START),"end":iso(END),"timeframe":"5m"},"cases":cases} (ROOT/"review.json").write_text(json.dumps(review,ensure_ascii=False,indent=2)) def candle_plot(ax,data,title): xs=mdates.date2num(data.index.to_pydatetime()); width=(5/1440)*.72 for x,row in zip(xs,data.itertuples()): color="#15803D" if row.close>=row.open else "#B91C1C"; ax.vlines(x,row.low,row.high,color=color,linewidth=.45,alpha=.75) bottom,height=min(row.open,row.close),abs(row.close-row.open) if height: ax.add_patch(Rectangle((x-width/2,bottom),width,height,facecolor=color,edgecolor=color,linewidth=.2,alpha=.7)) ax.grid(color="#E2E8F0",linewidth=.5,alpha=.8); ax.set_title(title,loc="left",fontsize=8,fontweight="bold") ax.xaxis.set_major_formatter(mdates.DateFormatter("%m-%d\n%H:%M",tz=timezone.utc)); ax.tick_params(labelsize=6) def resample15(data): return data.resample("15min",label="left",closed="left").agg({"open":"first","high":"max","low":"min","close":"last","volume":"sum"}).dropna() def overlay_trade(ax,trade,row=None,show_entry=True,show_exit=True): if show_entry and row: ax.scatter(pd.Timestamp(row["orderTime"]),row["openRate"],marker="v" if row["side"]=="short" else "^",s=46,color="#2563EB",edgecolor="white",linewidth=.4,zorder=7,label="strategy signal") ax.scatter(pd.Timestamp(row["fillTime"]),row["openRate"],marker="o",s=17,color="#111827",zorder=8,label="fill") ax.axhline(trade["initial_stop_loss"],color="#DC2626",linewidth=.8,label="initial stop") if show_exit and trade["close_date"]: ax.scatter(pd.Timestamp(trade["close_date"],tz="UTC"),trade["close_rate"],marker="D",s=34,color="#0F766E",edgecolor="white",linewidth=.4,zorder=8,label="exit") plot_rows=[] for tid in carried_ids: plot_rows.append((f"carried-in {trades[tid]['pair'].split('/')[0]} #{tid}",trades[tid],None)) for row in new_rows: plot_rows.append((f"new {row['pair'].split('/')[0]} #{row['id']}",trades[row["id"]],row)) fig,axes=plt.subplots(8,2,figsize=(17,29),constrained_layout=True) for idx,(label,trade,row) in enumerate(plot_rows): data=frames[trade["pair"]] if row: anchor=pd.Timestamp(row["orderTime"]); end=pd.Timestamp(row["closeTime"]) if row["closeTime"] else END left=data[(data.index>=anchor-pd.Timedelta(hours=3))&(data.index<=anchor+pd.Timedelta(hours=8))] right=resample15(data[(data.index>=anchor-pd.Timedelta(hours=3))&(data.index<=end+pd.Timedelta(hours=2))]) candle_plot(axes[idx,0],left,label+" · 5m execution"); candle_plot(axes[idx,1],right,label+" · 15m structure") overlay_trade(axes[idx,0],trade,row,show_exit=bool(row["closeTime"] and pd.Timestamp(row["closeTime"])<=left.index.max())) overlay_trade(axes[idx,1],trade,row) else: exit_time=pd.Timestamp(trade["close_date"],tz="UTC") if trade["close_date"] else END left=data[(data.index>=exit_time-pd.Timedelta(hours=8))&(data.index<=exit_time+pd.Timedelta(hours=2))] right=resample15(data) candle_plot(axes[idx,0],left,label+" · 5m exit context"); candle_plot(axes[idx,1],right,label+" · 15m week path") overlay_trade(axes[idx,0],trade,None,False,True); overlay_trade(axes[idx,1],trade,None,False,True) cancel=cancelled[0]; data=frames[cancel["pair"]]; submitted=pd.Timestamp(cancel["orderTime"]); canceled=pd.Timestamp(cancel["cancelTime"]) left=data[(data.index>=submitted-pd.Timedelta(hours=3))&(data.index<=submitted+pd.Timedelta(hours=8))]; right=resample15(data[(data.index>=submitted-pd.Timedelta(hours=3))&(data.index<=submitted+pd.Timedelta(hours=18))]) candle_plot(axes[6,0],left,"ZEC zero-fill · 5m"); candle_plot(axes[6,1],right,"ZEC zero-fill · 15m hindsight") for ax in axes[6]: ax.scatter(submitted,cancel["price"],marker="v",s=46,color="#2563EB",zorder=7,label="strategy signal") ax.hlines(cancel["price"],submitted,canceled,color="#D97706",linestyle="--",linewidth=1.4,label="pending order") ax.scatter(canceled,cancel["price"],marker="x",s=44,color="#D97706",zorder=8,label="zero-fill cancel") best=None for item in gate_clusters: metric=path_metric(item["pair"],item["time"],item["side"],6) if metric and (best is None or metric["mfe"]>best[1]["mfe"]): best=(item,metric) item,_=best; when=pd.Timestamp(item["time"]); data=frames[item["pair"]]; price=float(data[data.index>=when.floor("5min")].iloc[0].close) left=data[(data.index>=when-pd.Timedelta(hours=3))&(data.index<=when+pd.Timedelta(hours=8))]; right=resample15(data[(data.index>=when-pd.Timedelta(hours=3))&(data.index<=when+pd.Timedelta(hours=18))]) candle_plot(axes[7,0],left,f"best blocked {item['pair']} · 5m"); candle_plot(axes[7,1],right,f"best blocked {item['pair']} · 15m hindsight") for ax in axes[7]: ax.scatter(when,price,marker="x",s=48,color="#64748B",zorder=8,label="risk block") for row_axes in axes: for ax in row_axes: h,l=ax.get_legend_handles_labels() if h: unique=dict(zip(l,h)); ax.legend(unique.values(),unique.keys(),fontsize=5.5,loc="best") fig.suptitle("Bot1 week-to-date causal review · authentic Freqtrade analyzed candles",fontsize=14,fontweight="bold") fig.savefig(ROOT/"layered-week.png",dpi=155,facecolor="white"); plt.close(fig) print(json.dumps(metrics,ensure_ascii=False,indent=2))