#!/usr/bin/env python3
"""
Backtest SPY Strategy V2 — Calibré sur les vrais trades
Corrige le biais de granularité 7s → 1min en ajustant les seuils d'entrée.

Logique: un +1% en 7 secondes sur le vrai spy = une bougie 1min 
avec un corps très fort (≥2%), volume extrême, et dominance acheteur.
"""
import json
import os
import time
import math
from datetime import datetime, timedelta
from collections import defaultdict

# ═══════════════════════════════════════════════════════════════
# PARAMÈTRES CALIBRÉS (ajustés pour bougies 1-minute)
# ═══════════════════════════════════════════════════════════════
POSITION_SIZE = 500
MAX_POSITIONS = 3
MAX_TRADES_PER_HOUR = 6

# ENTRY — seuils relevés pour compenser la granularité 1min  
# 1% en 7s ≈ un mouvement exceptionnel sur une bougie 1min
SURGE_BODY_MIN_PCT = 1.8       # Corps de bougie minimum (close vs open)
SURGE_MOVE_MIN_PCT = 2.0       # Move minimum (close vs prev close)
SURGE_VOL_RATIO = 4.0          # Volume 4x la moyenne (vs 3x en 7s)
SURGE_BODY_DOMINANCE = 0.55    # Corps ≥ 55% de la range totale (pas de grandes mèches)
SURGE_TAKER_BUY_MIN = 0.55     # Ratio acheteurs minimum
MIN_GREEN_CANDLES_3 = 2        # Au moins 2/3 dernières bougies vertes
CONTEXT_LOOKBACK = 5           # Vérifier le contexte sur 5 bougies
COOLDOWN_CANDLES = 5           # Attendre 5 min entre trades sur même symbole

# EXIT — identiques au vrai spy
HARD_SL_PCT = -1.2
TRAILING_ACTIVATION = 1.0
MAX_HOLD_MINUTES = 15
STAGNATION_MINUTES = 10
STAGNATION_THRESHOLD = 0.5
MOMENTUM_EXIT_DROPS = 3
VOLUME_EXIT_SELL_RATIO = 0.38
EARLY_SL_DELAY_MIN = 1
EARLY_SL_MAX_PEAK = 0.2
EARLY_SL_PCT = -0.5
INSTANT_REVERSAL_MIN = 1      # 1 bougie ≈ ~45s
INSTANT_REVERSAL_PCT = -0.6

TRAIL_TIERS = [
    (0.5, 0.3), (1.0, 0.7), (2.0, 1.0), (3.0, 1.8),
    (5.0, 2.0), (10.0, 2.5), (15.0, 4.0), (999, 5.0)
]

CACHE_DIR = "backtest_cache"

def load_klines(symbol, days=90):
    """Charge les klines depuis le cache"""
    cache_file = os.path.join(CACHE_DIR, f"{symbol}_1m_{days}d.json")
    if not os.path.exists(cache_file):
        return None
    with open(cache_file) as f:
        return json.load(f)


class SpyBacktestV2:
    def __init__(self):
        self.trades = []
        self.hourly_trade_count = defaultdict(int)
        
    def get_trail_pct(self, pnl_pct):
        for threshold, trail in TRAIL_TIERS:
            if pnl_pct < threshold:
                return trail
        return 5.0
    
    def compute_rsi(self, klines, idx, period=7):
        """RSI rapide sur 7 périodes"""
        if idx < period:
            return 50
        gains, losses = 0, 0
        for i in range(idx - period + 1, idx + 1):
            change = klines[i]['close'] - klines[i-1]['close']
            if change > 0:
                gains += change
            else:
                losses -= change
        avg_gain = gains / period
        avg_loss = losses / period
        if avg_loss == 0:
            return 100
        rs = avg_gain / avg_loss
        return 100 - (100 / (1 + rs))
    
    def detect_surge(self, klines, idx):
        """Détecte un surge avec des critères calibrés pour 1min"""
        if idx < 30:
            return None
        
        curr = klines[idx]
        prev = klines[idx - 1]
        
        # --- Corps de la bougie ---
        body_pct = ((curr['close'] - curr['open']) / curr['open']) * 100
        if body_pct < SURGE_BODY_MIN_PCT:
            return None
        
        # --- Move vs previous close ---
        move_pct = ((curr['close'] - prev['close']) / prev['close']) * 100
        if move_pct < SURGE_MOVE_MIN_PCT:
            return None
        
        # --- Body dominance (pas de grandes mèches) ---
        total_range = curr['high'] - curr['low']
        if total_range == 0:
            return None
        body = abs(curr['close'] - curr['open'])
        dominance = body / total_range
        if dominance < SURGE_BODY_DOMINANCE:
            return None
        
        # --- Volume spike ---
        avg_vol = sum(k['quote_volume'] for k in klines[idx-20:idx]) / 20
        if avg_vol == 0:
            return None
        vol_ratio = curr['quote_volume'] / avg_vol
        if vol_ratio < SURGE_VOL_RATIO:
            return None
        
        # --- Taker buy pressure ---
        if curr['quote_volume'] > 0:
            buy_ratio = curr['taker_buy_quote'] / curr['quote_volume']
            if buy_ratio < SURGE_TAKER_BUY_MIN:
                return None
        
        # --- Bougies vertes (contexte) ---
        last3 = klines[idx-2:idx+1]
        green = sum(1 for k in last3 if k['close'] > k['open'])
        if green < MIN_GREEN_CANDLES_3:
            return None
        
        # --- Pas de dump récent (filtre rebond) ---
        # Si les 5 bougies précédentes ont chuté > 2%, c'est un rebond — pas un vrai surge
        price_5m_ago = klines[idx - 5]['close']
        context_change = ((klines[idx-1]['close'] - price_5m_ago) / price_5m_ago) * 100
        if context_change < -2.0:
            return None  # Rebond après dump — éviter
        
        # --- RSI pas trop haut (éviter surachat) ---
        rsi = self.compute_rsi(klines, idx)
        if rsi > 92:
            return None  # Trop suracheté
        
        # --- Pas de wick rejet supérieur ---
        upper_wick = curr['high'] - curr['close']
        if body > 0 and upper_wick > 1.5 * body:
            return None
        
        # --- Change 24h (anti-pump/dump) ---
        if idx >= 1440:
            price_24h = klines[idx - 1440]['close']
            change_24h = ((curr['close'] - price_24h) / price_24h) * 100
            if change_24h > 50 or change_24h < -15:
                return None
        
        return {
            'type': 'FLASH_SURGE' if move_pct >= 2.5 else 'BREAKOUT_SURGE',
            'strength': move_pct,
            'vol_ratio': vol_ratio,
            'body_pct': body_pct,
            'dominance': dominance,
            'buy_ratio': buy_ratio if curr['quote_volume'] > 0 else 0,
            'rsi': rsi,
            'price': curr['close'],
            'ts': curr['ts']
        }
    
    def simulate_exit(self, klines, entry_idx, entry_price, surge_strength):
        """Simule la sortie minute par minute"""
        max_price = entry_price
        max_pnl_pct = 0
        trailing_active = False
        trailing_stop_price = 0
        consecutive_reds = 0
        
        for i in range(entry_idx + 1, min(entry_idx + MAX_HOLD_MINUTES + 2, len(klines))):
            candle = klines[i]
            hold_min = i - entry_idx
            
            # Simuler les prix intra-bougie (open, high, low, close)
            # Ordre réaliste: open → high/low → close
            if candle['close'] >= candle['open']:  # Bougie verte
                tick_sequence = [candle['open'], candle['low'], candle['high'], candle['close']]
            else:  # Bougie rouge
                tick_sequence = [candle['open'], candle['high'], candle['low'], candle['close']]
            
            for tick_price in tick_sequence:
                pnl_pct = ((tick_price - entry_price) / entry_price) * 100
                
                if tick_price > max_price:
                    max_price = tick_price
                    max_pnl_pct = ((max_price - entry_price) / entry_price) * 100
                
                # INSTANT REVERSAL (1ère bougie)
                if hold_min <= INSTANT_REVERSAL_MIN and max_pnl_pct < 0.05 and pnl_pct <= INSTANT_REVERSAL_PCT:
                    return tick_price, i, 'INSTANT_REVERSAL', pnl_pct
                
                # EARLY SL
                if hold_min >= EARLY_SL_DELAY_MIN and max_pnl_pct < EARLY_SL_MAX_PEAK:
                    early_sl = max(EARLY_SL_PCT, -surge_strength * 0.8)
                    early_sl = max(early_sl, HARD_SL_PCT * 0.9)
                    if pnl_pct <= early_sl:
                        return tick_price, i, 'EARLY_SL', pnl_pct
                
                # HARD SL
                if pnl_pct <= HARD_SL_PCT:
                    return tick_price, i, 'HARD_SL', pnl_pct
                
                # TRAILING STOP
                if max_pnl_pct >= TRAILING_ACTIVATION:
                    trailing_active = True
                    trail_pct = self.get_trail_pct(max_pnl_pct)
                    new_ts = max_price * (1 - trail_pct / 100)
                    trailing_stop_price = max(trailing_stop_price, new_ts)
                    
                    if tick_price <= trailing_stop_price:
                        return tick_price, i, 'TRAILING', pnl_pct
            
            # Fin de bougie — checks additionels
            current_price = candle['close']
            pnl_pct = ((current_price - entry_price) / entry_price) * 100
            
            # Compteur bougies rouges consécutives
            if candle['close'] < candle['open']:
                consecutive_reds += 1
            else:
                consecutive_reds = 0
            
            # MOMENTUM EXIT
            if consecutive_reds >= MOMENTUM_EXIT_DROPS and pnl_pct < 5.0:
                return current_price, i, 'MOMENTUM_EXIT', pnl_pct
            
            # STAGNATION
            if hold_min >= STAGNATION_MINUTES and abs(pnl_pct) < STAGNATION_THRESHOLD:
                return current_price, i, 'STAGNATION', pnl_pct
            
            # REVERSAL
            if max_pnl_pct >= 3.0 and pnl_pct < 1.0 and max_pnl_pct < 8.0:
                return current_price, i, 'REVERSAL', pnl_pct
            
            # VOLUME ROUGE
            if hold_min >= 2 and candle['quote_volume'] > 0:
                buy_ratio = candle['taker_buy_quote'] / candle['quote_volume']
                if buy_ratio < VOLUME_EXIT_SELL_RATIO and pnl_pct < 1.0:
                    return current_price, i, 'VOLUME_ROUGE', pnl_pct
            
            # MAX HOLD
            if hold_min >= MAX_HOLD_MINUTES and pnl_pct < 3.0:
                return current_price, i, 'MAX_HOLD', pnl_pct
        
        # Fallback
        last = klines[min(entry_idx + MAX_HOLD_MINUTES, len(klines)-1)]
        pnl_pct = ((last['close'] - entry_price) / entry_price) * 100
        return last['close'], min(entry_idx + MAX_HOLD_MINUTES, len(klines)-1), 'TIMEOUT', pnl_pct
    
    def run(self, symbol, klines):
        """Backtest sur un symbole"""
        trades = []
        last_trade_idx = 0
        hourly_counts = defaultdict(int)
        
        for idx in range(30, len(klines) - MAX_HOLD_MINUTES - 5):
            if idx < last_trade_idx + COOLDOWN_CANDLES:
                continue
            
            # Limite horaire
            hour_key = klines[idx]['ts'] // 3600000
            if hourly_counts[hour_key] >= MAX_TRADES_PER_HOUR:
                continue
            
            surge = self.detect_surge(klines, idx)
            if not surge:
                continue
            
            entry_price = surge['price']
            exit_price, exit_idx, reason, pnl_pct = self.simulate_exit(
                klines, idx, entry_price, surge['strength']
            )
            
            pnl_usdt = (pnl_pct / 100) * POSITION_SIZE
            
            entry_time = datetime.fromtimestamp(klines[idx]['ts'] / 1000).isoformat()
            exit_time = datetime.fromtimestamp(klines[exit_idx]['ts'] / 1000).isoformat()
            
            trade = {
                'symbol': symbol,
                'entry_time': entry_time,
                'exit_time': exit_time,
                'entry_price': entry_price,
                'exit_price': exit_price,
                'pnl_pct': round(pnl_pct, 2),
                'pnl_usdt': round(pnl_usdt, 2),
                'reason': reason,
                'surge_type': surge['type'],
                'surge_strength': round(surge['strength'], 2),
                'vol_ratio': round(surge['vol_ratio'], 1),
                'hold_minutes': exit_idx - idx
            }
            trades.append(trade)
            last_trade_idx = exit_idx
            hourly_counts[hour_key] += 1
        
        return trades


def main():
    SYMBOLS = [
        'ONTUSDT', 'CHRUSDT', 'NOMUSDT', 'FIDAUSDT', 'STOUSDT',
        'HEMIUSDT', 'ZBTUSDT', 'KERNELUSDT', 'CTSIUSDT', 'XPLUSDT',
        'DUSDT', 'BLURUSDT', 'SEIUSDT', 'ENJUSDT', 'ANKRUSDT',
        'THETAUSDT', 'COSUSDT', 'GASUSDT', 'WIFUSDT'
    ]
    
    DAYS = 90
    
    print(f"╔══════════════════════════════════════════════════════════════╗")
    print(f"║  🔍 BACKTEST SPY V2 (CALIBRÉ) — {DAYS}j — {len(SYMBOLS)} symboles        ║")
    print(f"╚══════════════════════════════════════════════════════════════╝")
    print(f"  Seuils calibrés: body≥{SURGE_BODY_MIN_PCT}%, move≥{SURGE_MOVE_MIN_PCT}%,")
    print(f"  vol≥{SURGE_VOL_RATIO}x, dominance≥{SURGE_BODY_DOMINANCE}, taker≥{SURGE_TAKER_BUY_MIN}")
    print()
    
    # Charger les données depuis le cache
    print("📥 Chargement des données (cache)...")
    all_data = {}
    for sym in SYMBOLS:
        data = load_klines(sym, DAYS)
        if data and len(data) > 1000:
            all_data[sym] = data
            print(f"  ✅ {sym}: {len(data)} klines")
        else:
            print(f"  ⚠️ {sym}: pas en cache")
    
    print(f"\n✅ {len(all_data)}/{len(SYMBOLS)} symboles chargés")
    print()
    
    # Backtest
    print("🧪 Simulation V2...")
    bt = SpyBacktestV2()
    all_trades = []
    
    for sym, klines in all_data.items():
        trades = bt.run(sym, klines)
        all_trades.extend(trades)
        
        if trades:
            sym_pnl = sum(t['pnl_usdt'] for t in trades)
            wins = sum(1 for t in trades if t['pnl_usdt'] > 0)
            losses = sum(1 for t in trades if t['pnl_usdt'] <= 0)
            wr = (wins / (wins + losses) * 100) if (wins + losses) > 0 else 0
            print(f"  {sym:16} | {len(trades):4} trades | WR={wr:5.1f}% | P&L={sym_pnl:+8.1f} USDT")
        else:
            print(f"  {sym:16} |    0 trades")
    
    # Résultats
    print()
    print(f"╔══════════════════════════════════════════════════════════════╗")
    print(f"║  📊 RÉSULTATS BACKTEST SPY V2 — {DAYS} JOURS                      ║")
    print(f"╚══════════════════════════════════════════════════════════════╝")
    
    if not all_trades:
        print("  ❌ Aucun trade détecté ! Seuils trop stricts ?")
        return
    
    total_pnl = sum(t['pnl_usdt'] for t in all_trades)
    wins = [t for t in all_trades if t['pnl_usdt'] > 0]
    losses = [t for t in all_trades if t['pnl_usdt'] <= 0]
    win_rate = len(wins) / len(all_trades) * 100
    
    avg_win = sum(t['pnl_usdt'] for t in wins) / len(wins) if wins else 0
    avg_loss = sum(t['pnl_usdt'] for t in losses) / len(losses) if losses else 0
    total_wins = sum(t['pnl_usdt'] for t in wins)
    total_losses = sum(t['pnl_usdt'] for t in losses)
    pf = abs(total_wins / total_losses) if total_losses != 0 else 999
    
    # Max drawdown
    running = 0
    peak = 0
    max_dd = 0
    equity_curve = []
    for t in sorted(all_trades, key=lambda x: x['exit_time']):
        running += t['pnl_usdt']
        peak = max(peak, running)
        dd = running - peak
        max_dd = min(max_dd, dd)
        equity_curve.append(running)
    
    avg_hold = sum(t['hold_minutes'] for t in all_trades) / len(all_trades)
    trades_per_day = len(all_trades) / DAYS
    pnl_per_day = total_pnl / DAYS
    
    print()
    print(f"  📈 Total trades:     {len(all_trades)}")
    print(f"  ✅ Gagnants:         {len(wins)} ({win_rate:.1f}%)")
    print(f"  ❌ Perdants:         {len(losses)}")
    print(f"  💰 P&L TOTAL:        {total_pnl:+.2f} USDT")
    print(f"  📊 P&L moyen/trade:  {total_pnl/len(all_trades):+.2f} USDT")
    print(f"  🟢 Gain moyen:       {avg_win:+.2f} USDT")
    print(f"  🔴 Perte moyenne:    {avg_loss:+.2f} USDT")
    print(f"  📐 Profit Factor:    {pf:.2f}")
    print(f"  📉 Max Drawdown:     {max_dd:+.2f} USDT")
    print(f"  📅 Trades/jour:      {trades_per_day:.1f}")
    print(f"  📅 P&L moyen/jour:   {pnl_per_day:+.2f} USDT")
    print(f"  📅 P&L moyen/mois:   {pnl_per_day * 30:+.2f} USDT")
    print(f"  ⏱️  Hold moyen:       {avg_hold:.1f} min")
    
    # Par mois
    by_month = defaultdict(list)
    for t in all_trades:
        month = t['exit_time'][:7]
        by_month[month].append(t)
    
    print()
    print("  ═══ PAR MOIS ═══")
    for month in sorted(by_month.keys()):
        trades_m = by_month[month]
        pnl_m = sum(t['pnl_usdt'] for t in trades_m)
        w = sum(1 for t in trades_m if t['pnl_usdt'] > 0)
        wr_m = (w / len(trades_m) * 100) if trades_m else 0
        print(f"  {month}: {len(trades_m):4} trades | WR={wr_m:5.1f}% | P&L={pnl_m:+8.1f} USDT")
    
    # Par raison
    by_reason = defaultdict(list)
    for t in all_trades:
        by_reason[t['reason']].append(t)
    
    print()
    print("  ═══ PAR RAISON DE SORTIE ═══")
    for reason in sorted(by_reason.keys(), key=lambda r: sum(t['pnl_usdt'] for t in by_reason[r]), reverse=True):
        trades_r = by_reason[reason]
        pnl_r = sum(t['pnl_usdt'] for t in trades_r)
        w = sum(1 for t in trades_r if t['pnl_usdt'] > 0)
        wr_r = (w / len(trades_r) * 100) if trades_r else 0
        print(f"  {reason:20} | {len(trades_r):4} trades | WR={wr_r:5.1f}% | P&L={pnl_r:+8.1f} USDT | avg={pnl_r/len(trades_r):+.2f}")
    
    # Par surge type
    by_surge = defaultdict(list)
    for t in all_trades:
        by_surge[t['surge_type']].append(t)
    
    print()
    print("  ═══ PAR TYPE DE SURGE ═══")
    for stype in sorted(by_surge.keys(), key=lambda s: sum(t['pnl_usdt'] for t in by_surge[s]), reverse=True):
        trades_s = by_surge[stype]
        pnl_s = sum(t['pnl_usdt'] for t in trades_s)
        w = sum(1 for t in trades_s if t['pnl_usdt'] > 0)
        wr_s = (w / len(trades_s) * 100) if trades_s else 0
        print(f"  {stype:18} | {len(trades_s):4} trades | WR={wr_s:5.1f}% | P&L={pnl_s:+8.1f} USDT")
    
    # Top 10 trades
    print()
    print("  ═══ TOP 10 MEILLEURS TRADES ═══")
    top10 = sorted(all_trades, key=lambda t: t['pnl_usdt'], reverse=True)[:10]
    for t in top10:
        print(f"  {t['symbol']:12} | {t['entry_time'][:16]} | surge={t['surge_strength']:+.1f}% | P&L={t['pnl_usdt']:+.1f} | {t['reason']}")
    
    # Top 10 pires
    print()
    print("  ═══ TOP 10 PIRES TRADES ═══")
    worst10 = sorted(all_trades, key=lambda t: t['pnl_usdt'])[:10]
    for t in worst10:
        print(f"  {t['symbol']:12} | {t['entry_time'][:16]} | surge={t['surge_strength']:+.1f}% | P&L={t['pnl_usdt']:+.1f} | {t['reason']}")
    
    # Sauvegarder
    results = {
        'version': 'v2_calibrated',
        'period_days': DAYS,
        'symbols': len(all_data),
        'total_trades': len(all_trades),
        'total_pnl': round(total_pnl, 2),
        'win_rate': round(win_rate, 1),
        'profit_factor': round(pf, 2),
        'max_drawdown': round(max_dd, 2),
        'avg_pnl_per_day': round(pnl_per_day, 2),
        'avg_pnl_per_month': round(pnl_per_day * 30, 2),
        'avg_hold_min': round(avg_hold, 1),
        'trades_per_day': round(trades_per_day, 1),
        'entry_thresholds': {
            'body_min_pct': SURGE_BODY_MIN_PCT,
            'move_min_pct': SURGE_MOVE_MIN_PCT,
            'vol_ratio': SURGE_VOL_RATIO,
            'body_dominance': SURGE_BODY_DOMINANCE,
            'taker_buy_min': SURGE_TAKER_BUY_MIN,
        },
        'trades': all_trades
    }
    with open('backtest_spy_v2_results.json', 'w') as f:
        json.dump(results, f, indent=2)
    print(f"\n  💾 Résultats sauvegardés dans backtest_spy_v2_results.json")
    
    # Comparaison
    print()
    print("  ═══════════════════════════════════════════════════════════")
    print("  COMPARAISON AVEC TESTNET RÉEL")
    print("  ═══════════════════════════════════════════════════════════")
    tn_trades_day = 200 / 3.67
    tn_pnl_day = 954 / 3.67
    tn_wr = 59
    
    print(f"  {'Métrique':25} | {'Testnet (3.5j)':>15} | {'Backtest ({DAYS}j)'}")
    print(f"  {'-'*25}-+-{'-'*15}-+-{'-'*15}")
    print(f"  {'Trades/jour':25} | {tn_trades_day:>12.0f}    | {trades_per_day:>12.1f}")
    print(f"  {'Win Rate':25} | {tn_wr:>12}%   | {win_rate:>12.1f}%")
    print(f"  {'P&L/jour':25} | {tn_pnl_day:>+12.0f}    | {pnl_per_day:>+12.1f}")
    print(f"  {'P&L/mois':25} | {tn_pnl_day*30:>+12.0f}    | {pnl_per_day*30:>+12.1f}")
    
    print()
    if total_pnl > 0 and win_rate > 40:
        print("  ✅ STRATÉGIE VALIDÉE — Performance positive confirmée sur 3 mois")
    elif total_pnl > 0:
        print("  ⚠️ POSITIF MAIS FRAGILE — Win rate faible, dépend de gros gains")
    elif total_pnl > -500:
        print("  ⚠️ QUASI-FLAT — La stratégie est neutre sur cette période")
    else:
        print("  🔴 NÉGATIF — La stratégie perd de l'argent sur cette période")
    
    print()
    print("  ⚠️ LIMITES DU BACKTEST:")
    print("  - Résolution 1min vs 7sec réel → timing d'entrée moins précis")
    print("  - Pas de simulation du carnet d'ordres (slippage réel)")
    print("  - Pas de coin scoring/circuit breaker/AI predictor")
    print("  - Nombre de symboles limité (19 vs ~200+ scannés en réel)")
    print("  - Résultats à considérer comme borne BASSE conservative")


if __name__ == '__main__':
    main()