
    i2              
          d Z ddlZddlZddlmZ dej                  dedej                  fdZ	ddej                  dedej                  fdZ
ddej                  ded	efd
Z	 ddej                  dededee   fdZdej                  dedefdZdej                  dedefdZ	 ddee   dededej                  fdZg dZg dZy)u   
SPY Optimizer — Feature Engineering
Calcule les features techniques à partir des klines 1m pour chaque moment de trade.
Utilisé à la fois pour l'entraînement (trades historiques) et l'inférence (signal en temps réel).
    N)Optionalseriesperiodreturnc                 D    | j                  |d      j                         S )NFspanadjust)ewmmean)r   r   s     D/home/ubuntu/crypto_trading_bot/spy_optimizer/feature_engineering.pyemar      s    ::6%:05577    c                 L   | j                         }|j                  d      }|j                  d       }|j                  |d      j                         }|j                  |d      j                         }||j	                  dt
        j                        z  }ddd|z   z  z
  S )Nr   )lower)upperFr   d      )diffclipr   r   replacenpnan)r   r   deltagainlossavg_gainavg_lossrss           r   rsir       s    KKME::A:DZZaZ  DxxVEx2779HxxVEx2779H	H$$Q/	/B#R.!!r   stdc                     | j                  |      j                         }| j                  |      j                         }||||z  z   |||z  z
  fS )N)rollingr   r!   )r   r   r!   midss        r   bollinger_bandsr&      sM    
..
 
%
%
'Cv""$AcAgsS1W},,r   dftimestamp_mslookback_minutesc                   & ||dz  dz  z
  }| d   |k\  | d   |k  z  }| j                   |   j                         }t        |      dk  ry|d   }|d   }|d   }|d	   }	|d
   }
|d   }|d   }i }|j                  d   }||d<   t	        |d      |d<   t	        |d      |d<   t	        |d      |d<   t	        |d      |d<   t	        |d      |d<   t	        |d      |d<   t	        |d      |d<   |j                         j                         }t        |      dk\  r"|j                  d      j                         dz  nd|d<   t        |      dk\  r"|j                  d      j                         dz  nd|d<   t        |      dk\  r"|j                  d      j                         dz  nd|d<   t        |d       }t        |d!      }t        |d"      }|j                  d   |d#<   |j                  d   |d$<   |j                  d   dkD  r||j                  d   z  dz
  dz  nd|d%<   |j                  d   dkD  r||j                  d   z  dz
  dz  nd|d&<   |j                  d   dkD  r%|j                  d   |j                  d   z  dz
  dz  nd|d'<   t        |d      |d(<   t        |d      |d)<   t        |d      |d*<   ||z
  |z  dz  j                         }t        |      dkD  r|j                  d   nd|d+<   t        |      d,k\  r|j                  d-   n|d+   |d.<   t        |d+         t        |d.         k  rdnd|d/<   t        |d0      }t        j                  |j                  d         s|j                  d   nd"|d1<   t        |      d,k\  r|j                  d-   n|d1   |d2<   |d1   |d2   z
  |d3<   |d1   dk  rdnd|d4<   |d1   d5kD  rdnd|d6<   t        |d7      \  }}}||z
  |z  dz  j                         }t        |      dkD  r|j                  d   nd|d8<   |j                  d   |j                  d   z
  dkD  r2||j                  d   z
  |j                  d   |j                  d   z
  z  nd9|d:<   |d8   |j                  d      j!                  d;      k  rdnd|d<<   |
j                  d      j#                         }|
j                  d      j#                         }|dkD  r|
j                  d   |z  nd|d=<   |dkD  r||z  nd|d><   |d=   d?kD  rdnd|d@<   t        |
d      |dA<   |
j                  d      j%                         }|j                  d      j%                         }|dkD  r||z  dz  nd"|dB<   |
j                  d   dkD  r"|j                  d   |
j                  d   z  dz  nd"|dC<   ||dD   z
  }||z
  }|j                  d      dkD  j%                         dz  |dE<   |j                  d      dkD  j%                         dz  |dF<   t        |j                  d            |j                  d      j'                  dt        j(                        z  j#                         |dG<   |t+        j,                  ||dD   gdH      j/                  dH      z
  }|j                  d   dkD  r|j                  d   |j                  d   z  nd|dI<   t	        |d      |dJ<   t	        |d      |dK<   t	        |d      |dL<   |j                  d      j0                  &t3        &fdMt5        t        &      dz
        D              rdnd|dN<   |j                  d      j/                         } |j                  d      j7                         }!| |!z
  dkD  r||!z
  | |!z
  z  nd9|dO<   t        |      dk\  r|j                  d      j/                         n|j/                         }"t        |      dk\  r|j                  d      j7                         n|j7                         }#|"|#z
  dkD  r||#z
  |"|#z
  z  nd9|dP<   t        |      dQk\  r|j                  dRd j/                         n|j/                         }$|$dkD  r||$z  dz
  dz  nd|dS<   |j                  d      j#                         |dT<   |j                  d      j#                         dkD  r/|j                  d   |j                  d      j#                         z  nd|dU<   t+        j8                  |dVdWX      j:                  }%|%|dY<   d|%cxk  rdZk  rn ndnd|d[<   d |%cxk  rdQk  rn ndnd|d\<   d]|%cxk  rd^k  rn ndnd|d_<   |S )`u  
    Calcule ~50 features techniques à partir des klines 1m
    pour un moment précis (timestamp d'entrée du trade).

    Args:
        df: DataFrame klines avec colonnes standard (open_time, open, high, low, close, volume, ...)
        timestamp_ms: timestamp en millisecondes du moment d'analyse
        lookback_minutes: nombre de minutes de données avant le timestamp

    Returns:
        dict de features ou None si données insuffisantes
    <     	open_time   Nclosehighlowvolumequote_volume
num_tradestaker_buy_quote_volpricer   	return_1m   	return_3m   	return_5m
   
return_10m   
return_15m
return_30m
return_60mr   r   volatility_5mvolatility_15mvolatility_30m      2   ema7ema21price_vs_ema7price_vs_ema21ema7_vs_ema21ema7_slope_3mema7_slope_5mema21_slope_5mema_gap_current   ema_gap_3m_agoema_converging   rsi_14
rsi_3m_agorsi_delta_3mrsi_oversoldF   rsi_overbought   bb_widthg      ?bb_positiong?
bb_squeezevolume_ratio_1mvolume_ratio_3mg      @volume_spikevolume_trend_5mbuy_pressure_5mbuy_pressure_1mopengreen_candles_5mgreen_candles_3mavg_body_ratio_5m)axisupper_wick_ratiomomentum_3mmomentum_5mmomentum_10mc              3   :   K   | ]  }|   |d z      k    yw)r   N ).0irecent_5s     r   	<genexpr>z0compute_features_at_timestamp.<locals>.<genexpr>   s"     *hA8A;(1Q3-+G*hs   monotonic_up_5mprice_position_30mprice_position_60m   ibreakout_pctavg_trades_per_min_5mtrade_intensity_ratiomsUTC)unittzhour_utc   is_asia_sessionis_europe_session      is_us_session)loccopyleniloc_pct
pct_changedropnatailr!   r   _slopeabsr    r   isnanr&   quantiler   sumr   r   pdconcatmaxvaluesallrangemin	Timestamphour)'r'   r(   r)   start_msmaskwindowr/   r0   r1   r2   	quote_volr4   taker_buy_qvolfeatures
last_pricereturnsrI   rJ   ema50ema_gaprsi_valsbb_midbb_upperbb_lowerr^   avg_vol_30m
avg_vol_5mrecent_qvolrecent_buy_qvolbodycandle_range
upper_wickhigh_30mlow_30mhigh_60mlow_60mprev_high_15mr   rt   s'                                         @r   compute_features_at_timestampr      s	   $ /"4t;<H{Ox'B{O|,KLDVVD\ F
6{R7OE&>D
-CHF~&I%J12NH BJ"HW NH[ NH[ NH[!%_H\!%_H\!%_H\!%_H\  '')G?B7|q?PQ 3 3 5 ;VWH_ADWQSASb!1!5!5!7#!=YZHADWQSASb!1!5!5!7#!=YZH ua=DrNErNEyy}HV

2HWJN))TV-Z[J[diim!;a!?3 FabH_LQJJWYN]^L^*uzz"~"="AS!HdeHNSjjY[n_`N`2B!?!!Cs JfgH_ !'tQH_ &tQH_!'q!1H u%+335G69'lQ6F',,r"2AH58\Q5Fb!1HUfLgH&)(3D*E&FXVfMgIh&hnoH 5"~H24((8==;L2Mr*SUHX25h-12DX]]2.(S[J\H\'1H\4JJH^$,X$6$;qH^&.x&82&=1H "1!;FHhH$.4<<>H03H0A8==,qHZ#+==#4x}}R7H#HA"M !+X]]2-> >8==QSCTW_WdWdegWhChiSV ]"*:"6r9J9S9STW9X"XQ^_H\ ..$))+K"'')JFQTUo).."4{"B[\H>IAo*{":STH$,->$?#$Eq1H^"(A"6H ..#'')K$))!,002OITWX/K"?#"E^`H&/nnR&81&< $2#6#6r#:Y^^B=O#ORU#UBD  6&>!D#:L$(IIaL1$4#9#9#;a#?H $(IIaL1$4#9#9#;a#?H %(1%69J9J19M9U9UVWY[Y_Y_9`%`$f$f$hH ! 		5&."9BFFAFNNJ'3'8'8'<q'@ %/OOB$7,:K:KB:O$OFG   #5!nH]"5!nH]#E2H^ zz!}##H'**hQVWZ[cWdefWfQg*h'h!noH yy}  "Hhhrl G*2W*<)A (2G';7@R&SGJ !"&)$i2otyy}  "488:H$'HNchhrl 	G*2W*<)A (2G';7@R&SGJ !" 034yBDIIc"%))+DHHJMIVYZIZ
] :Q >#E`aH^ )3(:(?(?(AH$%,6OOB,?,D,D,F,J *4)<zr?R?W?W?Y)YPQ $%
 <<4E:??DHZ'(D}1}!!H)*dRAQH !%'4_"_!H_Or   lookbackc                     t        |       |k  ry| j                  |dz       }| j                  d   }|dkD  r||z  dz
  dz  S dS )u6   Pourcentage de variation sur les N dernières bougies.        r   r6   r   r   )r   r   )r   r   oldnews       r   r   r      sP    
6{h
++1o
&C
++b/C$'!GC#IMS 44r   c                     t        |       |k  ry| j                  |dz         j                  }|t        |      dz     }|dk(  ry|d   |d   z
  |z  dz  S )u/   Slope normalisée sur les N dernières valeurs.r   r      r   r6   r   )r   r   r   )r   r   valsr$   s       r   r   r      sb    
6{h;;x!|$++D
s4yA~
C
axHtAw#%++r   trades
klines_dirc           
         ddl m} ddlm} g }d}i }t        |       D ]  \  }}	|	j	                  dd      }
|	j	                  dd      }|
r|s|dz  }5|
|vr ||      |
 dz  }|j                         s\|
j                  d	      r|
j                  d	d
      n|
j                  d
d	      } ||      | dz  }|j                         r|}n|dz  }t        j                  |      ||
<   ||
   }	 t        j                  |      j                         dz  }t        |      }t        |||      }||dz  }|
|d<   |	j	                  dd      |d<   t        |	j	                  dd            |d<   ||d<   t        |	j	                  dd            }t        |	j	                  dd            }t        |	j	                  dd            }|dkD  rdnd|d<   ||d<   ||d<   |	j	                  dd      |d<   |j!                  |       |dz   dz  dk(  st#        d|dz    dt%        |        d| d        t#        dt%        |       d| d       t        j&                  |      S # t        $ r	 |dz  }Y Tw xY w)u   
    Construit le dataset d'entraînement en recalculant les features
    à partir des klines 1m pour chaque trade historique.

    Returns:
        DataFrame avec features + colonnes target (profitable, pnl_pct)
    r   )Path)datetimesymbol 
entry_timer   z.parquetUSDCUSDTr,   
surge_typeUNKNOWNsurge_strengthpnl_pctpnl_usdtmax_pnltarget_profitabletarget_pnl_pcttarget_max_pnlexit_reasontarget_exit_reasonr   z  Processed /z	 trades (z	 skipped)z  Done: z trades avec features, z skipped)pathlibr   r   	enumerategetexistsendswithr   r   read_parquetr   	timestamp	Exceptionintr   floatappendprintr   	DataFrame)r   r   r)   r   r   rowsskippedloaded_klinesrs   trader   r   parquet_path
alt_symbolalt_pathr'   tsts_msr   r   r   r   s                         r   build_dataset_from_tradesr      s    !DGMf% 8S58R(YY|R0
ZqLG &
+.AAL&&(?Ev?VV^^FF;\b\j\jkqsy\z

+H.EE??$#+LqLG$&OOL$AM&!6"	j)335<B
 B 1U<LMqLG $!&<!C%*5995Eq+I%J!"!+ 		)Q/0:q12		)Q/0-5\q$%%,!"%,!").=")E%&HES=ALQqVYwiyQRq8St 
HSYK6wix
HI<<C  	qLG	s   /&I!!I32I3)1r8   r:   r<   r>   r@   rA   rB   rC   rD   rE   rK   rL   rM   rN   rO   rP   rQ   rU   rW   rY   rZ   r\   r^   r_   r`   ra   rb   rc   rd   re   rf   rh   ri   rj   rl   rm   rn   ro   rv   rw   rx   rz   r{   r|   r   r   r   r   r   )FLASH_SURGEBREAKOUT_SURGEMOMENTUM_SURGETREND_MOMENTUM_SURGE)rV   )r]   g       @)x   )__doc__numpyr   pandasr   typingr   Seriesr   r   r    r   r&   r   dictr   r   r   liststrr   FEATURE_COLUMNSSURGE_TYPESrq   r   r   <module>r      s:  
   8		 83 8299 8"		 "3 "		 "-BII -s -e -  Y
YY Y d^	Yx5 5c 5e 5,299 , , ,  NJNN N \\	Nd> Zr   