
    qioA                        d Z ddlZddlmZ ddlmZ ddlmZmZm	Z	m
Z
mZmZmZ ddlmZ  G d de      Z	 	 	 d"d	eee	   z  d
edededz  dededeeedz        fdZ G d d      Zej0                   edd      ej2                   edd      ej4                   edd      ej6                   eddd      ej8                   eddd      iZd	eeedz        deeedz        fdZ	 d#d	eeedz        d edz  ddfd!Zy)$z
This visualizer requires matplotlib to be installed.

Example usage:

ops = get_schedule_ops("InterleavedZeroBubble", 4, 8)
visualize_schedule(ops, "test.png")
    N)
NamedTuple)mock)_Action_ComputationType_PipelineSchedule_PipelineScheduleRuntimeget_schedule_classPipelineScheduleMultiPipelineScheduleSingle)PipelineStagec                   ,    e Zd ZU eed<   eed<   eed<   y)OpKeystage_indexcomputation_typemicrobatch_indexN)__name__
__module____qualname__int__annotations__r        w/home/ubuntu/crypto_trading_bot/.venv/lib/python3.12/site-packages/torch/distributed/pipelining/_schedule_visualizer.pyr   r      s    &&r   r   schedule	pp_degreenum_microbatchesnum_stages_per_rankadd_spacing
with_commsreturnc                    |r|rt        d      t        | t              rt        |       }n!t	        | t
              r| }nt        d|        t        j                  t        d      }d|_	        ||_
        d|_        t	        |t              r|d}|dk(  sJ |}||z  |_        nSt	        |t              r5|d}|dk\  sJ t        |      D 	cg c]  }	| }}	|D ]  }
||z  |
_         nt        d|        |||      }|j                   J g }|r_t#        ||      }|j%                  |j                          t        |      D ])  }|j'                  t)        |j*                  |                + n.t        |      D ]   }|j'                  |j                   |          " |r-|D cg c]  }|D cg c]  }||	 c} }}}t-        |      }|S c c}	w c c}w c c}}w )	a[  
    Get all actions for a given schedule, pp_degree, and num_microbatches. The actions are returned in a list of lists
    where each inner list represents a rank and each element in the inner list represents an action.

    The schedule can be specified as a string which is passed into get_schedule_class() or a _PipelineSchedule instance.
    z2Cannot add spacing and view comms at the same timezInvalid schedule: T)instancer   N      )
ValueError
isinstancestrr	   
issubclassr   r   create_autospecr   
group_rank
group_sizesubmodr   
num_stagesr
   rangepipeline_orderr   _prepare_schedule_with_commsappendlistpipeline_order_with_commsadd_schedule_op_spacing)r   r   r   r   r   r   schedule_classmock_pipeline_stagestages_stageschedule_instanceall_actionsruntimerankactions                   r   get_schedule_opsr?   "   s2    zMNN(C +H5	H/	0!-hZ899 ..}tL%&"%."!% ."89&"#"a'''$/);	N$9	:&"#"a'''/45H/IJ!%JJ 	?E2Y>E	? -n-=>?? 'v/?@++777 /1K*63CD,,->-M-MN)$ 	NDtG$E$Ed$KLM	N )$ 	GD0??EF	G  KV
BF$=&*<V=
 
 .k: C K8 >
s$   	G'	G0G8G<GGc                   &    e Zd Z	 	 ddededefdZy)_ComputationTypeVisualcolortextwidthc                 .    || _         || _        || _        y N)rB   rD   rC   )selfrB   rC   rD   s       r   __init__z_ComputationTypeVisual.__init__q   s     

	r   N) r#   )r   r   r   r'   r   rH   r   r   r   rA   rA   p   s+     	  	r   rA   blueForwardtealzBackward InputgreenzBackward WeightorangezFull Backwardr$   purplezOverlap F+B   c                 D   | s| S t        d | D              dz   t        |       }t        |      D cg c]  }g  c}| D cg c]  }t        j                  |       }}i dt
        dt        dt        ffddt        dt        t
           ffddt        dt        dt        ffd	dt        d
t        dt        dt        ffd}d}d}t        j                  t        |      d      }|rt        d|        t        |      D ]~  \  }	}
|
s	||	   }
|
d   }t        d|	 d|       |'|	   j                  d       |
j                          d}M|||	   k\  sV ||      s` |||	|      ||	<   |
j                          d} t        |      D ]  }	|||	   k\  s|	   j                  d       ! |D 
cg c]  }
|
s|
	 }}
|dz  }|dz  }|t        d t         j#                         D              kD  rt%        d      |rS c c}w c c}w c c}
w )a  
    Add spacing to the schedule based on dependencies between ranks.

    Before adding an operation to the list, this function checks if there are
    dependencies from other ranks. If there are dependencies (other ranks have
    not finished processing the required microbatch), it adds None instead.

    For example, Forward microbatch 0 on rank 1 depends on rank 0 processing
    Forward microbatch 0 first.

    Args:
        schedule: The original schedule as a list of lists where each inner list
                 represents a rank and each element represents an action.

    Returns:
        A new schedule with proper spacing based on dependencies.
    c              3   F   K   | ]  }|D ]  }||j                      y wrF   )r   ).0rank_actionsr>   s      r   	<genexpr>z*add_schedule_op_spacing.<locals>.<genexpr>   s9      
&
 ! 

s   !r#   dependency_keytimestepr    c                      | v xr ||    k\  S )zDCheck if a dependency operation has completed by the given timestep.r   )rV   rW   scheduled_opss     r   is_dependency_readyz4add_schedule_op_spacing.<locals>.is_dependency_ready   s$     m+ :M.99	
r   r>   c                    | j                   }| j                  }| j                  }|J d|  d       |dk(  r|t        j                  k(  rg S |dz
  k(  rA|t        j
                  t        j                  fv rt        |dz
  t        j                  |      gS |t        j                  k(  rt        |dz
  t        j                  |      gS |t        j
                  t        j                  fv r<t        |dz   t        j
                  |      t        |dz   t        j                  |      gS |t        j                  k(  rt        |t        j                  |      gS t        d|       )z+Get the list of dependencies for an action.Action  has None microbatch_indexr   r#   Unknown computation type: )
r   r   r   r   FORWARDFULL_BACKWARDBACKWARD_INPUTr   BACKWARD_WEIGHTRuntimeError)r>   	stage_idx	comp_typemb_idxr-   s       r   get_dependenciesz1add_schedule_op_spacing.<locals>.get_dependencies   sn   &&	++	(( !OWVH4N#OO! >i+;+C+CCI 
Q&9**++9
 ,
 )a-)9)A)A6JKK (000)a-)9)A)A6JKK **++
 

 i!m%5%C%CVLi!m%5%D%DfM  (888)%5%D%DfMNN7	{CDDr   c                 .   | j                   t        j                  t        j                  t        j                  fv r |       }t        fd|D              S | j                   t        j                  k(  r |       }t        fd|D              S | j                   t        j                  k(  rM| j                  J d|  d       g }| j                  D ]  }|j                   |              t        |      S t        d| j                          )zBCheck if an action is ready to be scheduled at the given timestep.c              3   0   K   | ]  } |        y wrF   r   rS   deprZ   rW   s     r   rU   zCadd_schedule_op_spacing.<locals>.is_action_ready.<locals>.<genexpr>        Rc*39R   c              3   0   K   | ]  } |        y wrF   r   rj   s     r   rU   zCadd_schedule_op_spacing.<locals>.is_action_ready.<locals>.<genexpr>   rl   rm   OVERLAP_F_B action  has None sub_actionsr^   )r   r   r`   ra   rb   anyr_   allOVERLAP_F_Bsub_actionsr1   rc   )r>   rW   dependenciesdep_list
sub_actionrg   is_action_readyrZ   s    `   r   rx   z0add_schedule_op_spacing.<locals>.is_action_ready   s    ""**++,,'
 

 ,F3LR\RRR$$(8(@(@@+F3LR\RRR$$(8(D(DD%%1 %fX-BC1 $&H$00 G

H EFGx= !;F<S<S;TUVVr   r=   c                 @   
|   j                  |        | j                  }t        |   j                  }||z   }|t        j
                  k(  r| j                  J d|  d       d}| j                  D ]l  }|j                  J d| d       t        |j                     j                  }||z  }||z   	t        |j                  |j                  |j                        <   n |S | j                  J d|  d       |	t        | j                  || j                        <   |S )z.Schedule an action and return completion time.ro   rp   r   zSub-action r]   r\   )
r1   r   action_type_to_color_mappingrD   r   rs   rt   r   r   r   )r>   r=   rW   re   	comp_timecompletion_timecumulative_timerw   sub_comp_timerY   spaced_schedules            r   schedule_actionz0add_schedule_op_spacing.<locals>.schedule_action   sq   $$V,++	0;AA	"Y.(444%%1 %fX-BC1  O$00 /
!22> !*-GH> !=//!%   =0 . ".."33"33/.  **6 &!;<6
   f(()V5L5LM r   r   zCurrent timestep: zRank: z	, action=Nc              3   4   K   | ]  }|j                     y wrF   )rD   )rS   visuals     r   rU   z*add_schedule_op_spacing.<locals>.<genexpr>E  s      ,
#FLL,
s   z2No progress made in scheduling - possible deadlock)maxlenr.   collectionsdequer   r   boolr   r2   dictfromkeysprint	enumerater1   popleftrz   valuesrc   )r   	num_ranksr8   opsrank_opsr   current_timesteptimesteps_without_progressrank_completion_timesr=   op_queuer>   rg   rx   rZ   r-   rY   r   s               @@@@@@r   r4   r4      s   (  	 
 (
 	
 	  HI?DY?O2P!22PO2:;3!!#&;H; ')M
E 
S 
T 
&E &ET%[ &EPW W3 W4 W2$ $s $c $c $N !" MM%	*:A>
"#3"456'1 	/ND(~Ha[FF4&
6),-~%,,T2  "-.*!%:4%@@_(F />D"2/%d+   "-.*%	/* )$ 	3D#8#>>%,,T2	3
 .6BHBBA"a'"% ,
'C'J'J'L,
 )
 
 STTI L } 3Q;h Cs   	HHHHfilenamec                 :   ddl m} ddlm} d|j                  d<   t        |       }t        d | D              }|j                  |dz   |dz   f      \  }}d	}t        ||      d
z   }	t               }
t        |       D ]  \  }}d}|D ]  }|t        j                  |j                  t        d            }|
j                  |j                         |j                   }|j"                  }|j$                  d}d}nd}d} ||||z
  dz
  f|d|d|      }|j'                  |       |j)                  ||dz  z   ||z
  dz
  dz   t+        |      dd|	d|       ||z  }n|dz  }t        ||      }  |j-                  d|dz          |j/                  d|dz          |j1                  t3        |      D cg c]
  }||z
  dz
   c}       |j5                  t3        |      D cg c]  }d| 	 c}|	       |j7                  g        |j9                  d       |
D cg c]2  } |dddt        |   j                   dt        |   j(                        4 }}|j;                  |d|	       |r|j=                  |d       y|j?                          yc c}w c c}w c c}w )aq  
    Visualize the schedule using matplotlib.
    The schedule is a list of lists where each inner list represents a rank and each element in the inner list represents an action.
    The actions are represented as rectangles with different colors based on their computation type.
    The filename is optional and if provided, the plot will be saved to that file.

    Args:
        schedule: The schedule to visualize.
        filename: The filename to save the plot to. If not provided, the plot will be displayed.
        add_schedule_spacing: If True, add spacing to the schedule based on dependencies between ranks.

    r   N)	RectanglezDejaVu Sanszfont.familyc              3   2   K   | ]  }t        |        y wrF   )r   )rS   r=   s     r   rU   z%visualize_schedule.<locals>.<genexpr>e  s     5Dc$i5s   r$   )figsize   blacknormalr#   )	facecolor	edgecolor	linewidthg      ?centerwhite)havafontsizerB   weightg      zRank )r   F)r   r   )r   r   labelzupper right)handleslocr   tight)bbox_inches) matplotlib.pyplotpyplotmatplotlib.patchesr   rcParamsr   r   subplotsminsetr   rz   getr   rA   addrB   rD   rt   	add_patchrC   r'   set_xlimset_ylim
set_yticksr.   set_yticklabelsset_xticklabelsgridlegendsavefigshow)r   r   pltr   r   max_actionsfigaxmax_draw_position	font_sizeused_computationrank_idxactionsdraw_positionr>   comp_type_colorrB   rD   r   text_weightrectire   legend_elementss                           r   visualize_scheduler   M  s   " $, 	 LL HI5H55K llK!OY]#ClDGCK+a/Iu&x0 -F' +	FF!">"B"B++-CG-L# !$$V%<%<='--'-- %%1 !I"*K !I"*K !"I$81$<=#%' T" !EAI-(1,s2K&!&  	 &" #$5} EW+	F-F\ KK'!+,KKi#o&MMeI>NO(9x'#-OPU9-=>%s>Sr GGEN *
  	29=CC.y9>>	

O 
 IIo=9IMH'2
/ P>
s   1JJ7J)NFFrF   ) __doc__r   typingr   unittestr   &torch.distributed.pipelining.schedulesr   r   r   r   r	   r
   r   "torch.distributed.pipelining.stager   r   r'   typer   r   r2   r?   rA   r_   ra   rb   r`   rs   rz   r4   r   r   r   r   <module>r      s        =J  '+KD*++KK K t	K
 K K 
$w~
K\	 	 4VYG##%;FDT%U$$&<"' ""$:/1%   "8=RS"T
  @4$'(@	$w~
@J  h4$'(hDjh 
hr   