qiA ddlZddlZddlZddlZddlmZddlmZmZddl Z ddl Z ddl m cm ZddlmZddlmZddlmZddl mZmZmZdd lmZej4eZGd d eZGd d eZej>defdZ de!defdZ"dejFdefdZ$d4de jJde&de&fdZ'd4de jJde&de&fdZ(dejFde&fdZ)dejFde&fdZ*GddeZ+GddeZ,GddeZ-d gd!ggZ.d"gd"ggd#gd#ggd$gd%gggZ/gd&gd'gd'gZ0dee1fd(Z2d)e&d*e&d+ede1fd,Z3dejFde1fd-Z4d.e jjjlde&fd/Z7d.e jjjlde&fd0Z8 d5d.e jjjld1ee&d2e9de1fd3Z:y)6N)IntEnum)AnyOptional)hint_int)normalize_function)ir)get_dtype_sizesnode_args_kwargs sympy_product)Vc eZdZdZdZdZdZdZy) NCCL_COLLrrN)__name__ __module__ __qualname__ ALL_REDUCE ALL_GATHERREDUCE_SCATTER ALL_TO_ALL UNSUPPORTEDc/home/ubuntu/crypto_trading_bot/.venv/lib/python3.12/site-packages/torch/_inductor/comm_analysis.pyrrsJJNJKrrceZdZdZdZdZy)NVIDIA_GPU_TYPErrrN)rrrVOLTAAMPEREHOPPERrrrrrs E F Frrreturnc8tjjjtjjjxsd}d|vrt j Sd|vrt jSd|vrt jSt jS)NV100A100H100) torchutils collect_env get_gpu_inforunrr r!r")gpu_infos r get_gpu_typer/%s|{{&&33EKK4K4K4O4OPVTVH $$$ 8 %%% 8 %%%%%%r kernel_namecJdvrtjSdvrtjSdvrtjSt fddDrtj Stj S)N all_reduce all_gatherreduce_scatterc3&K|]}|v ywNr).0commr0s r z7get_collective_type_from_kernel_name..;s HTT[ Hs) all_to_allalltoall)rrrranyrr)r0s`r$get_collective_type_from_kernel_namer=3sr  "" "{"###  $### [ (''' H-G H H###$$$rnodect|tjstd||j}|Jt |S)Nz!node is not a collective kernel: ) isinstancer _CollectiveKernel ValueErrorpython_kernel_namer=)r>names rget_collective_typerEAsG dB00 1sz_bytesinprPs rget_collective_input_size_bytesr_WsQH{{=&szz7EN3::+;+;<<<= Orct|tjr5t|tjsddlm}||j dStd|)Nr_get_group_size_by_namezUnsupported collective type: )r@r rA _WaitKernel"torch.distributed.distributed_c10drb constant_args TypeError)r>rbs rget_collective_group_sizerh_sK$,,-jr~~6VN&t'9'9"'=>>7v>??rceZdZdZdZdZy)NCCL_HWrrrN)rrrNVLINKPCINETrrrrjrjms F C CrrjceZdZdZdZy) NCCL_ALGOrrN)rrrTREERINGrrrroross D DrroceZdZdZy) NCCL_PROTOrN)rrrLLrrrrsrsxs  Brrsg333333@gffffff@g333333??g@g@)C@rvgffffff4@)gU@g6@g3@cH|j}|Jt|dd}|jd}ddlm}||}t j j|}t jd|}t|}t|\} } d|vr | dd| dz} t j j|| 5} || i| } t jjjj| ddd j } | dkry| d z }|S#1swY"xYw) NrCr%rcr)_resolve_process_groupzcuda:all_gather_into_tensor_outr)groupdevice@@)r>getattrrfrerxr) distributedget_rankr{evalr _time_estimatorops_c10d_functional wait_tensordefaultestimated_time)snodekernelpy_kernel_namepg_namerxpgrankr{fnargskwargstime_estimatorw est_time_us est_time_mss r/estimate_nccl_collective_runtime_nccl_estimatorrs+ ZZF   V%92>N""2&GI  (B!!**2.D\\E$. )F n B$U+LD&$~5ABx$q'!    * *F * C:~    ""..66q9:!//KQ#K ::s ;L!Q;VaZAF#q[\aFvv&GaKWBI NE  !ty/C/C'C9) U E  y###fqj! %% %fqj! )**I,@,@A A!6\V #E I#c/nnG y### A:f*KK )**I,@,@)BVBVW Wqj i ,GW~i(4HW[[!),Z8HK z 8[)H $0;3IIIG3J*,??L  "B cB Irc^t|}t|}t|}t|||S)9 Returns estimated NCCL collective runtime in nanoseconds (ms). The following heuristics are copied from https://github.com/NVIDIA/nccl/blob/master/src/graph/tuning.cc. We aim to estimate the runtime as accurately as possible. Assumptions: - only ring algorithm (NCCL_ALGO_RING) is used - only Low-Latency protocol (NCCL_PROTO_LL) is used, i.e. Simple or LL128 is not used - 8 gpus per node # TODO: Need to find a way to get accurate "gpus per node" and "# nodes" info. - collective is one of: allreduce, reducescatter, allgather )r_rhrEr)r>rrrs r estimate_nccl_collective_runtimerIs8!@ E*40J t $D 0!:t rfx_nodecd|j|j}}t|}|jdddtj dt fddtjjffd }tjtjj|||f|jjdd}t|tj sy |}|zS) zSEstimate the size of a collective operation in bytes, including inputs and outputs.Nouttr#c`t|jt|jzSr6)rXrFr r\)rs r tensor_bytesz1estimate_fx_collective_size..tensor_byteshs!%affh/.2IIIrr^c|jjdd}t|tjsyd|z y)Nvalr)metagetr@r)Tensor)r^inp_val input_bytesrs r add_inp_bytesz2estimate_fx_collective_size..add_inp_bytesksD((,,ud+'5<<0   K|G,, rrr)rrdictpopr)rrLfxNodepytree tree_map_onlyrrr@)rrrr output_val output_bytesrrs @@restimate_fx_collective_sizer^sK<<&D &\F JJudJ JJ-588==-    v !!%.J*Z"F +L  %%rcBddlm}t|}||s|S|dzS)zEstimate the memory footprint of a collective operation in bytes. This returns the total bytes that need to be live concurrently in memory. For all_reduce, we divide by 2 since it can be done in-place. r)is_all_reduce_tensorr)#torch._inductor.fx_passes.bucketingrr)r is_all_reducerFs r'estimate_fx_collective_memory_footprintrs)  'w /D$W-4<419tj|ng||S)N)r\r{)r)empty)rFr\r{rs r_tensorzVestimate_nccl_collective_runtime_from_fx_node.._nccl_estimate.._tensors&;;%-M? rscXtjjj|dS)NrrI)r rMrNrO)rs r try_size_hintz\estimate_nccl_collective_runtime_from_fx_node.._nccl_estimate..try_size_hints"77##--a!-< ._nccl_estimate..to_real_tensorsa!UXX]]+%affUm44!U\\* 6qvvx @A177AHHUUHr)rzr|)rerrxr)r~distributed_c10d get_backend _get_backendsupports_time_estimater tree_flattenrrJExprrLrtree_unflattentargetr@_ops OpOverloadrrrrrr)rrxrr{backend flat_argsflat_args_pytree_specra real_args real_kwargsrrrrrrrrrrrrs @@r_nccl_estimatezEestimate_nccl_collective_runtime_from_fx_node.._nccl_estimates $J /    - - 9 9" = G'+//&)--+1+>+>f~+N( ( ELL  =UZZ =C = c c 1::1^A&: :!'!6!6yBW!X ; ^^"ejj33444    . .R . 8 >NI--A II & & 2 2 : :1 = >%33  ?!C' ;  > >sF!rsx  $$:8CCg!g   &o & & %c %i %6bii6I6@@s@S@sS "))@BII@#@ g               ,   ,huoBe"e03e;De eZ299*$&$&3$&N =UXX]] =s = $(#b XX]]bC=bb br