ti= (ddlZddlZddlZddlZddlZddlZddlZddlmZm Z ddl m Z ddlm Z ddl mZddlmZmZddlmZddlmZmZmZmZmZdd lmZddlZddlmZddlm Z ddl!m cm"Z#dd l$m%Z%dd l&m'Z'dd l(m)Z)m*Z*m+Z,dd l-m.Z.ddl/m0Z0m1Z1ddl2m3Z3ddl4m5Z5m6Z6m7Z7ddl8m9Z9ddl:m;Z;mm?Z?m@Z@mAZAddlBmCZCmDZDmEZEmFZFddlmGZGmHZHddlImJZKddlLmMZMmNZNmOZOmPZPddlQmRZRmSZSmTZTmUZUmVZVmWZWddlXmYZYdZZeUrdZ[dZ\ejjZZn.eVrdZ[dZ\n'eWrdZ[d Z\ejjZZnd!Z[d"Z\d#ZZGd$d%eZ`Gd&d'eZaGd(d)e jeZcd*e jd+ejd,e fd-Zed.Zf dzd*e jd/egfd0Zhd{d1Zid2Zjd3Zkd|d*e jd4egfd5Zld*e jd6egfd7Zmd*e jd8egfd9ZnGd:d;ZoGd<d=ecZpGd>d?ecZqGd@dAeqZrGdBdCeqZsGdDdEecZtGdFdGetZuGdHdIe jZvGdJdKeqZwGdLdMe jZxGdNdOe jZzGdPdQe jZ{ejdRe fdSZ}ejdTe fdUZ~ejdVe fdWZejdXe fdYZejdZe fd[Zeejd\e fd]Zeejd^e fd_Zeejd`e fdaZeejdbe fdcZdde d,e deedfefdgZ d}dhe jdie jdjeedkffdlZejeWdmGdndoeNZGdpdqeMZd~dreefdsZGdtdue jZGdvdwe jZGdxdye jZy)N)ABCabstractmethod)Callable) nullcontext)deepcopy)autoEnumwraps)Anycast no_type_checkOptionalUnion)mock) checkpoint) DeviceMesh) CPUOffload fully_shardFullyShardedDataParallel) TrainingState)FSDPParamGroupRegisterPostBackwardFunction)#NO_RESHARD_AFTER_FORWARD_STRATEGIES)BackwardPrefetchMixedPrecisionShardingStrategy)ShardedGradScaler)always_wrap_policyModuleWrapPolicywrap)distribute_tensorDTensorShard)ColwiseParallelparallelize_moduleRowwiseParallelSequenceParallel)TransformerDecoderLayerTransformerEncoderLayer)DistributedDataParallel)MultiProcessTestCaseMultiThreadedTestCase run_subtests TEST_SKIPS) FILE_SCHEMAget_cycles_per_ms set_rng_seed TEST_CUDATEST_HPUTEST_XPU) has_tritoncudancclzhpu:0hcclxpuxcclcpuglooc(eZdZeZeZy) FSDPInitModeN)__name__ __module__ __qualname__rNO_FSDP RECURSIVEi/home/ubuntu/crypto_trading_bot/.venv/lib/python3.12/site-packages/torch/testing/_internal/common_fsdp.pyrArAYsfGIrHrAc6eZdZeZeZeZy)DEVICEInitModeN)rBrCrDr DEVICE_BEFORE DEVICE_AFTER DEVICE_NEVERrGrHrIrKrKbsFM6L6LrHrKceZdZdZedeejdffdZedejfdZ ed dZ e ede d e de jfd Zy) FSDPTestModelzZThis defines the interface expected from all models used commonly for FSDP unit tests.return.cy)z+Returns an input for the model as as tuple.NrGselfdevices rI get_inputzFSDPTestModel.get_inputo rHcy)z,Returns the loss given the input and output.NrG)rTinputoutputs rIget_losszFSDPTestModel.get_losstrWrHNcy)z.get_futures'+0==+?+?+AF   a MrH)rMockr)rTr`ra dist_waitrs rI allreducezDummyProcessGroup.allreduces"IIK   * rHN)rBrCrDintrrrrrGrHrIrrs2Scc rHrceZdZdejdededeffd ZdZdZ dZ d Z e ddejd e ded eeeefdeded eej(effd ZdZxZS)TransformerWithSharedParamsrqdevice_init_modeadd_bn deterministicct||j|_|j|_|rt j dd}d}tj|||_ tj|dddd|_ tj|||_ |jj|j_|jd|jjj!|f|jd t j"|j$t j& d|_|r)t jj+|j(nt jj-|_|t0j2k(r|j5t6}|r|j9yy) Nrg?)d_modelnum_encoder_layersnum_decoder_layersdim_feedforwarddropout vocab_bias long_buffer)dtype)superrrrrrg manual_seedrj Embedding embed_tokens Transformer transformerLinear output_projweightregister_buffernew_ones zeros_likerlongbs BatchNorm1dIdentitybnrKrLrreval)rTrqrrrd_vocabr __class__s rIrz$TransformerWithSharedParams.__init__sa JJL **,    a LL':>>    99Wg6#'"3"3":":  $++22;;WJG      T__EJJ ? 39%((&&tww/uxx?P?P?R ~;; ;77;'D  IIK rHc,tjd|jztjd|j d|j }tj|j dz|j d|j }||fS)Nr? rUr7)rgrrarangeviewr)rTrUsrctgts rIrVz%TransformerWithSharedParams.get_input1sl !dii-(ll2f-221dgg>ll477Q;v6;;AtwwGSzrHc|j|}||jz|jj|z}|j|}|j |}|j ||}|j |Sr)rrrtype_asrrr)rTsrc_idstgt_idsrrxs rIforwardz#TransformerWithSharedParams.forward7sv(DOO#d&6&6&>&>s&CC(ggcl   S# &""rHc|\}}tjj|jd|j d|jddS)Nsum) reduction)rj functional cross_entropyrr)rTrYrZrrs rIr[z$TransformerWithSharedParams.get_loss?sI3}}** KKFKKO ,chhrle+  rHc$|jyrbackwardr]s rIr_z(TransformerWithSharedParams.run_backwardE  rHfsdp_init_mode fsdp_kwargsrQcN|i}|tjk(r&t|tr|d}n|}t ||||S|tj k(rd|vrt tth}n|jd}d|vr8|dtjtjhvrt|tsd}n|}t|tr|d} n|} t | |||} t| |fd|i|} |tjk(r| j!t"} | St%d|)au Initializes a :class:`TransformerWithSharedParams` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps with top-level FSDP. By default, the top-level FSDP uses the ``ModuleWrapPolicy`` for encoder and decoder layers, but a different auto wrap policy may be specified via ``fsdp_kwargs``. device_init_mode (DEVICEInitMode): Determines model movement to DEVICE. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. add_bn (bool): Whether to include batch norm in the model. Nrauto_wrap_policysharding_strategyUnsupported FSDP init mode: )rArE isinstancerfrrFr r*r)popr HYBRID_SHARD_HYBRID_SHARD_ZERO2rrKrMrr ValueError) rqr rr rrpgr fsdp_pg tformer_pgm fsdp_models rIrcz TransformerWithSharedParams.initHsX6  K \11 1%'1X.$fm |55 5!4#3//$ $/??3E#F ${2 34$113C3W3WXY"5%0%'"1X " +,fmA"2 J  >#>#>>']];7  77GHIIrHc|jgSr)rrs rIget_ignored_modulesz/TransformerWithSharedParams.get_ignored_moduless  !!rH)NFT)rBrCrDrw ProcessGrouprKboolrrVrr[r_rirArrrr rrjrkrrcr __classcell__rs@rIrrs(  ()( (  (T #  15# KJ  KJ$KJ)KJd38n- KJ  KJ  KJ ryy$ KJKJZ"rHrceZdZdejdededeffd ZdZdZ dZ d Z e ddejd e ded eeeefded ej&f d ZxZS)NestedWrappedModulerqrrrc t|j|_j|_|t j k(}fd}|rtjdtjttjdd||tj|ttjdd|ttjdd||ttjdd|ttjdd||_ y)Nc&r t|fiS|Srrlayerr rqrs rI _maybe_wrapz1NestedWrappedModule.__init__.._maybe_wrapE58K88LrHrrr7r)rrrrrrKrLrgrrj Sequentialrrmodule rTrqrrrr rr&rs `` ` rIrzNestedWrappedModule.__init__s JJL **,)^-I-II     a mm BIIaO^ <   !R0@. QR#BIIb"$5~F   "a(8.I J BIIaO^ <   rHcxtjd|jztjdd|fS)Nr?r7rr)rgrrrandrSs rIrVzNestedWrappedModule.get_inputs. !dii-( 1a/11rHc$|j|Srr)rTrs rIrzNestedWrappedModule.forward{{1~rHc&|j}|Sr)rrTrYrZr^s rIr[zNestedWrappedModule.get_lossszz| rHc$|jyrrr]s rIr_z NestedWrappedModule.run_backwardr rHr r rQc|i}|tjk(rt|d||S|tjk(r:t|fd||d|}|tj k(r|j t}|Std|)a Initializes a :class:`NestedWrappedModule` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps some nested modules with FSDP but not the top-level module. The model may later be wrapped with a top-level FSDP external to this method if desired. device_init_mode (DEVICEInitMode): Determines model movement to DEVICE. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. FrrrTr) rArEr!rFrKrMrrr)rqr rr rrs rIrczNestedWrappedModule.inits.  K \11 1&!1+   |55 5,!1+   J >#>#>>']];7  77GHIIrHNF)rBrCrDrwrrrKrrVrr[r_rirArrrr rjrkrcrrs@rIr!r!s     )    @2 15# +J  +J$+J)+Jd38n- +J  +J  +J+JrHr!c heZdZe ddej dededee e e fde f fd Z xZS)AlwaysWrapNestedWrappedModulerqr rr rc0ttt |tj|||}|tjk(r|S|tj k(rB|xsi}t |fdti|}|tjk(r|jt}|Sy)z Initializes a :class:`NestedWrappedModule` instance, but unlike :meth:`NestedWrappedModule.init`, for the ``RECURSIVE`` init mode, this wraps with top-level FSDP and the ``always_wrap_policy()`` auto wrap policy. )rqr rr rr N) rr8rcrArErFrrrKrMrr)rqr rr rrlrrs rIrcz"AlwaysWrapNestedWrappedModule.inits )+H '//-#'    \11 1L |55 5%+KeX6HXKXJ>#>#>>']];7   6rHr6)rBrCrDrirwrrArKrrrr rrcrrs@rIr8r8s^ 15#   $)d38n-   rHr8c eZdZdejdededeffd Zed dZ e d dejde dede e e efdef d ZxZS) NonUniformReqGradNWMrqrrrc tt| j|_j |_|t jk(}fd}|rtjdtjttjdd||tj|ttjdd|ttjdd||tjttjdd|ttjdd||_y)Nc&r t|fiS|Srrr$s rIr&z2NonUniformReqGradNWM.__init__.._maybe_wrap.r'rHrrr7r)rr!rrrrrKrLrgrrjr(rrr)r*s `` ` rIrzNonUniformReqGradNWM.__init__s !413JJL **,)^-I-II     a mm BIIaO^ <   !R0@. QR#BIIb"$5~F   #BIIb!$4nE#BIIaO^D   rHc|jD]-\}}tj||r|jd/yr6)rtrematchrequires_grad_)rl req_grad_masknps rI_set_nonuniform_req_gradz-NonUniformReqGradNWM._set_nonuniform_req_gradEs:**, (DAq88M1-  ' (rHr r ctjd}|tjk(r't |d||}tj |||S|tj k(rT|i}t |fd||d|}|tjk(r|jt}tj |||Std|)a Initializes a :class:`NestedWrappedModule` instance, but unlike :meth:`NestedWrappedModule.init`, it wraps a second :class:`torch.nn.Sequential` container to enable the desired non-uniform ``requires_grad`` ``use_orig_params=True`` tests. For both ``RECURSIVE`` and ``NO_FSDP`` init modes, freezes all parameters except the last two to validate ``ShardedGradScaler`` support for ranks with no (non-zero sized) local shards in FSDP ``use_orig_params=True`` mode. zmodule\.2.*\.1.*Fr5Tr) r?compilerArEr;rErFrKrMrrr)rqr rr rreq_grad_pattern ddp_modelrs rIrczNonUniformReqGradNWM.initKs(::&9: \11 1,!1+ I ! 9 9)EU V  |55 5" -!1+   J >#>#>>']];7 9 9*FV W 77GHIIrHrdr6)rBrCrDrwrrrKrrirErArrrr rcrrs@rIr;r;s(   ( ( ) (  ( T((  15# +J  +J$+J)+Jd38n- +J  +J+JrHr;c eZdZdZdej dedeffd ZdZdZ dZ d Z e d e ed ededed ef d ZxZS)ModuleWithDelayzThis class wraps a :class:`FSDPTestModel` to optionally add a delay after computing the loss and/or before the gradient reduction.r)delay_after_loss_msdelay_before_reduction_mscLt|||_||_||_yr)rrrLrMr))rTr)rLrMrs rIrzModuleWithDelay.__init__~s' #6 )B& rHc8|jj|Sr)r)rVrSs rIrVzModuleWithDelay.get_inputs{{$$V,,rHc$|j|Srr.r/s rIrzModuleWithDelay.forwardr0rHcB|jj||}|jdkDrststr$t j |jdz |Str=tjjt|jtz|SNr) r)r[rLr4r5timesleepr3rgr8_sleeprr1r2s rIr[zModuleWithDelay.get_losssy{{##E62  # #a '8 433d:;  !!#d&>&>ARAT&T"UV rHctjjfd}tjd|5j j |dddy#1swYyxYw)NcjdkDrrtr>tjj t jt zn.tstr"tjjdz |i|SrR) rMr3rgr8rVrr1r4r5rTrU)r`raorig_reduce_scatterrTs rI_delayed_reduce_scatterz=ModuleWithDelay.run_backward.._delayed_reduce_scattersk--1JJ%%D::=N=PPQJJt==DE&77 7rHz'torch.distributed.reduce_scatter_tensor)rg distributedreduce_scatter_tensorrpatchr)r_)rTr^rZrYs` @rIr_zModuleWithDelay.run_backwardsW#//EE 8ZZ 57N  + KK $ $T * + + +s AA' module_class model_args model_kwargsc<t|j|i|||S)aA Args: module_class (Type[FSDPTestModel]): Wrapped module class to which to add delays. model_args: Positional arguments forwarded to the ``module_class`` ``init()``. delay_after_loss_ms (int): Delay after computing the loss/before the optimizer step (in ms). delay_before_reduction_ms (int): Delay before reduce-scattering gradients (in ms). model_kwargs: Keyword arguments forwarded to the ``module_class`` ``init()``. )rKrc)r^rLrMr_r`s rIrczModuleWithDelay.inits,* L  z :\ :  %  rH)rBrCrDrerjrkrrrVrr[r_rityperPr rcrrs@rIrKrKzsF   ! $' -+$ =)  ! $'     rHrKc~eZdZeej ddddfdejdedede e e e fde d ed efd Zy) NestedWrappedModuleWithDelayNFrrqr rr rrLrMc Dtjt|||||||S)Nrqr rr rrLrM)rKrcr!rfs rIrcz!NestedWrappedModuleWithDelay.inits4## )-#' 3&?$  rH)rBrCrDrirKrMrwrrArrrr rrrcrGrHrIrdrds,:+F+F04##$)*    $ ) d38n-    !  $'  rHrdc$eZdZfdZdZxZS)DummyDDPc0t|||_yr)rrr))rTr)rs rIrzDummyDDP.__init__s  rHc&|j|i|Srr.rTr`ras rIrzDummyDDP.forwardst{{D+F++rHrBrCrDrrrrs@rIrhrhs ,rHrhceZdZdejdedededef fd ZdZ dZ e d dejd e ded e eeefdedef d ZxZS) MixtureOfExpertsrqrrdelay_before_free_msrc t|||||||_||_||_|t j k(|_|r"tjd|jzd}d}d} ttj|||j} td| jD|_| jD] } d| _ |rtjdttj|||j} |rHtj$j'|jg} t)| | fi|} t)| |fi|} tj*ttj| ||j| | ttj|| |j|_y) N)rqrrr*rrrc3<K|]}|jywr)numel).0rDs rI z,MixtureOfExperts.__init__..s$L1QWWY$LTr)rrrqrorrKrLrrgrrrrjrrrnum_expert_paramsexpertr[ new_grouprr(r))rTrqrrrorr d_expertd_sharedd_inputrxrDshared expert_grouprs rIrzMixtureOfExperts.__init__s -'   $8!"..2N2NN    b499n - 8X!>@S@ST!$$L8I8I8K$L!L""$ AAH     a  8X!>@S@ST  ,,66L&,>+>F&%7;7Fmm BIIgx8$:M:M N   BIIh8$:M:M N   rHcfjdkDrjd}t|tretj j jjfd}tjd|5j|cdddSj|S#1swYxYw)Nrrctr>tjjt j t zn.tstr"tjj dz |i|S)NrS) r3rgr8rVrror1r4r5rTrU)r`ra orig_reshardrTs rI_delayed_reshardz2MixtureOfExperts.forward.._delayed_reshard"s]  )) 9 9** {{1~ **s ;B''B0c|j|jstj5|j D]v}t |dr|j |j j|jtjj|j |jx dddyy#1swYyxYw)Nrxrp) rrrgrrhasattrgraddiv_rr[ all_reducerq)rTr^rDs rIr_zMixtureOfExperts.run_backward4s ~~ O*OAq(+ vv) DOO4))44QVV4::4N O O O O Os-CACCr r c|i}|tjk(rt|d|||S|tjk(r;t|fd|||d|}|tj k(r|j t}|Std|)a Initializes a :class:`MixtureOfExperts` instance. Args: fsdp_init_mode (FSDPInitMode): If ``NO_FSDP``, then does not wrap any modules with FSDP. If ``RECURSIVE``, then wraps some nested modules with FSDP, including the expert and shared layers, but not the top-level module. The model may later be wrapped with a top-level FSDP external to this method if desired. device_init_mode (DEVICEInitMode): Determines model movement to DEVICE. fsdp_kwargs (Optional[Dict[str, Any]]): Optional keyword arguments forwarded to the FSDP constructor. deterministic (bool): Whether to make the model deterministic across constructions. delay_before_free_ms (int): Delay before resharding expert parameters in the forward pass (in ms). F)rrrorTr) rArErnrFrKrMrrr)rqr rr rrors rIrczMixtureOfExperts.init@s4  K \11 1#!1%9+  |55 5)!1%9+   J >#>#>>']];7  77GHIIrH)NFr)rBrCrDrwrrrKrrrr_rirArrrr rcrrs@rIrnrns2   2 2 ) 2 " 2  2 h0 O 15#$% 0J  0J$0J)0Jd38n- 0J  0J " 0J0JrHrnc eZdZ ddddddedeej deded ef fd Zd ejd ejfd Z dZ xZ S)MLPTFr7)bias with_bufferdim_multiplierdimrUrrrc t|tj|||z|||_tj||z||||_|r)|j dtj|f|yd|_ y)N)rUrrr) rrrjrin_projout_projrrgrandnr)rTrrUrrrrs rIrz MLP.__init__uso yyns&:6PTU  .3"6FQUV   5;;vf+M NDKrHrrQc|j|}tj|}|j|}tj|}|j||jz}|Sr)rFrelurr)rTrzs rIrz MLP.forwardsS LLO FF1I MM!  FF1I ;; "DKKArHc|j4tjjj |jyyr)rrgrjrcnormal_rs rIreset_parameterszMLP.reset_parameterss+ ;; " HHMM ! !$++ . #rHr) rBrCrDrrrgrUrrrhrrrrs@rIrrtsv*. ! &    "%,,/rHrcFeZdZdddedeffdZdededed dfd ZxZS) MLPStackF)with_seq_parallelmlp_dimrct|dt|t|dg}|r&|jtj|dt ||||_y)N)rFr)rappendrj LayerNormrrr)rTrrmodulesrs rIrzMLPStack.__init__sX  * L  * $   NN2<<e< = '"!2rHtp_meshdp_meshuse_activation_checkpointingrQc tdtdtdtdtd|jrttdn td}|jrt d|d<t ||||D]8}t |tjr|r t|t|fd |i|:t|fd |i||S) NF)use_local_outputr?)output_layouts)z 0.in_projz 0.out_projz 1.in_projz 1.out_projz 2.in_projz 2.out_proj) sequence_dim3) device_meshparallelize_planmesh) r%r'rr$r(r&rrjrrr)rTrrrr rr)s rI parallelizezMLPStack.parallelizes)%@)5A(%@)5A(%@%%*qB "    ! !$4!$D S !4WGWX =F&",,/+6"  ?'rHrnew_all_gather_into_tensorc#Ktj}tj|t_ dtj|t_y#tj|t_wxYwwr)rwall_gather_into_tensorbarrier)rorig_all_gathers rIpatch_all_gatherrsO11OLLN"sR(55LLN#4N :  '9$ '9$r new_backwardc#Ktj}tj|t_ dtj|t_y#tj|t_wxYwwr)rrrwr)r orig_backwards rI*patch_register_post_backward_hook_backwardrKsT199MLLN,8 )>  0=$- 0=$-rrYr`ract|dkDr|d}nd|vr|d}ntd|d|||||i|S)NrrZz,Cannot get reduce-scatter output from args: z kwargs: )lenAssertionError)clsrYrnr`rarZs rIreduce_scatter_with_assertrXsa 4y1}a V !;D6F8 T  f  / //rHreplicated_modulesharded_moduleprefixes_to_ignore.ct|j|jdD]\\}}\}}|}|D]} |j| d}|j|||j |t t |t sJ|j|j} } t| tdtdfk(r tdt|| | } |j|j| j|j|j|j |j!|jt|j| | } |j |jt t |jt sJ|j|jj| jy)NTrrrzmFSDP's (Shard(0), Shard(0)) layout differs from distribute_tensor(), so we cannot check for equality using it)r{rtreplace assertEqualassertIsInstancer#rr placementsrfr$rr"to_localr assertIsNoneassertIsNotNone)rrrrreplicated_namereplicated_param sharded_name sharded_paramclean_sharded_nameprefixrrsharded_ref_paramsharded_ref_grads rIcheck_sharded_parityrks OR**,'')OTJ+*-JlM *( HF!3!;!;FB!G  H );< ]G4-111(44m6N6Nj  q58 4 4 ; ..>jQ  ..02C2L2L2NO  (   ]// 0  M../,-=-B-BD*U ]//9-,,g666  **3357G7P7P7RS5TrHznot-support-multithreadc@eZdZedZfdZdZdZdZxZ S)FSDPTestMultiThreadctSr DEVICE_COUNTrs rIrzFSDPTestMultiThread.world_sizerHcBt||jyr)rsetUp_spawn_threadsrTrs rIr zFSDPTestMultiThread.setUps   rHc t|g|i|Srr.rks rIr.z FSDPTestMultiThread.run_subtestsD242622rHc@tjjyrrg_dynamoresetrs rIperThreadSetUpz"FSDPTestMultiThread.perThreadSetUp rHc@tjjyrrrs rIperThreadTearDownz%FSDPTestMultiThread.perThreadTearDownrrH) rBrCrDpropertyrr r.rrrrs@rIrrs) 3rHrc$eZdZfdZedZedZedefdZedZ dZ dZ d Z d Z ed Z d'dej"dedededeededeedededeeeeffdZd dded d d d d d d d d f deedededeededed ed!ee d"ee!deed#ed$ededed%eeeefdeeeeff d&Z"xZ#S)(FSDPTestcht|dtjd<|j y)N0TORCH_NCCL_DESYNC_DEBUG)rr osenviron_spawn_processesr s rIr zFSDPTest.setUps)  14 ,- rHctSrrrs rIrzFSDPTest.world_sizer rHc>tjjSr)rwdistributed_c10d_get_default_grouprs rIrmzFSDPTest.process_groups$$7799rHrQcyr6rGrs rIdestroy_pg_upon_exitzFSDPTest.destroy_pg_upon_exitsrHc*t|jSr)r0 file_namers rI init_methodzFSDPTest.init_methodst~~.//rHc<|j||jyr)rr)rTrrs rI_check_cpu_offloadzFSDPTest._check_cpu_offloads j&<&<=rHc<|j||jyr)rbackward_prefetch)rTrr.s rI_check_backward_prefetchz!FSDPTest._check_backward_prefetchs *J,H,HIrHc<|j||jyr)rforward_prefetch)rTrr1s rI_check_forward_prefetchz FSDPTest._check_forward_prefetchs ):+F+FGrHc t|g|i|Srrrks rIr.zFSDPTest.run_subtestsrrHc ||}||_||_|jdd}td|jd|jt j j|jkr3tjtd|jj |r`t jjjjj!}t#j$d|j||nDt#j$|j&t(t+|j|jd} |jt0z} t2st4rt j j7| | g} t#j8| t j:j=t?|jA||t j:j=t#j8| t#jBy#t,$r=} d | j.d vr&tjtd jd} ~ wwxYw) Nfake_pgFz dist init r=z, world=z multi-gpu-fake)backendrrstore)r*r7rr recompilerbackend_unavailable) device_ids)"rr)getprintrrg accelerator device_countsysexitr/ exit_codetesting _internalr[r5 FakeStorerwinit_process_groupr*DISTRIBUTED_BACKENDr RuntimeErrorr`rr3r5set_device_indexrrrr2run_testdestroy_process_group) rr test_namer)piperarTr5r8er; device_ids rI_runz FSDPTest._runs9~ "**Y. TYYKx/@AB    ) ) +doo = HHZ*T__,= >?II J  //;;CCMMO''"# '' $ 0 0/"4??3  II ,     . .y 9[ +  i&  + ""$1 affQi'$9:DDE   s-B&H// I588I00I5NFrl num_stepsautocastlrfsdp_cpu_offload save_modelmixed_precisionenable_sharded_grad_scaler use_pure_fp16sharded_grad_scaler_kwargsc |xr |j} t|jj} | i} t dd|i| } t j j|j|d}t|D]}|jt jjt|5|jjt jt}| s|rMt|t s=t|t j"r|j%}nt'd|D}||}| rft|t rV|j(t*vrD|jD]1}|j-|jt jd3|jj/||j1| }ddd| j3}|s&| s$|j4t j6k(sJd| r+|j-|j4t j8net|t r+|J|j-|j4|j:n*|j-|j4t j6|jj=|| rTt|t rD|jD]1}|j-|jt jd3| j?|| jA|s|jCjEDcic]\}}||jG}}}tI||jK|t|t r|jMtNjPjSS#1swYxYwcc}}w) Nenabledg?)rSmomentum)r[c3<K|]}|jywr)r)rtrs rIruz4FSDPTest._train_for_several_steps.. s%>1affh%>rvr=zeloss data type should be float32, as the original parameter data type is float32.rG)*offload_paramsnextrrUrrgoptimSGDry zero_gradamprRrr)rVrrrhrrfrrrr[rscalerfloat32float16 param_dtyper_stepupdaterrclonerload_state_dict _assert_staterIDLEru)rTrlrQrRrSrTrUrVrWrXrYcpu_offload_params model_devicesharded_grad_scalerr`rrYrZrDr^kvrs rI_train_for_several_stepsz!FSDPTest._train_for_several_stepss4.Q2B2Q2QE,,./66 % -)+ &/ . 2L    0 0 2rCHy!9 2A OO ##K#B M ..u||K/HI _Zt=T!%6 %  %%>%> >'"5$///>?#--/H((5<<3FGH||,,UF;>>|L- M.',,T2D"=zzU]]252 !$$TZZ?t,*666$$TZZ1L1LM$$TZZ? LL % %d +!j&=))+DA$$QXXu||E/BCD $ $U +  & & (7<7G7G7I7O7O7QRtq!alR RE"%%j1s9 2v eT "    2 2 3{{}w M MfSs9DOO(O% rT model_classr r ref_init_fn num_itersrr.rr1use_orig_params init_kwargsc |tjk7sJd|i}d}|jj}|j|jtjt j fddi|}|.trt|tgt}nt||g|}n||}|r|j}|j||| du||| | || }t|j}|j||| | | | d |j|j|||fddi|}t%|t&st'||jfi|}|r|j}|t j(k(r|j+t}|duxr |j,}|xr|t j(k(}|xr|t j(k7}|rFt/j0d }|jD]}|j3|j0| |r|j5t6d tn t9}|5|j||d |||| | || } ddd|ry|r[t/j0d }|jD]}|j3|j0|  j+t} t;|}!t.j<j?| d | |s|j3||!ddyyy#t$r }t!d |d t#||d}~wwxYw#1swYxYw)a Tests FSDP training against a reference, which defaults to DDP but may be customized with ``ref_init_fn``. Args: model_class (Type[FSDPTestModel]): A model class that inherits from ``FSDPTestModel``, which defines the expected interface. fsdp_init_mode (FSDPInitMode): The mode to initialize the FSDP-wrapped model. This should not be ``NO_FSDP``. ref_init_fn (Optional[Callable]): A callable to invoke that wraps a non-wrapped model to construct the reference model, where this wrapper should provide data parallel semantics. If ``None``, then the callable defaults to the DDP constructor. z.Expects an FSDP init mode that wraps with FSDPN{Gz?rT)r; output_device)rRrSrTrVrWrXrY)rr.rrVr1rwz Initializing z raised error r=zOAn FSDP-managed module with parameter CPU offloading enabled has parameters on F)rRrSrTrUrVrWrXrY) check_dtypezFSDP did not match DDP) exact_devicemsg) rArErmrrcrKrLr4DDPrrrsrrri ExceptionrrrrrMrr^rgrUrassertRaisesRegexrHrrrC assert_close)"rTrtr rrurvrUrr.rrVr1rwrWrXrxrYr rSrrl ref_modelref_loss ddp_paramsrrNr^expects_device_errorexpects_cpu_device cpu_devicer}context fsdp_lossfsdp_unsharded_paramss" rI_test_fsdp_parityzFSDPTest._test_fsdp_parityVsD!5!55 < 5  K !!&&(        ( (       {m;  4&M #E*I !(I00  $D0(+'A''A1  )..01 *%6%6#2$4#2    Y)))""   #   J*d+j$*<*<L LJ #*J ~:: :#{3J$D0O[5O5O  N/>3N3NN   N/>3N3NN  e,J#..0 ;  z: ;$  " "%%0M3    55!,% /+E++E6 I   e,J#..0 ;  z: ;! [1I / ;  ""8YE"J  "=   %!,  ,9 "K Y}[MAxPQWX X YF  s$"LL? L<L77L<?M)rzNFNFFN)$rBrCrDr rrrmrr'r*r,r/r2r. classmethodrPrjrkrfloatrrrrrr rsrbrPrArKrrrrrrs@rIrrsC ::d00>JH34%4%v15 48+0#?CUyyUU U  U #:. UU".1U%)UU%-T#s(^$<Ux+/",,8<8<48!& %+0#04?C#g-(g%g) g h' g  gg g$$45g$$45g".1ggg%)gg d38n-!g"%-T#s(^$<#grHrcompile_compute_on_modulec@fdGddtfd}|S)Nctjjj|i|t |dr|dj yy)Nr)rgr[rrrrG)r`rars rI!fully_shard_with_compiled_computez=compiled_fsdp_test..fully_shard_with_compiled_computesN **D;F; $ , G.1  GOO 1 rHc(eZdZeZeZy)*compiled_fsdp_test..FullyShardModeN)rBrCrDrEAGERCOMPILED_COMPUTErGrHrIFullyShardModers6rHrc4tfd}|S)Nctjjj}D]}|jk7r"t st jdd5tjjj}tjjj}tjj|jk(r|}n^|jk(rAdtjj_ dtjj_ }ntd|| j |j"< |i|tjj| j |j"<|tjj_ |tjj_ y)Nz0Inductor on GPU needs Triton and recent GPU archr) stacklevelTr?z!Need to implement FullyShardMode=)rgr[rrrr6warningswarnrconfigskip_fsdp_hooks _inductorcompile_threadsrrNotImplementedError __globals__rB) r`raoriginal_fully_shardmodeoriginal_skip_fsdp_hooksoriginal_compile_threadsfully_shard_patchrrfuncs rIwrapperz6compiled_fsdp_test..decorator..wrapper sk(-(9(9(>(>(J(J & R>/// MMJWX+0==+?+?+O+O(+0??+A+A+Q+Q(!!))+>///(<%^<<<;?EMM((8=>EOO**:(I%-;D6B CT  !5!>!>?d%f%!!))+BV  !5!>!>?7O $$49Q&&6= RrHr )rrrrs` rI decoratorz%compiled_fsdp_test..decorator s# t R  RDrH)r )rrrrs` @@rIcompiled_fsdp_testrs"""$L rHc&eZdZdfd ZdZxZS) SkipModulec\t|tjddd|_yN Fr)rrrjrlinr s rIrzSkipModule.__init__6s" 99R%0rHc$|j|Sr)rr/s rIrzSkipModule.forward:sxx{rHrdrlrs@rIrr5s 1rHrc$eZdZfdZdZxZS) NestedLinearct||r:ttjdddj t |_ytjdddj t |_yr)rrr!rjrrr nested_linear)rT fsdp_wraprs rIrzNestedLinear.__init__?sV  !%biiBU&C&F&F{&S!TD !#2r!>!A!A+!ND rHc$|j|Sr)rr/s rIrzNestedLinear.forwardFs!!!$$rHrlrs@rIrr>sO%rHrc$eZdZfdZdZxZS) SkipModelct|tjdddj t |_tj t |_tt|t |_ y)NrFr)r)rO) rrrjrrrlinearr linear_skipr!rr)rT double_nestrs rIrzSkipModel.__init__KsW iiBU366{C %rs #$"<< 44 ?S IRRFF FH +  K ::**,L K  K 99))+LK L4T BIIs 4 99$$>% # 99##" " 2299 2t 2>299>d>DBIID$D.Q"-Q"h[J-[J|$7D]J.]J@J mJ Z ? .,ryy,JJ*JJZ/"))/@*r}}*Z'299'6 666  X  "  X  " 9H99 *X** .x.. .x.. :8:: >X>>0!00  0  0.+- Tyy TII Tc3h TF45/6&[#[| 2(4.2j %299 %  rH