qi UddlmZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZmZddlmZddlmZmZmZmZmZmZmZmZddlmZmZddl Z ddl!Z!ddl"Z!ddl#m$Z$dd l%m&Z'dd l(m)Z)dd l*m+Z+dd l,m-Z-dd l.m/Z0ddl1m2Z2m3Z3m4Z4ddl5m6Z6m7Z7ddl8m9Z9m:Z:ddl;mZ>m?Z?ddl@mAZAddlBmCZCmDZDmEZEmFZFmGZGmHZHmIZImJZJmKZKmLZLmMZMmNZNddlOmPZPmQZQmRZRmSZSmTZTmUZUmVZVerkddlWmXZXmYZYmZZZm[Z[ddl"m\Z\ddl]m^Z^ddl_m`Z`maZambZbmcZcddldmeZeddlfmgZgmhZhmiZiddl@mjZjddlkmlZled ZmeXeehgegfZneoelZpeeqe jfZseqZte!jjewd!ZxejewZzd d"Z{ejGd#d$Z}Gd%d&ej$Z~Gd'd(eZeHd)*Gd+d,eZGd-d.ZejGd/d0ZejGd1d2ZejGd3d4ZejGd5d6ZejGd7d8ZeeeeeefZiZd9ed:<Gd;d<ZiZd=ed><iZd?ed@<iZdAedB< d d dCZGdDdEeZ ddFZ ddGZddHZ d ddIZddJZddKZej,ddLZ ddMZ ddNZddOZe!j6e!j8e!j:e!j8ie!j<e!j>e!j@e!jBe!jDe!jFe!jHe!jJe!jLe!jNe!jPf Dcic]}||c}ZdPedQ< ddRZ ddTZ ddUZddVZGdWdXZGdYdZe0Z/Gd[d\Ze j`d]e jb^Zdd_ZGd`dae>eeReZejGdbdcZediddee$jndedfgdhee$jndidjdkldmee$jndndodpldqee$jndrdsdtlduee$jndvdwdxldyee$jndzd{dy|d}ee$jnd~ddldee$jndddddee$jnddgdee$jnddgdee$jnddgdee$jnddgdee$jndddddee$jnddd|dee$jndddldee$jnddgdee$jnddgdee$jndddldee$jndddldee$jnddgdee$jnddgdee$jnddgdee$jnddgdee$jnddgdee$jnddgdee$jnddgdee$jnddgdee$jndÄdĬgdee$jndƄdǬgdee$jndɄdʬgdee$jnd̄dͬgdee$jndτdЬgdee$jnd҄dӬgdee$jndՄd֬gdee$jnd؄d٬gdee$jndۄdܬgdee$jndބd߬gdee$jnddgdee$jnddgdee$jnddgZded<ddZGddeDZGddeGZGddeZejGddZGddZeZGddZGddZedeqZedSeeZeree!jeTeeeedfffZGddeeefZGddZGddeeeZejGddZej,d dZGddZGd d e?Zycc}w(!) annotationsN)ABCabstractmethod)autoEnum)chain)AnycastClassVarGeneric NamedTupleOptional TYPE_CHECKINGUnion)SelfTypeVar)ELEMENTWISE_TYPE_PROMOTION_KIND)_pytree) ConfigModule) OrderedSet)int_oo) PythonPrinter)free_symbol_is_typesymbol_is_typeSymT) bound_sympy ValueRanges)configmetrics)DtypePropagationOpsHandler)BasicMathOpsMixinDefaultHandler)ShapePropagationOpsHandler) boolean_opsDeferredLineBasegenerate_assertget_current_backendIndentedBuffer ir_dataclass ScopedDict sympy_dotsympy_index_symbol sympy_subs triton_typeunique) NullHandlerops OpsHandlerOpsValue ReductionType StoreModeV)CallableIteratorMutableMappingSequence) GraphModule)CustomGraphModulePass)Buffer ChoiceCaller FixedLayoutIRNodeLoopBody)BaseScheduling Scheduler SchedulerNode)BlockShapeTypePythonWrapperCodegen_Tschedulecxtjtjrtj d|yy)NzData type propagation: %s) schedule_log isEnabledForloggingDEBUGdebug)msgs d/home/ubuntu/crypto_trading_bot/.venv/lib/python3.12/site-packages/torch/_inductor/codegen/common.pydata_type_loggerrUas*  /6<0cZeZdZUdZded<ded<d dZed dZd dZed d Z y )FileBackedGraphModulez Output of FX wrapper codegen. Exposes the same methods as ModuleType, but these map back to a GraphModule instead of Python source. r<gmzCallable[..., Any] compiled_fnc&tjddd|_tjtj |jj |j5}|j|jdddy#1swYyxYw)Nzw+z.pyF)modesuffixdelete) tempfileNamedTemporaryFileatexitregisterosremovenamewritevalue)selffs rT __post_init__z#FileBackedGraphModule.__post_init__psi!33eE    4==#5#56 ]] a GGDJJ    s "BBc.|jjSN)r_rerhs rT__file__zFileBackedGraphModule.__file__zs}}!!!rVc |j|Srl)rZrhargss rTcallzFileBackedGraphModule.call~st&&rVc.|jjSrl)rYcoderms rTrgzFileBackedGraphModule.valuesww||rVNreturnNonervstr)rq list[Any]rvr ) __name__ __module__ __qualname____doc____annotations__rjpropertyrnrrrgrVrTrXrXfsF O## ""'rVrXc<eZdZdZdZdZeddZeddZy) WorkspaceZeroModerrHrc||k(s|tjk(r|S|tjk(r|Std|d|d)NzWorkspaceZeroMode.combine(, ))r UNINITIALIZEDNotImplementedErrorabs rTcombinezWorkspaceZeroMode.combinesK 6Q+999H !// /H!$>qe2aU!"LMMrVcF|rtjStjSrl)r ZERO_ON_CALLr) zero_fills rT from_boolzWorkspaceZeroMode.from_bools $11 1 ...rVN)rrrrrvr)rboolrvr) r{r|r}rrZERO_PER_GRAPH staticmethodrrrrVrTrrs9MLNNN//rVrc4eZdZdZeddZeddZy) CodegenSymbolzP An IR object possibly corresponding to a variable in the wrapper code. cyrlrrms rTget_namezCodegenSymbol.get_name rVcyrlrrms rT get_examplezCodegenSymbol.get_examplerrVNrxrvz!Union[torch.Tensor, sympy.Symbol])r{r|r}r~rrrrrVrTrrs/    rVrT)frozenc&eZdZUdZded<ded<ded<ded <d Zded <ejZd ed <e dddZ e d dZ e d!dZ e d!dZ d"dZeZd#dZd$dZd%dZed%dZeZeZeZd&dZd'dZd'dZd(dZd)dZd*dZy)+ WorkspaceArga2A temporary buffer used for a single kernel, then discarded. Not registered as a traditional buffer since there are no users, so it would be dead code eliminated. Args: nbytes: The size of the buffer in bytes. zero_fill: Whether the buffer should be initialized to zero. sympy.Exprcountr zero_mode torch.devicedevicery outer_namews_ptr inner_name torch.dtypedtypecP|ttjjSrl)nextr7graph workspace_id)prefixs rT unique_namezWorkspaceArg.unique_names!$qww334566rVc|j|jk(xr4|j|jk(xr|j|jk(Srl)rrrrs rTcan_joinzWorkspaceArg.can_joins@ LLALL ( XQWW-? XAHHPQPXPXDX rVct|j|jztj|j|j|j |j |j|jSN)rrrrrr) rrrrrrrrrrs rTjoinzWorkspaceArg.joinsS''AGG#'// Q[[I''88||||   rVc|j|jk(r2|j|jk(r|j|jk(sJtt j |j |j tj|j|j|j|j|j|jSr) rrrrsympyMaxrrrrrrs rTmaximumzWorkspaceArg.maximums GGqww 188qxx#7ALLALL|jjSrl) get_layoutrrms rTrzWorkspaceArg.get_examples ,,..rVcfddlm}||j|j|jgdgS)Nr)r@rH)rrsizestride)irr@rrr)rhr@s rTrzWorkspaceArg.get_layouts.$;;****3   rVc"|jSrl)rrms rTlayoutzWorkspaceArg.layouts  rVc6tjjSrl)rSZerorms rT get_offsetzWorkspaceArg.get_offsetsww||rVc|jgSrl)rrms rTget_sizezWorkspaceArg.get_sizes |rVc8tjjgSrl)rrOnerms rT get_stridezWorkspaceArg.get_strides }rVc|jSrl)rrms rTrzWorkspaceArg.get_names rVcy)NFrrms rT get_is_pinnedzWorkspaceArg.get_is_pinned srVcgSrlrrms rTget_inputs_that_alias_outputz)WorkspaceArg.get_inputs_that_alias_output s rVN) workspace_)rryrvry)rrrrrvr)rrrrrvr)rvr)rvrr)rvr@)rvr)rvlist[sympy.Expr]rx)rvr)rv list[str])r{r|r}r~rrtorchuint8rrrrrrrget_device_or_errorrrrrrget_output_specmaybe_get_output_specmaybe_get_layoutrrrrrrrrVrTrrs    OJE;$77        %/ !!!O&!rVrceZdZddZddZy)TritonScratchWorkspacec ||_||_yrl)r_generate_dtype_str)rhrgenerate_dtype_strs rT__init__zTritonScratchWorkspace.__init__s #5 rVc"|jSrl)rrms rTrz)TritonScratchWorkspace.generate_dtype_strs''))rVN)rintrCallable[..., str]rx)r{r|r}rrrrVrTrrs 6*rVrcpeZdZUded<ded<ded<ej j Zded<dZd ed <y) TensorArgryrebufferrrroffsetN Optional[str]alias_of) r{r|r}rrrrrrrrVrTrrs. I K FJ%"Hm"rVrc4eZdZUded<ded<eddZy)SizeArgryrerexprcyrlrrms rTrzSizeArg.alias_of'srVNrvr)r{r|r}rrrrrVrTrr"s I  rVrceZdZUded<y) ConstexprArgryreNr{r|r}rrrVrTrr,s IrVrc6eZdZUded<ded<ded<ded<y) TMADescriptorArgryreapi_typezOptional[list[sympy.Expr]] block_shapeOptional[torch.dtype]rNrrrVrTrr1s IM++  rVrc>eZdZUded<ded<dZded<dZded<y) DeviceCodegenSchedulingConstructor schedulingWrapperConstructorwrapper_codegenNOptional[WrapperConstructor]cpp_wrapper_codegenfx_wrapper_codegen)r{r|r}rrrrrVrTrr9s&%%''8<5<7;4;rVrzdict[str, DeviceCodegen]device_codegensceZdZddZddZddZddZddZddZddZ ddZ dd Z dd Z dd Z dd Zdd ZddZddZddZ d ddZy)DeviceOpOverridesctrlrrhres rTimport_get_raw_stream_asz*DeviceOpOverrides.import_get_raw_stream_asG!!rVctrlrrh device_idxs rT set_devicezDeviceOpOverrides.set_deviceJr rVctrlrrms rT synchronizezDeviceOpOverrides.synchronizeMr rVctrlrr s rT device_guardzDeviceOpOverrides.device_guardPr rVctrlrrms rTcpp_device_guardz"DeviceOpOverrides.cpp_device_guardSr rVctrlrrms rTcpp_aoti_device_guardz'DeviceOpOverrides.cpp_aoti_device_guardVr rVctrlrrms rTcpp_stream_guardz"DeviceOpOverrides.cpp_stream_guardYr rVctrlrrms rTcpp_aoti_stream_guardz'DeviceOpOverrides.cpp_aoti_stream_guard\r rVctrlrrms rTcpp_getStreamFromExternalz+DeviceOpOverrides.cpp_getStreamFromExternal_r rVctrlrrms rT kernel_headerzDeviceOpOverrides.kernel_headerbr rVctrlrrms rT kernel_driverzDeviceOpOverrides.kernel_driverer rVctrlrrms rTcpp_stream_typez!DeviceOpOverrides.cpp_stream_typehr rVctrlrrms rTaoti_get_streamz!DeviceOpOverrides.aoti_get_streamkr rVctrlrrms rTcpp_kernel_typez!DeviceOpOverrides.cpp_kernel_typenr rVctrlrrms rTcpp_device_ptrz DeviceOpOverrides.cpp_device_ptrqr rVctrlrrms rTtma_descriptor_helpersz(DeviceOpOverrides.tma_descriptor_helperstr rVNctrlr)rhidx workspacers rT cpp_scratchzDeviceOpOverrides.cpp_scratchws "!rVreryrvry)r rrvryrxrl)r+rr,rrrrvzOptional[tuple[list[str], str]])r{r|r}rr rrrrrrrrrr!r#r%r'r)r-rrVrTrrFs~""""""""""""""""TX""#9"CP" ("rVrzdict[str, DeviceOpOverrides]device_op_overrides_dictz*dict[str, Optional[CustomGraphModulePass]]custom_backend_passesz!dict[str, Optional[ConfigModule]]custom_backend_codegen_configsct||||t|<|t|<|r)t|tr|t usJd|dt |t |<y)Nzdevice_custom_config=z: cannot be the same as the default inductor config config=)rrr0 isinstancerrr1)rdevice_schedulingdevice_wrapper_codegendevice_cpp_wrapper_codegendevice_fx_wrapper_codegendevice_custom_passdevice_custom_configs rTregister_backend_for_devicer:sv,"! OF %7&! +\ :$F2 %#%%`Y_Xa b   3 .B"6*rVceZdZeZeZeZeZeZeZ eZ eZ eZ eZ y)BackendFeatureN)r{r|r}rFOREACH BUCKETIZEINPLACE_BUFFERSMASKED_SCATTER_WITH_INDEXSCANSORTTUPLE_REDUCTIONPREFER_STORE_LOOP_ORDERTRITON_TEMPLATESREDUCE_TO_SINGLE_ELEMENTrrVrTr<r<sLfGIfO $ 6D 6DfO"fv#vrVr<c:| tStt|tjr |j }n7t|t sJt ||}tj|}t|}|sJ|d}|j|Srl) rinit_backend_registrationr3rrtyperyget_scheduling_for_deviceget_backend_features)r device_typescheduling_ctorrs rTrKrKs~|&%,,'kk &#&4V 4& k*/ j@jC} | dk7rLt5| @ddl"m#} |d}|d}|d}|d}|r|r|rt7| ||||yyyyyy#tH$rYywxYw) z Register the backend for different devices, including the scheduling for kernel code generation and the host side wrapper code generation. rH) CppScheduling) CppWrapperCpu)CppWrapperCpuArrayRef) CppWrapperGpu) CppWrapperMps)CUDACombinedScheduling)HalideScheduling)MetalScheduling)PallasScheduling)PythonWrapperMtia)TritonSchedulingrI)WrapperFxCodegencpuN)cpphalidetritonpallasc6tj|Srl)r cpu_backend)r cpu_backendss rTz+init_backend_registration..s?|F,>,>? KrVcuda)rlrkrmc6tj|Srl)r cuda_backend)r cuda_backendss rTrqz+init_backend_registration..!sA}V-@-@A*MrVxpumpsmtia privateuseoner)_get_custom_mod_func SchedulingrJCppWrapperCodegenrh)%rjr]cpp_wrapper_cpur^cpp_wrapper_cpu_array_refr_cpp_wrapper_gpur`cpp_wrapper_mpsracuda_combined_schedulingrbrkrcrwrdrmrepython_wrapper_mtiarfrlrgwrapperrJ wrapper_fxirrhrJr:r aot_inductorallow_stack_allocationr_C_get_privateuse1_backend_name torch.utils.backend_registrationrz RuntimeError)r]r^r_r`rarbrcrdrerfrgrJrhprivate_backendrzr4rrrrprus @@rTrHrHs #.@..@($(6(-. '/ &&&  $  K ""99 "  !(0-&& $  M    !'/#      !'/#      !(0#       hh<<>O?* %o 6 >I  4\ B 23IJO"67J"K !56H!I  _9L+#%#'& :M_  ? +$   s75E22 E>=E>cLddlm}g|t||j|S)Nr)FlexibleLayout)rrr,contiguous_strides)index index_varssizesrs rTindex_prevent_reorderingrZs, $ UU TIj.*K*KE*RS TTrVc|t|<yrl)r/)rdevice_op_overridess rTregister_device_op_overridesres(;V$rVct|tsJt|tsddlm}m}ddlm}ddl m}ddl m}t|S)NrH)cpu_device_op_overridesmps_device_op_overrides)r) r3ryrIr/rrrrrrxrv)rrrrmtia_op_overridesxpu_op_overridess rTget_device_op_overridesrks5 fc "0DL0 " #F-B@ #F ++rVzdict[torch.dtype, torch.dtype]DTYPE_TO_COMPUTATION_DTYPEcr|tvrtjS|dvrd|vr|dS|dS|dvrtjS|dvrtjS|dk(rd|vr|dS|dS|dk(rd|vr|dS|dS|d vr$|d}t j j|S|d k(rd|vr|dS|d Sy ) zK Given op name and a list of input dtypes, deduce the output dtype )to_dtype index_exprr)randrandn) get_index randint64 load_seed reductionrHconstant)loadstorestore_reductionto_dtype_bitcastN)r%rrfloatint64r7rr)op_namerqkwargsbuf_names rTdeduce_output_dtype_by_namers +-zz  #*V"3vgAbA  {{   {{ K ")V"3vg@a@ J ")V"3vgAbA   7ww  ** & &")V"3vgAbA rVCSEVariableTypect}tjjr'|dk(r"|j d|dt |dytjj r|dk(rddlm}m }t||sJt||tjk(r|jrd|d }n.d |d |d }n$d |d}|jrd|d}d|d||d}|j d|dyyy)Nrltl.static_assert(z .dtype == rrjrH)CppCSEVariable DTYPE_TO_CPPzIsVecMaskType::valuezstd::is_same_v || std::is_same_vz decltype(z typename z ::value_typezstd::is_same_vzstatic_assert(z);)r(r test_configsruntime_triton_dtype_assert writeliner/static_cpp_dtype_assert cpp_utilsrrr3rIrris_vec)rvarrbackendrr is_same_dt c_var_types rT check_dtypers"#G 667h;N,SEK*R89!:J 4rVct}|JtjjrM|dk(rGt |dk7rdj d|Dn|dd}|j d|d|d yyy) NrlrHrc32K|]}t|ywrlry).0ds rT zcheck_shape..s,c!f,sr,rz .shape == ()))r(rrruntime_triton_shape_assertlenrr)rrshaper shape_strs rT check_shapers"#G    667h;N03E aDII,e, ,azQR^  ,SEYKrJK .sS)  # #qvv - B*../55TA B) sAA)all_input_nodesr3rfxNodeoprall functoolsreduce promote_typesrrrr)rhnodeinputsr input_nodesall_input_nodes_propagateds rTdeduce_node_dtype_by_inputsz/DataTypePropagation.deduce_node_dtype_by_inputss%% Auxx}}!=!$$-BWA   { q %() !) & " *   T$++->>+ ;; ! !"3 455d; ;8 ++ L    //55rVcn|jsJd}|jD]}tj|jvr|jtj}n t}|j ||_||jtj<|j dk(s|j }|S)Nr)nodesrrrrrr)rhr graph_dtyperopt_ctxs rTrz#DataTypePropagation.propagate_graph&s{{{-1 KK ,D"&&$))3))$7$;$;<-/ 2248GM18DII)-- .{{h&%mm  ,rVc>|j|jdSr)rrrms rT propagatezDataTypePropagation.propagate8s##DKK$788rVc.||jSrl)r)clsrs rTpropagate_loopbodyz&DataTypePropagation.propagate_loopbody;s4y""$$rVcddlm}ddlm}t ||sJt |t |j |sJt |j tj|j S)NrrB)rF) loop_bodyrC schedulerrFr3rI_bodyrr )r rrCrFs rTpropagate_scheduler_nodez,DataTypePropagation.propagate_scheduler_node?sX(-$ .:T :.$**h/Adjj1AA/"55djjAArVN)rrCrvrw)r torch.fx.Nodervr)rrrvr)rztorch.fx.Graphrvr)rvr)rrCrvr)rrFrvr) r{r|r}rrrrrr classmethodr rrrVrTrrsJ% * 6:$9%%BBrVrcDeZdZddd dfdZddfd ZxZS)rT)simplifypc|r]t|tjrCttj dr)tj j j|}t|%|S)Nsizevars) r3rExprhasattrr7rrrsuperdoprint)rhrrr __class__s rTrzPythonPrinter.doprintJsK  44*9U77##,,T2Dwt$$rVct|tjrd|j|dSt||||SN(r)r3rMod_printr parenthesize)rhitemlevelstrictrs rTr"zPythonPrinter.parenthesizeRs@ dEII &t{{4()+ +7'eV< ?@@rVc tjdtj}tj|tj |tj tj|Sr-)r2rrr/r.rKr<neg)r1ones rTsigmoidzOpDecompositions.sigmoidsCll1ekk*{{3SWWSWWQZ-@ ABBrVcrtj|tjdtjSNr)r2rrrr/r0s rTreluzOpDecompositions.relus"{{1cll1ekk:;;rVcVtjtj|||Srl)r2rKr4r1yzs rTfmazOpDecompositions.fmaswwswwq!}a((rVcTtjtj||Srl)r2rfloorrrs rT floor_to_intzOpDecompositions.floor_to_int||CIIaL%00rVcTtjtj||Srl)r2rceilr[s rT ceil_to_intzOpDecompositions.ceil_to_ints||CHHQK//rVcTtjtj||Srl)r2rtruncr[s rT trunc_to_intzOpDecompositions.trunc_to_intr]rVc tj||}tjtj|tjdt j tjtj|tj|}tj|tj|||SrR) r2modand_nerrr/signbitwhererK)rrrconds rT remainderzOpDecompositions.remaindersy GGAqMxx FF1cll1ekk2 3 FF3;;q>3;;q> 2 yyswwq!}a00rVcTtjtj||Srl)r2rroundr[s rT round_to_intzOpDecompositions.round_to_intr]rVN)rgOpVarTrvrpr1rprvrp)r1rprVrprWrprvrp)rrprrrvrprrprrprvrp)r{r|r}r~rr+r2r5r:r=r?rErGrIrLrPrSrXr\r`rcrlrorrVrTr)r)[s}<<CC<<CCRRQQMMAACC<<))1100111111rVr)z[a-z0-9_.]+|\([^)]*\)|)flagsc|ddk7st|dkryd}t|ddD]3\}}|dk(r|dz }n |dk(r|dz}|dk(s!|t|dz k7s3y|dk(sJy)NrrrFrHrT)r enumerate)stringrichars rT_all_in_parensrys ayC3v;? EVABZ(4 3; QJE S[ QJE A:!s6{Q.  A:: rVcXeZdZed"dZed#dZed$dZed%dZed&dZed&dZ ed&dZ ed&dZ ed&d Z ed'd Z ed(d Z d) d*d Z d+dZd,dZ d- d.dZd/dZd0dZ d1dZ d2dZ d3dZ d4 d5dZd6dZd&dZdej8d dd d7dZd8dZd9dZed:dZ e!d;d Z"e!d> rrs rTbitwise_right_shiftzOpOverrides.bitwise_right_shiftrrVc.tj||Srl)r2r.rs rT int_truedivzOpOverrides.int_truedivs {{1a  rVcTtj|tj|Srl)r2rrInteger)rers rTrzOpOverrides.load_seedsxxemmF344rVTc*tt|Srl)r-ry)rhrrcheckwrap_negs rTindirect_indexingzOpOverrides.indirect_indexings"#c(++rVcDtt|jd)Nz,: check_bounds should be handled by CSEProxyrrIr{rhrrloweruppers rT check_boundszOpOverrides.check_boundss'"Dz""##O P  rVcDtt|jd)Nz$: load should be handled by CSEProxyrrhrers rTrzOpOverrides.load s%!Dz""##G H  rVNcDtt|jd)Nz%: store should be handled by CSEProxyrrhrerrgr\s rTrzOpOverrides.stores'"Dz""##H I  rVcDtt|jdNz3: device_assert_async should be handled by CSEProxyrrhrkrSs rTdevice_assert_asynczOpOverrides.device_assert_async%!Dz""##V W  rVcDtt|jd)Nz/: store_reduction should be handled by CSEProxyrrhrerrgs rTrzOpOverrides.store_reductions%!Dz""##R S  rVcDtt|jd)Nz): reduction should be handled by CSEProxyrrhr src_dtypereduction_typergs rTrzOpOverrides.reduction!s'"Dz""##L M  rVcDtt|jd)Nz$: scan should be handled by CSEProxyrrhdtypes combine_fnvaluess rTscanzOpOverrides.scan,s'"Dz""##G H  rVcDtt|jd)Nz$: sort should be handled by CSEProxyrrhrrstable descendings rTsortzOpOverrides.sort9s'"Dz""##G H  rVcDtt|jd)Nz): bucketize should be handled by CSEProxyrrhr boundariesboundary_indicesindexing_dtyperightsortersorter_indicess rT bucketizezOpOverrides.bucketizeDs'"Dz""##L M  rVcDtt|jd)Nz2: halide_clamp only implemented for Halide backendr)rhrgrrs rT halide_clampzOpOverrides.halide_clampRs%!Dz""##U V  rVcDtt|jd)Nz): dot only implemented for Triton backendr)rhr1rVs rTdotzOpOverrides.dotWs%!Dz""##L M  rVrH) constraintsris_purepackcDtt|jd)Nz<: inline_asm_elementwise only implemented for Triton backendr)rhasmrrrrrs rTinline_asm_elementwisez"OpOverrides.inline_asm_elementwise\s'"Dz""##_ `  rVcDtt|jd)Nz.: ops.output should not appear at codegen timeAssertionErrorrIr{rps rTrzOpOverrides.outputis%Dz""##Q R  rVcDtt|jd)Nz3: ops.placeholder should not appear at codegen timerrhrs rTrzOpOverrides.placeholderns%Dz""##V W  rVc0dfd }|_d|_|S)NcJtt|jd)Nz does not implement ops.r)rhrqrres rT unimplementedz1OpOverrides._unimplemented..unimplementedus*%:&&''?vF rVT)rhr{rqr rr rvrp)r{is_unimplemented)rers` rT_unimplementedzOpOverrides._unimplementedss   "& )- &rVcpt||d}tt|d}| xs||k(xs t|ddS)NrF)getattrr3)r refn default_fns rT_is_unimplementedzOpOverrides._is_unimplemented~s? S$ %Zt4 vSz)SWR9KU-SSrVcP|dvsJ|tjD]\}}t||}|/|j|s&t |||j |C||j vsJd|d|j||_t ||t|y)N)rlrjcppvecrkrwzmultiple definitions of z on ) pointwise_overrides_datarrrsetattrr__dict__r{r)r rfuncnamedataimpls rT_initialize_pointwise_overridesz+OpOverrides._initialize_pointwise_overridessEEMvME6<<> ;NHd4(D|((2C3+=+=h+GHs||3.xjS\\NK3!) X|D'9: ;rV)rvrprvrp)rgzUnion[bool, float, int]rrrvrprq)rrprvrp)r1rprVrprvrprr)reryrrprvrpTT) rrprUnion[sympy.Expr, int]rrrrrv sympy.Symbol rrrrrrrrrvrw)reryrrrvrprl) reryrrrgrpr\r6rvrwrkr}rSryrvrw)reryrrrgrprvrw) rrrrrr5rg!Union[OpVarT, tuple[OpVarT, ...]]rvr)rtuple[torch.dtype, ...]rzFCallable[[tuple[OpVarT, ...], tuple[OpVarT, ...]], tuple[OpVarT, ...]]rtuple[OpVarT, ...]rvr) rrrrrrrrrvrNN)rrpr.tuple[str, sympy.Expr, sympy.Expr, sympy.Expr]rrprrrrr Optional[tuple[str, sympy.Expr]]rzOptional[OpVarT]rvrp)rgrprrrrrvrp)rrprryrrrrrrrrrvrp)rqrprvrw)rrrvrp)reryrvzCallable[..., OpVarT]reryrvr)rryrvrw)$r{r|r}rrrrrrrrrrrrrrrrrrrrrrrrrr8rrrrrrrrrVrTr{r{s1  **..BBBBBBCCCC!! 55 , ,%, ,  ,  ,  &0 9= FJ   NR  * 39 AJ         &  1  +    '     #       '  #        $48+/    C  !  $    1  )       &*"]]      #             TT  ; ;rVr{c|eZdZUded<ded<dZded<dZded<ejZd ed <dZ ded <dZ ded <y) OverridesDataryrerrjNzOptional[Callable[..., str]]rlrrtype_promotion_kindrkrw) r{r|r}rrlrrDEFAULTrrkrwrrVrTrrsQ I +/F (/+/F (/'//8,0F (/(,C %,rVrairy_aicd|dS)Nzairy_ai_forward(rrr0s rTrqrqs(1-rVspecial_airy_ai)rrjre bessel_j0cd|dS)Nzbessel_j0_forward(rrr0s rTrqrq*1#Q/rVcd|dS)Nz libdevice.j0(rrr0s rTrqrq=1-rVspecial_bessel_j0)rrjrlre bessel_j1cd|dS)Nzbessel_j1_forward(rrr0s rTrqrqrrVcd|dS)Nz libdevice.j1(rrr0s rTrqrqrrVspecial_bessel_j1 bessel_y0cd|dS)Nzbessel_y0_forward(rrr0s rTrqrqrrVcd|dS)Nz libdevice.y0(rrr0s rTrqrqrrVspecial_bessel_y0 bessel_y1cd|dS)Nzbessel_y1_forward(rrr0s rTrqrqrrVcd|dS)Nz libdevice.y1(rrr0s rTrqrqrrVspecial_bessel_y1digammacd|dS)Nz calc_digamma(rrr0s rTrqrqs aS*rVc |dS)Nz .digamma()rr0s rTrqrqs A3j)rV)rrjrrer=cd|dS)Nz calc_erfcx(rrr0s rTrqrq A3a(rVcd|dS)Nzlibdevice.erfcx(rrr0s rTrqrqs+A3a0rV special_erfcxrXcd|d|d|dS)Nz std::fma(rrrrUs rTrqrqsis"QCr!A6rVcd|d|d|dS)Nzfmadd(rrrrUs rTrqrqss"QCr!A6rVcd|d|d|dS)Nzlibdevice.fma(rrrrUs rTrqrqss"QCr!A>rV)rrjrrlreigammacd|d|dSNz calc_igamma(rrrrs rTrqrq<s"QCq1rVigammaccd|d|dSNz calc_igammac(rrrrs rTrqrq=2aS2rVgammainccd|d|dSrrrs rTrqrqrrVspecial_gammainc gammaincccd|d|dSrrrs rTrqrqrrVspecial_gammaincci0cd|dS)Nzcalc_i0(rrr0s rTrqrq1orVcd|dSNzlibdevice.cyl_bessel_i0(rrr0s rTrqrq3A3a8rVc |dS)Nz.i0()rr0s rTrqrqs A3erV)rrjrlrrei0ecd|dS)Nz calc_i0e(rrr0s rTrqrq !A&rVc |dS)Nz.i0e()rr0s rTrqrqs A3frV special_i0ei1cd|dS)Nzcalc_i1(rrr0s rTrqrqr(rVcd|dSNzlibdevice.cyl_bessel_i1(rrr0s rTrqrqr+rV special_i1i1ecd|dS)Nz calc_i1e(rrr0s rTrqrqr/rV special_i1elog_ndtrcd|dS)Nzcalc_log_ndtr(rrr0s rTrqrqsqc+rVspecial_log_ndtrmodified_bessel_i0cd|dS)Nzmodified_bessel_i0_forward(rrr0s rTrqrq3A3a8rVcd|dSr*rr0s rTrqrqr+rVspecial_modified_bessel_i0modified_bessel_i1cd|dS)Nzmodified_bessel_i1_forward(rrr0s rTrqrqr?rVcd|dSr5rr0s rTrqrqr+rVspecial_modified_bessel_i1modified_bessel_k0cd|dS)Nzmodified_bessel_k0_forward(rrr0s rTrqrqr?rVspecial_modified_bessel_k0modified_bessel_k1cd|dS)Nzmodified_bessel_k1_forward(rrr0s rTrqrqr?rVspecial_modified_bessel_k1ndtrcd|dS)Nz calc_ndtr(rrr0s rTrqrq%s 1#Q'rV special_ndtrndtricd|dS)Nz calc_ndtri(rrr0s rTrqrq*rrV special_ndtri polygammac *|d|d|d|d|d|d S)Nz == 0 ? calc_digamma(z) : (z == 1 ? trigamma(z) : calc_polygamma(rrrrs rTrqrq/s8S%aSaS0A!DWXYWZZ\]^\__a brVscaled_modified_bessel_k0cd|dS)Nz"scaled_modified_bessel_k0_forward(rrr0s rTrqrq7:1#Q?rV!special_scaled_modified_bessel_k0scaled_modified_bessel_k1cd|dS)Nz"scaled_modified_bessel_k1_forward(rrr0s rTrqrq<rVrV!special_scaled_modified_bessel_k1spherical_bessel_j0cd|dS)Nzspherical_bessel_j0_forward(rrr0s rTrqrqBs4QCq9rVspecial_spherical_bessel_j0zetacd|d|dS)Nzzeta(rrrrs rTrqrqGs52aS*rV special_zetachebyshev_polynomial_tcd|d|dS)Nzchebyshev_polynomial_t_forward(rrrrs rTrqrqL:1#Rs!DrVspecial_chebyshev_polynomial_tchebyshev_polynomial_ucd|d|dS)Nzchebyshev_polynomial_u_forward(rrrrs rTrqrqQrcrVspecial_chebyshev_polynomial_uchebyshev_polynomial_vcd|d|dS)Nzchebyshev_polynomial_v_forward(rrrrs rTrqrqVrcrVspecial_chebyshev_polynomial_vchebyshev_polynomial_wcd|d|dS)Nzchebyshev_polynomial_w_forward(rrrrs rTrqrq[rcrVspecial_chebyshev_polynomial_wlegendre_polynomial_pcd|d|dS)Nzlegendre_polynomial_p_forward(rrrrs rTrqrq`9!BqcCrVspecial_legendre_polynomial_pshifted_chebyshev_polynomial_tcd|d|dS)Nz'shifted_chebyshev_polynomial_t_forward(rrrrs rTrqrqeB1#Rs!LrV&special_shifted_chebyshev_polynomial_tshifted_chebyshev_polynomial_ucd|d|dS)Nz'shifted_chebyshev_polynomial_u_forward(rrrrs rTrqrqjrtrV&special_shifted_chebyshev_polynomial_ushifted_chebyshev_polynomial_vcd|d|dS)Nz'shifted_chebyshev_polynomial_v_forward(rrrrs rTrqrqortrV&special_shifted_chebyshev_polynomial_vshifted_chebyshev_polynomial_wcd|d|dS)Nz'shifted_chebyshev_polynomial_w_forward(rrrrs rTrqrqtrtrV&special_shifted_chebyshev_polynomial_whermite_polynomial_hcd|d|dS)Nzhermite_polynomial_h_forward(rrrrs rTrqrqys82aSBrVspecial_hermite_polynomial_hhermite_polynomial_hecd|d|dS)Nzhermite_polynomial_he_forward(rrrrs rTrqrq~rprVspecial_hermite_polynomial_helaguerre_polynomial_lcd|d|dS)Nzlaguerre_polynomial_l_forward(rrrrs rTrqrqrprVspecial_laguerre_polynomial_lzdict[str, OverridesData]rctfdtjjtjjtjj tjj fDS)Nc3&K|]}|v ywrlr)rr1res rTrz$is_buffer_removed..s   )anyr7rremoved_bufferskernelinplaced_to_removeres`rTis_buffer_removedrsU  GG # # HH $ $ GG & & HH ' '   rVc4eZdZdZdfd ZddZddZxZS) DeferredLinezHA line that can be 'unwritten' by adding name to V.graph.removed_bufferscVt||||_t|trJyrl)rrrer3r&)rhreliners rTrzDeferredLine.__init__s+  d$45555rVcFt|js |jSyrl)rrerrms rT__call__zDeferredLine.__call__s +99 rVc.t|j|Srl)rre)rhrs rT _new_linezDeferredLine._new_linesDIIt,,rV)reryrryr)rryrvr)r{r|r}r~rrrr&r's@rTrrsR6  -rVrceZdZdddZy) BracesBuffercHtjdfd }|S)Nc3KtD](}jdxjdz c_*t D](}xjdzc_jd*dt D](}jdxjdz c_*tD](}xjdzc_jd*yw)N{rH})ranger_indent)_rrhs rTctxz BracesBuffer.indent..ctxs6] "s# !  "F7^ $ ! s# $ F7^ "s# !  "6] $ ! s# $sC C#)rvIterator[None]) contextlibcontextmanager)rhrrs`` rTindentzBracesBuffer.indents$  " " $ # $u rVN)rH)rrrvz'contextlib.AbstractContextManager[None])r{r|r}rrrVrTrrsrVrc"eZdZUded<ded<y)InplacedBufferryrr other_namesNrrrVrTrrs OrVrc.eZdZUded<dZded<ddZy) ArgNameryreFr is_constexprcB|j|jrdSdS)Nz : tl.constexprr)rerrms rT full_namezArgName.full_names*))$2C2C.LMMLMMrVNrx)r{r|r}rrrrrVrTrrs IL$NrVrceZdZddZy) RemovedArgcy)NREMOVEDrrms rT__str__zRemovedArg.__str__srVNrx)r{r|r}rrrVrTrrsrVrceZdZe ddZddZddZeddZddZddZ ddZ e jf ddZ dd Zdd Zdd Zd d Zd!d Zd"dZd#dZ d$ d%dZ d&dZd'dZd(dZd)dZy)* KernelArgsc~|j|t}t|tr|t |x||<}|S|Srl)rXrr3rr)rodictreresult new_results rT_lookupzKernelArgs._lookupsD */4)A fj )*0#e*'> >E$K*  rVcJi|_i|_i|_i|_g|_yrl) input_buffersoutput_buffersinplace_buffersrworkspace_argsrms rTrzKernelArgs.__init__s)-/ACMO/1 24rVc djdjtt|j|j |j |jgS)NzKernelArgs({})r)formatrmaprrrrrrms rT__repr__zKernelArgs.__repr__sS&& II**++,,    rVc"t|tSrl)r3rrs rT_buffer_is_marked_removedz$KernelArgs._buffer_is_marked_removeds$ ++rVc:tjjr4tjjjj ||}|tjj vsJ|||j vrtt|j |S||jvr'tt|j|jS|jdr|jd|j|S|jd|j|S)Nseedin_ptr)r7rrmutation_real_namerXrrr ryrrrrrrrs rTinputzKernelArgs.inputs 77  77$$77;;D$GD1772228D82 4&& &T0067 7 4'' '(<(R!R  /01[)C03D  ,03D  -! s E3E8cxt|tj|tjj tj |}t|jD]\}}tj||rJ|j}tj|||j|<|j|j|fcS|j|jk7r|j|jk7rJ||jj||j|jdfS)aZ Allocate or extend a workspace buffer of nelem elements. This function manages the allocation of a workspace buffer. It either creates a new WorkspaceArg or extends an existing one. Note: - Calling this function will in-place mutate the args by adding or updating a WorkspaceArg. - The codegen for generating the Python argdefs and call_defs will check this field and allocate the buffer accordingly. - A new argument "ws_ptr" will be present in the generated code. Args: nelem (sympy.Expr): The number of elements to allocate. zero_fill (bool): Whether to initialize the buffer to zero. dtype (torch.dtype): the dtype of the workspace tensor Returns: Tuple[str, str, int]: A tuple containing: - "ws_ptr": A string identifier for the workspace pointer. - "workspace_{i}": agraph level unique identifier for the workspace tensor. - offset: An integer representing the item offset in the workspace. )rrrrrr)rrrr7rget_current_device_or_throwrrurrrrrrr)rhnelemrrargrw existing_argrs rTr,zKernelArgs.workspace*s8'11)<77668#//1   ))<)<= OA|$$\37%++)5):):<)M##A&#.. 0G0GOO''3>>9 ++s~~=  >   ""3'~~s~~q00rVc tjj}t|tj t jdd|jd|j|}|jD]*}|j|jk(s||k(r#J||f|jj||jS)a Lazily allocate a graph-wide semaphores buffer with at least min_size. This is a single buffer shared by all kernels and zero initialized once at graph start. Each kernel must leave the buffer zeroed on exit. Warning: multiple calls to this function will return the same buffer. Args: min_size: the number of int32 semaphores required Returns: name of the semaphores buffer sem_ptr semaphores_r)rrrrrr) r7rrrrrruint32rIrrrr)rhmin_sizecurrent_devicerrs rT semaphoreszKernelArgs.semaphoresYs<<>'66,, $^%8%8$9>;O;O:PQ!  !// @L&&#..8l*?S,,??* @ ""3'~~rVcft|tsJt||ftj|}||j vr|j |S|j j vr0tfd|j j D|j |<S)Nc3FK|]}|jsdyw)rHN)r)rrres rTrz)KernelArgs.seed_offset..}sU1!,,tBTQUs!!)r3rrIrrrrsum)rhrergs ` rT seed_offsetzKernelArgs.seed_offsetus%%;U U';;% e$ DMM !==' ' 4=='') )&U (<(<(>UUVW  $ e rVct|tjsJt||f|jdk(rd|j |<y|j d|j |S)Nrks)r3rSymbolrIrerrrs rTrzKernelArgs.sizesX$ -AT D/AA- 99 "(DMM$ ||D$--66rVct|jj|jj|jjSrl)rrkeysrrrms rT call_nameszKernelArgs.call_namessA    # # %t':':'?'?'A4==CUCUCW  rVc|jj|d}|t|ts |jS|j j|d}|t|ts|S|j j|dS)z; Returns inner name of a given outer name. N)rrXr3rrrr)rhreinplacedrs rTarg_namezKernelArgs.arg_names}''++D$7   8Z(H&& &))--dD9  ":k:+N !!%%dD11rVc|Srlr)rhrrs rT wrap_ptr_argzKernelArgs.wrap_ptr_args rVct|Srlr)rhrs rT wrap_size_argzKernelArgs.wrap_size_args 4yrVNcddlm}|ddlm}|}g}g}g}t|jj D]}t |tr|jd}|j} tjj|} || } |j| d| |j|j|| |j| d|jj!D]\}} ||jvrtjj|} || } |jd| d| |j|j|| |jd| d|j"j!D]\}} ||jvst | tr%tjj|} || } |j| d| |j|j|| |j| d|j$j!D]\}} t |t&j(r@t+|t,j.r&|jd| |jd n+|jd|d | |jd||j|j1|tjj2stjj2j5||j6rJd |||fS) NrH) INDEX_TYPE)rrz* *zconst z const float z const float zWorkspace not supported on CPU )rrrr0rrr3rrrr7rrrrrrrrrrrrUNBACKED_FLOATr wrapper_codeensure_size_computedr) rhdtype_to_cpp_typerr call_argsarg_defs arg_typesrouterinnerr cpp_dtype maybe_inners rT cpp_argdefszKernelArgs.cpp_argdefss  *  $ / ,   t33::<= .H(J/((,E''EGG%%e,E)%0I OOykE73 4   T..ue< =    {!_ - .!..446 4LE5,,,GG%%e,E)%0I OOfYKr%9 :   T..ue< =   vi[2 3 4#'"5"5";";"= . E;,,, ; 0SGG%%e,E)%0I OOykK=9 :   T..ue< =    {!_ - .!MM//1 ALE5%.>++4,ug 67  /& AeW =>  6*!67   T//6 7ww##$$99%@ A&&I(II&I--rVc g}g}g}g}t|jjD]}t|tr|j t |j|j |jd|j tjj|jd|j t|j|jdtjj|jdt|jj|j jD]\}}||jvst|tr%|j t ||j ||j tjj||j t||tjj||j"jD]\}}|j t ||j ||j t%||j t'||tjj(stjj(j+||j,D]m}|j t |j|j |j.|j ||j |j0o||||fS)Nr)rerr)r0rrr3rrrrrr7rrrrrrrrrIrrrrrr) rhrrrprecompile_argsrrrrs rTpython_argdefszKernelArgs.python_argdefss#%! ! /1t33::<= H(J/ OOGH$7$78 9   X11"5 6   QWW..x/C/CB/GH I  " "!,,#//3''++H,@,@,DE  "    $ $ &    % % '  LE5 ,,, 5*0M OOGEN +   U #   QWW..u5 6  " " ''++E2  "!MM//1 ALE5 OOGEN +   U #   T%[ )  " "75%#8 9ww##$$99%@  A&& (C OOGCNN3 4   S^^ ,  " "3 '   SYY '  ( OY>>rVc#Kt|jjD]}t|tr|j D]}|t jjvs|t jjvr<||jvr|j||jf||jvsvtt|j||jfywrl)r0rrr3rrr7rrrrrrr ry)rhrothers rTaliaseszKernelArgs.aliases st33::<= UH(J/!-- UQWW777 ; ;;D...,,U3X5H5HHHD///sD$7$7$>?ATATTT U Us B9C,<0C,ct|jj|ttxr.t|j j|ttSrl)r3rrXrrrrs rT is_removedzKernelArgs.is_removedsK    # #D' 2J N--11$@*M NrVclt}t|jjD]1}t |t r|j |jd3|jjD]5\}}||jvst |t r%|j |7|S)Nr) rr0rrr3rrKrrr)rh live_outsrrrs rTlive_output_bufferszKernelArgs.live_output_buffers s%/\ t33::<= 4H(J/ MM(..r2 3 4!//557 !LE5,,, 5*0M MM%  !rV)rryrz6Union[dict[_T, Union[str, RemovedArg]], dict[_T, str]]rerKrvryrurx)rer rvrr.)rryrryrvrw)rrrrrrrvztuple[str, str, int])rrrvry)reryrgrrvry)rerrvry)rvz Iterator[str])reryrvr)rryrrrvry)r SymbolLikervryrl)rz Optional[dict[torch.dtype, str]]rvz&tuple[list[str], list[str], list[str]])rvz?tuple[list[ArgName], list[str], list[KernelArgType], list[Any]])rvzIterator[tuple[str, str]]r)rvzOrderedSet[str])r{r|r}rrrrrrrrrrr,rrrrrrrrrr r rrrVrTrrs  E    5  ,, @B4:HM{{-1-1,0-19D-1 -1^8 7 2EI4.!A4. /4.l1? H1?f UN rVrc`eZdZdZ d d fd Zd dZd dZd dZd dZd dZ xZ S)r}aDA CSEVariable is just a name for an expression but it is useful to be able to annotate them on a backend dependent basis. To do so, the backends can simply overload `Kernel.create_cse_var` The "CSEVariable.update_on_args" method gives you a hook for annotations See example of TritonCSEVariable in triton.py ct|t|tsJt |||_||_d|_||_||_ yr-) rrr3rrIrebounds use_countrr)rhrerrrrs rTrzCSEVariable.__init__4sL &+.<V <.    rVc|jSrlrrms rTrzCSEVariable.__str__Cs yyrVc,t|jSrl)hashrerms rT__hash__zCSEVariable.__hash__FsDIIrVcXt|txr|j|jk(Srl)r3r}re)rhr s rT__eq__zCSEVariable.__eq__Is!%-I%** 2IIrVcyrlr)rhrerqrs rTupdate_on_argszCSEVariable.update_on_argsLs rVcN|jjd|jdSr)rr{rerms rTrzCSEVariable.__repr__Os$..))*!DII=::rVr)reryrValueRanges[Any]rrrrGrx)rvr)r objectrvr)reryrqr rr rvrw) r{r|r}r~rrrrrrr&r's@rTr}r}-sU(, $  ! %   J ;rVr} AugmentedKeyT)default)boundr!.cPeZdZdZ d ddZddZddZddZddZddZ dd Z dd Z dd Z e jd d ddd  ddZe jddf ddZe jddf ddZy)CSEz Common subexpression eliminationNc||_||_i|_||_|xsi|_|xsi|_|xst j|_t|_ |xsi|_ yrl) rr]_cache name_prefix store_cachereduction_cache itertoolsriter_buffer_idsrinvalidated_stores varname_map)rhrr]r' iter_buffersr(r)r-s rTrz CSE.__init__asm  FH &ALARPR  !r 6B5VY__EV3=<7B7HbrVc6g|jjD]2\}}||vs |j|=|jj|4|r9|jjDcic] \}}||vs ||c}}|_yi|_ycc}}wrl)r(rr,rKr&)rh keep_varsretmprrs rT invalidatezCSE.invalidateys44++1134 2ID#)#$$T*''++D1 2 ,0KK,=,=,?RDAq1 >1a4RDKDKSs 1 B>Bc t||j|j|j|j|j |j |jS)N)rr]r'r.r(r-r))rIrr]r'r+r(r-r)rms rTclonez CSE.clonesPtDz;;;;((--(((( 00  rVc|j}t|j|_t|j|_t|j|_|S)zNReturn a copy of using ScopedDict so changes to *_cache aren't visible in self)r4r+r&r)r()rhnew_cses rT scoped_copyzCSE.scoped_copysH**,#DKK0",T-A-A"B()9)9:rVc"tt|S)z@Override this method to augment cache key with backend specifics)r r rh cache_keys rT augment_keyzCSE.augment_keysM9--rVc@||j|j|<yrlr&r;)rhr:rs rTputzCSE.puts36 D$$Y/0rVc<|j||jvSrl)r;r&r9s rTcontainsz CSE.containss *dkk99rVcX|jj|j|dSrl)r&rXr;r9s rTtry_getz CSE.try_gets"{{t// :DAArVc>|j|j|Srlr=r9s rTrXzCSE.gets{{4++I677rVT)rrf assignmentrrc t|tr |j}|s|sJt|trE|jj ||_|xj dz c_tt|St|tr|j}n1t|tr |j}nt|tsJ|}|j|} ||sd}| s|j|||} |j!|| |rt"j$j&r+t"j$j&j)|dt|trP|r |j+|j,| d|j/||j+|j0| St|trM|sJ|j+|j3|j,| d|j|j0| S|r |j,| d||j0} n||j0} |j+| |rPt4j6j8st4j6j:r|t=dk7r t?|| || S| jj || _| xj dz c_| S)NrHrT) only_oncez =z = rj) r3r4rgr}rtightenrr rr)getvaluer&rryrBnewvarr>r7r current_nodecodegen_originating_inforrsplicer]rrrrrr(r) rhrrrrfrDrrr:rrs rTgeneratez CSE.generatesl dH %::D "" dK (++--f5DK NNa N. . n - I . / IdC( ((Ill9% =E++feU3C HHY $88((HH))BB$CdN3!((DKK=R)@AMM$'$$T[[1: 9 &67%%:$$$++se3tyyk$++'WX4 -""&++se3tfT[[MJ"& }5$$T*#"//KK%22JJ!-/1U:#FC7  ++F3CJ MMQ M rVc|jt|j}tjj ||||}||j |<|Srl)r'rr+r7rcreate_cse_varr-)rhrrrvar_namers rTrIz CSE.newvarsT &&'T-A-A(B'CDhh%%huE%(" rVctj|jvfdtjj |||}||j<|S)NcdS)Nzduplicate name: rrsrTrqzCSE.namedvar..s4DTF2KrV)r _check_valuer-r7rrO)rhrerrrrs ` rTnamedvarz CSE.namedvarsU  (( (*K hh%%dFE5A!$ rV)rrr1NNNN)rryr]ryr'ryr.zOptional[itertools.count[int]]r(z.Optional[MutableMapping[str, CSEVariableType]]r)zr@rBrXrunknownrMrIrTrrVrTr$r$^s* 7;FJ <@III I 5 I D I I:I0  .7:B8$7;#6#6#8'+ $KKRK ! K  KK%KK K^$7;#6#6#8'+ $   %    $7;#6#6#8'+ $  ! %     rVr$c0eZdZdfd ZddZddZxZS)CodeGencTt|tj|_yrl)rrr ExitStack exit_stackrhrs rTrzCodeGen.__init__s $..0rVc:|jj|Srl)r[ __enter__rms rTr^zCodeGen.__enter__s !!# rVc>|jj|||yrl)r[__exit__)rhexc_typeexc_valexc_tbs rTr`zCodeGen.__exit__s   7F;rVrurUrar rbr rcr rvrw)r{r|r}rr^r`r&r's@rTrXrX s1 &  %D D sAAAA AAc#K||}|dux}r t}|j}|j}|j}|j}||_||_||_|j |_ d||_||_||_||_|r |rJdyy#||_||_||_||_|r |rJdwwxYww)Nz$unexpected store inside swap_buffers)r)rkrlrmrrr7) rhlbcbsbdisallow_storesrkrlrmrrs rT swap_bufferszKernel.swap_buffersNs :B Dj (? (!B ,,hh   ??$ F DJ"DL DKDHEEEv2 DJ"DL DKDHEEEv2sA/C 2B6)C *C  C ctrlrrs rTrz Kernel.loadlr rVc|j} |j|_|j||||_S#||_wxYw)z+A load the depends on an index we have read)rkrlr)rhrerr~s rT indirect_loadzKernel.indirect_loados8  DJ99T5)DJDJs "8 Actrlrrs rTrzKernel.store_reductionyr rVctrlrrs rTrz Kernel.store| "!rVcDtt|jdrrrs rTrzKernel.device_assert_asyncrrVctrlrrs rTrzKernel.reduction "!rVctrlr)rhrerrg extra_metas rTpartial_accumulatezKernel.partial_accumulaterrVctrlrrs rTrz Kernel.scans "!rVctrlrrs rTrz Kernel.sortrrVctrlrrms rT var_rangeszKernel.var_rangesr rVct)z3 See [Note: Inductor bucketize op] rrs rTrzKernel.bucketizes "!rVctrlrrms rTassert_functionzKernel.assert_functions!!rVc vt|tr t|}t|tsJt||t|tsJ|t|tsJ|r|rd|d|d|d|d }|d|d|}n|r |d|}|}n |sJ|d|}|}|r d|d|d}|jd|d|dS) Nrz <= rz < rz) | ~(z, "index out of bounds: z"))r3r}ryrIr)rhrrrmaskrk cond_prints rTindirect_assertzKernel.indirect_asserts c; 'c(C#s#.T#Y.#} 5# 666} 5# 666 UugT#eC5E7! ?  %%a&:&:4&@A rVcH|jt| |||yrl)remove_kernel_local_buffersrr`)rhrarbrcrs rTr`zKernel.__exit__s  ((* 7F3rVctjjsytfd|jD}t|jD]c}||j vs||j jvs+j||s>|xjdzc_ j|eD]}||j jvrw|j j|}t|trEtfd|jD}|r|j!||j"j||j%|y)z Any buffers that are both created and have a last use in the same kernel can be removed. Note that V.graph.scheduler can be None when codegening triton template kernels. Nc3tK|]/}|jvrj|j1ywrl) name_to_bufdefining_op_name)rrrs rTrz5Kernel.remove_kernel_local_buffers..s;& i+++  ! !# & 7 7 9& s58rHc3&K|]}|v ywrlr)rrnames_to_removes rTrz5Kernel.remove_kernel_local_buffers.. sKaQ/1Kr)r7rrrrtrsrqr$can_buffer_be_removed_through_fusionrprKrr3rrrremove_inplace_bufferr remove_buffer)rhfused_node_namesrerrdrrs @@rTrz"Kernel.remove_kernel_local_bufferss3GG%%  %& ..&   ,6<++ *DD222 7 77BB*!###D) *$ )Dtyy000ii//5c:.K3??KK..t4''++D1""4( )rVctjd|t|jj|<|j j |y)Nzremove_buffer(%r))rCrRrrqrrrKrs rTrzKernel.remove_buffer s; %t,)0   &   &rVctjd|t|jj|<|j j |y)Nzremoving_inplace_buffer(%r))rCrRrrqrrrKrs rTrzKernel.remove_inplace_buffer$ s9 /6*1 !!$'   &rVc t|ttfr|Dcgc]}|j|c}Stj j j|}t|jd}|Dcic]f}t|tjtjtjtjfr||j j#|h}}t%||Scc}wcc}w)Nc|jSrlr)ss rTrqz(Kernel.rename_indexing..1 s !&&rV)r)r3listtuplerename_indexingr7rrrsorted free_symbolsrr UNBACKED_INTSIZEPRECOMPUTED_SIZErrqrr.)rhrr1sorted_symbols replacementss rTrzKernel.rename_indexing) s edE] +5:;D((+; ;  ))%0 2 28HI$  %%II))''  tyy~~a    %..!<  s C4;A+C9ct|i|Srl)r})rhrqrs rTrOzKernel.create_cse_varA sD+F++rVcZ|y|jj|jS)zC Returns arg name of a given input or output node. N)rqrr)rhrs rTrzKernel.arg_nameD s' <yy!!$--/22rV)NT)rqzOptional[KernelArgs]r{rrvrw)rrFrvrr)rr)rOptional[IndentedBuffer]rrrvrreryrrrvr}reryrrrgr}rvrwrl reryrrrgr}r\r6rvrwr rrrrrr5rg+Union[CSEVariable, tuple[CSEVariable, ...]]rvr) reryrr5rgr}rdict[str, Any]rvrwrrrzUCallable[[tuple[CSEVariable, ...], tuple[CSEVariable, ...]], tuple[CSEVariable, ...]]rtuple[CSEVariable, ...]rvr rrrrrrrrrvr)rvzdict[sympy.Symbol, sympy.Expr]rr}rrrr}rrrrrrrzOptional[CSEVariable]rvr}rx) rzUnion[CSEVariable, str]rrrrrz!Optional[Union[CSEVariable, str]]rvryr)rrrvryrUrdru)reryrvrw)rz;Union[list[sympy.Expr], tuple[sympy.Expr, ...], sympy.Expr]rvr)rqr rr rvr})rrArvr)%r{r|r}rgrr]rhrrrrrrrrrrrrrrrrrrrrrr^r`rrrrrOrr&r's@rTrfrfsM3FC9=I6=PT#/(#/HL#/ #/J&&(,'+ F F %F % F  FF:""SW"" *"3>"FO" "  """& " ; " 5 """&" " # "  ""'" " ( " !""'"(" "  " ! ""4804 " "C "& " $ "  "1 ". "  """37 V $VV V 0 V V<""&0"9="FJ" " "4&)P'' /P/ /0,3rVrfc8eZdZUdZded<dZded<dZded <y) rrz ClassVar[str]rNrrrryops_name)r{r|r}rrrrrrVrTrrM s!"C"#'E 'HcrVrcb ddl}|j|jS#t$rYywxYw)Nr) undefined)jinja2 EnvironmentStrictUndefined ImportError)rs rT jinja2_envrU s?!!,,"   s " ..ceZdZdZe d d dZeddZe ddZdddZe ddZ e ddZ dd Z dd Z dd Zy)KernelTemplatezg Base class for defining kernel templates. Children classes: TritonTemplate, CUDATemplate c|jd}t|dkDr|ddDcgc] }d|z|z|zc}|dddj|Scc}w)NTrHrr) splitlinesrr)source num_indentsindents_spacinglinesrs rTindent_except_firstz"KernelTemplate.indent_except_firsth sd!!$' u:>INqrAE&4<E!"Iwwu~sAct}|ytj|jd<ddlm} |j |S#|$r}Gdd|}|||d}~wwxYw)Nrr)TemplateSyntaxErrorc(eZdZdfd ZddZxZS)IKernelTemplate._template_from_string..DetailedTemplateSyntaxErrorct||j|j|j|j ||_yrl)rrmessagelinenorefilenameoriginal_error)rhrrs rTrzRKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__init__ s>G$&..&--&++&// +9D'rVcFd|jd}|d|jdz }t|jdr|jjj d}|dz }t d|jdz }tt||jdz}t||D]s}||jdz k(rN||dzd ||dz }t|jd s=|d d |jjdz zzd zz }c||dzd||dz }u|S)NzError in template at line  zError message: rz Context: rrrHz: --> columnz rz^ z: ) rrrrrsplitmaxminrrr)rh error_inforstartendrws rTrzQKernelTemplate._template_from_string..DetailedTemplateSyntaxError.__str__ sA#=dkk]"!MJODLL>"DDJt22H= $ 3 3 : : @ @ F"l2 #At{{Q 7!#e*dkkAo>!&uc!2 KA DKK!O3 *QveAhZr.J J #*4+>+>#I$.(/*-1D1D1K1Ka1O*P)Q*/)0%&J !+QveAhZr.J J  K&%rV)rrrvrwrx)r{r|r}rrr&r's@rTDetailedTemplateSyntaxErrorr s 9&rVr)rrrfiltersrr from_string)renvrers rT_template_from_stringz$KernelTemplate._template_from_strings skl ;-;-O-O )*.% 8??6* *"# 8 &.A &B.a0a 7G# 8sAA!AA!ctjjt|tt fr.|Dcic]!}|j |j#c}n |j |jidfd }|Scc}w)Nc>j|}||S|Srl)rX)rer_get_dtype_reallookups rTrz1KernelTemplate._fake_get_dtype..get_dtype s'ZZ%F! "4( (rV)reryrvr)r7rrr3rrr) fake_outsrrrrs @@rT_fake_get_dtypezKernelTemplate._fake_get_dtype sr''++ i$ /AJK#cllncmmo5KF((*I,?,?,ABF ) Ls&B Nc ||_||_yrl)re_hash)rhrers rTrzKernelTemplate.__init__ s  rVc|jS)a  entry point to override for templates to ensure a uid e.g. through a prefix the purpose of this is that every KernelTemplate/ExternKernelChoice is unique in the system, but reproducible e.g. restarting pytorch should yield the same id rrms rTuidzKernelTemplate.uid syyrVc|jS)a source hash for a Template. Templates can optionally provide a src hash to make it easier to cache/validate that a template has not changed from one version to another. Override this if that detection is different for your specific Template )rrms rTsrc_hashzKernelTemplate.src_hash szzrVc Xg}|j|fi|}|t|dk(r|dSy)z Maybe generates a new ChoiceCaller and returns it, or None if generation fails. kwargs: Additional kwargs to be passed to self.generate() to generate a new ChoiceCaller. NrHr)maybe_append_choicer)rhr temp_choicesrs rTchoice_or_nonezKernelTemplate.choice_or_none s> #% ))),A&A >c,/14? "rVc  |j|jdi|y#t$rQ}tj d|t |tj tjk|cYd}~Sd}~wwxYw)a% Maybe generates a new ChoiceCaller and appends it into existing choices. Returns None if success, otherwise returns the error. choices: A list of ChoiceCallers. kwargs: Additional kwargs to be passed to self.generate() to generate a new ChoiceCaller. Nz3Cannot Append Choice: %s. KernelTemplate type is %s) stack_infor) rrMrrCinforIgetEffectiveLevelrPINFO)rhchoicesrrs rTrz"KernelTemplate.maybe_append_choice sn  NN=4==262 3"  HHET 002W\\A   H s!$ A>AA93A>9A>c t)zM Generates a ChoiceCaller instance from the given arguments. r)rhrs rTrMzKernelTemplate.generate s "!rV))rryrrrrrvry)rryrvr )rzUnion[list[Buffer], Buffer]rvzCallable[[str], torch.dtype]rl)reryrrrvrwrx)rvzUnion[str, None])rr rvzOptional[ChoiceCaller])r rzrr rvzOptional[NotImplementedError])rr rvr?)r{r|r}r~rrrrrrrrrrrMrrVrTrra s >?"%8; ,8,8\. %"  ,/ &."rVrc6eZdZdZdZdfd ZddZddZ d ddZ ddZ ddZ ddZ d dd Z dd Z dd Zdd Z dd Z ddZ d dZ d! d"dZxZS)#rz~A ops handler that proxies calls to `kernel` and its handler and returns `CSEVariable`s with correct shape and dtype. cbt|ddlm}||_||_||_y)NrValueRangeAnalysis)rrrr vr_analysisrparent_handler)rhrrrrs rTrzCSEProxy.__init__ s+ /-/ ,rVc d |jgi t|ji}t}t }t t|}d ddk(r dk(r|j |jnydk(rT dk(rOtjjjjtjdj dn dvrt|}|i |i dvr Jd d  fd } tj | |S) Nmaskedrlrj)rlrjrw)rlrjrct ttfr n }t ttfr,t dkDrt dttfr n } dz t|tr+dk(r|j ||_|j ||_tjjjtjj| }|jtjjstjj r)|Jt#tjj||tjj$r) Jt'tjj| tj(r$t+tjj||S)NrrHrjrrr)r3rrrr}rrr7rrrrMrlrrrrrrrrruntime_triton_nan_assertsr) r var_dtype var_shapecsevarrqrrrrer output_idx output_shapes rTdo_csez!CSEProxy._default..do_cse) s lT5M:Z(! lT5M: %)|Au >Z("  !OJ![)e#'AG77?'AGXX\\**  "" +F  ! !$f 5##??&&>> ,,,AHH,,fi@"">>#///AHH,,flC00!((**F3MrV)rzUnion[str, CSEVariable]rvr})_bound_variablerrr!r$r(rrr7 interpreterrJrrXrrpytreetree_map)rhrerqrrg dtype_handler shape_handlershape_opdtype_oprrrrrrs ``` @@@@@rT_defaultzCSEProxy._default s@%%%d>#''e  L 0 0}d3H#T4V4L#T4V4L ' '+ ++ 0 0 dvu--rVc ddlm}ddlm}ddlm}t tj|rtjSt tj|rtjSt tjtrtjStjj j|k(r|jjt |jjt s$Jt#|jj|jjj% tjSt&j(rjt+||r^t- fddDrtjS|rJd d}t/t1||}t3|j4||StjS) z If the variable comes from an FX node, we forward the bound we have already computed Else, if the variable when codegen'ing another op, we try to compute its bounds rr)TritonTemplateKernelrH)CUDATemplateKernelc3:K|]}|jvywrl)r)rrfx_nodes rTrz+CSEProxy._bound_variable..x sV11&Vs) set_indirectrrct|tr |jSt|tjr t |S|Srl)r3r}rrrrr0s rT arg_to_boundz.CSEProxy._bound_variable..arg_to_bound s2a-88O5::.&q>)HrV)r1r rvr )rrselect_algorithmr*cuda.cuda_kernelr+r3r7rrrVr!r1rJrrwdictrIrXrcompute_all_boundsrrrrrr) rhrerqrrr*r+r0 arg_boundsr-s @rTr zCSEProxy._bound_variable] sm 0;8 ahh 4 5&&( ( ahh 2 3&&( ( amm[ 1&&( (--,, >>T !dkk&@&@&Ldkk88$?  **B ?;;--11';;N;N;PQ Q  & &73Et+LV0UVV"**,,  : c,56J274++T2J? ?""$$rVcPt|trtj|}t|tjsJt ||f|j jdkr|rtj|tj|tj}|j jdk\r0tj|d}tj|||}n|}t!j"}|j t!j"k7rt|tj$r|j t!t& dz}t!|j|z|j|z}|j jdk\r"|j t!dt&z} || z}|j(j*j-|j(j.|||j0|j2}|j4j7|||} t9|ro|j jdk\ } t|tj$ xs|j j|k } |j(j;| || | | S)Nrrr)r3rrrrrIrrr2rKrrlongrltrirrVNumberrrrrrMrlrrrrr'r) rhrrrrstmr8 new_bounds neg_boundspos sympy_var assert_lower assert_uppers rTrzCSEProxy.indirect_indexing s dC ==&D$ +?d4j$-??+ ::  a ggc3>>$ #CD::##q(QB))BS1C%,,.Jzz[0022z$ 7U!ZZ+vgr*BB ($$t+Z-=-=-D ::##q(**{1f'==C!+c!1J++//** ##!iiii +C''99#tUK 5 ! # 0 0A 56L)$ ==   4'BL KK $ $YlL QrVc>|jj||||Srl)rrrs rTrzCSEProxy.check_bounds s{{''dE5AArVc||jjjvr)tjjj |t |tjr|jj||S|jjj}||vr||S|jj||}|jdk(r|jxjdz c_ |Sr-)rrrr,r7rsrKrrTMPrr(rrro)rhrerr(outs rTrz CSEProxy.load s 4;;??55 5 HH & & * *4 0 udhh /;;,,T59 9kkoo11 ; t$ $kktU+ ==A  KK A %  rVcl||jjj|<|jjr{|tj j vr^|jjj|}|jD]%}||jjj|<'yyyrl) rrrr(rJr7rname_to_buffer get_output get_mutations)rhrergr other_names rT_update_store_cachezCSEProxy._update_store_cache s,1 ##D) ;; # #0F0F(F++**55d;C!//1 @ :? ++J7 @)G #rVc,|jjj|||j|||tj j vr?|jj|||||jxjdz c_yy)N)r\rH) rrtrKrJr7rrrrprs rTrzCSEProxy.store sx &&**40 <  $ $T5 1 qww.. . KK  dE5t  < KK ! !Q & ! /rVc<|jj||yrl)rrrs rTrzCSEProxy.device_assert_async s ''c2rVc6|jj|yrl)rrrps rTrzCSEProxy.partial_accumulate s& &&-rVc"|jjj||j|||tj j vr<|jxjdz c_|jj|||Syr-) rrtrKrJr7rrrprrs rTrzCSEProxy.store_reduction so &&**40   u- qww.. . KK ! !Q & !;;..tUEB B /rVc||jxjdz c_|jj||||Sr-)rrqrrs rTrzCSEProxy.reduction s4 !!Q&!{{$$UI~uMMrVc<|jj|||Srl)rrrs rTrz CSEProxy.scan s{{ F;;rVc>|jj||||Srl)rrrs rTrz CSEProxy.sort s{{ CCrVc D|jj|||||||S)a [Note: Inductor bucketize op] Inputs: ------- values: the values to be bucketized. boundaries: a tuple containing (a) the name of the boundaries tensor (which must be sorted, unless the sorting tensor is present), (b) the length of the tensor in the last dimension (i.e. the length of one set of boundaries), (c) the number of elements in the underlying storage (i.e. the length of the flattened tensor, ignoring striding), and (d) the stride of the tensor in the last dimension. boundary_indices: indices into a flattened version of the boundaries tensor, of the same size and shape as "values". Each index points to the first element in the set of boundaries to be used for the corresponding value. indexing_dtype: the dtype to use when indexing into the boundaries tensor. This must be int64 or int32. This additionally specifies the dtype of the return value. right: see "Details" below. sorter: an optional tuple containing (a) the name of an optional sorting tensor, used to access unsorted boundaries without reordering the boundaries tensor, and (b) the stride of the tensor in the last dimension. The values in the sorting tensor are used as indices into the *last* dimension of the boundaries tensor, with all other indices matching. The size of the sorting and boundaries tensors must be equivalent. sorter_indices: must be present if the sorting array is present; see "boundary_indices" for the equivalent definition for the boundaries tensor. Output: ------- The buckets each value belongs in, within a given set of boundaries. 0 indicates a position before the first boundary, and len(boundaries_set) represents a position after the last boundary. Details: -------- Given a value and a set of boundaries, calculate the bucket that each value belongs to. This works differently in 1-D and N-D cases. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [0, 4, 4, 8], right=True return = [[ 0, 1, 1, 1], [1, 3, 3, 4]]. for values [[-1, 0, 1, 2], [3, 4, 5, 9]], boundaries [[0, 4], [4, 8]], right=True return = [[ 0, 1, 1, 1], [0, 1, 1, 2]] Note that in the N-D boundaries case, the shape of "values" and "boundaries" must match in every dimension _except_ the last. When right == False, bucket i refers to range (boundaries[i], boundaries[i+1]]. When right == True, bucket i refers to range [boundaries[i], boundaries[i+1]). Boundaries must be non-decreasing, or a sorter must be provided which would re-index offsets in a non-decreasing order (e.g. the second output of torch.sort(offsets)). Otherwise, the result is undefined. )rrrs rTrzCSEProxy.bucketize s1L{{$$         rV)rz Kernel[Any]rzOpsHandler[Any])reryrqztuple[Any, ...]rrrvr )reryrqr rr rvrr) rr}rrrrrrrvrrr)reryrgr}rvrwrlrr)rqr rvrwrrrrrr)r{r|r}r~rerr(r rrrrJrrrrrrrrr&r's@rTrr s D-S.j.%h 5 5%5 5  5  5nBB&0B9=BFJB B "@SW'' *'3>'FO' '3.CNNN& N ; N 5 N <' <  <( < ! <D'D(D D  D ! D 4804N N CN & N $ N  N 1N .N  N rVr)rSryrvrw)NNNN)rryr4rr5rr6rr7rr8Optional[CustomGraphModulePass]r9Optional[ConfigModule]rvrw)rUnion[torch.device, str, None]rvzOrderedSet[BackendFeature])rrUrOr<rvr)rryrvzOptional[SchedulingConstructor])FF)rryrSrrTrrvr)rryrvrS)rryrvrTru)rSequence[sympy.Expr]rrVrrVrvr)rryrrrvrw)rryrvr)rryrqr rr rvr)rr)rrrrrvrw)rr)rrrrGrvrw)rr)rrrvrw)rvryrvrrr)rvr ) __future__rrar dataclassesenumrr*rPrDrrcrer_abcrrrrrtypingr r r r r rrrtyping_extensionsrrrrtorch.fxtorch._prims_commonr torch.utilsrr"torch.utils._config_modulertorch.utils._ordered_setrtorch.utils._sympy.numbersrtorch.utils._sympy.printersr_PythonPrintertorch.utils._sympy.symbolrrrtorch.utils._sympy.value_rangesrrrrr dtype_propagationr! ops_handlerr"r#shape_propagationr$utilsr%r&r'r(r)r*r+r,r-r.r/r0 virtualizedr1r2r3r4r5r6r7collections.abcr8r9r:r;r<custom_graph_passr=rr>r?r@rAr rCrrDrErFrGrrJrKrrIrryrrrp_logginggetArtifactLoggerr{rN getLoggerrCrU dataclassrXrrrrrrrrr KernelArgTyperrrr/r0r1r:r<rKrPrJrVrYr[cacherHrrrbfloat16rfloat16rr8float64int8int16r/rruint16ruint64rrrrrrr)compile IGNORECASEr~ryr{rr3 INT_TO_FLOATrrrrrrrrrr}r rrrReductionCacheKeyr$rXrfrrrrrs0rTrs. "    #   , ?)3/-GOOD:;:    LL$9>>$DD2- B$hy&9%:N%JK23sELL()JF~~//*E g!=  >/ /(  C  Td=ddN** ###   !!! <<<lIw8H,VW ,.).5"5"p:<6;DFAFDF AF8@D>B:>37B B,B/B!= B < B 8 B1B B4 &T &3 *33$3 *35C3 3U @E   " 8< ! -6 ccLU U$U U U; ;&7; ; , NNEKK MM5;;> JJ MM MM JJ KK KK KK KK LL LL LL    u >:,' ' '' 'T: :!0:9D: :2 L  L!0 L9G L  L aBaBH=N=$S1S1l"rzz";2==Q O;#%5z#O;d  - - - 6:`6 ;HH -  `6 ;HH /-  `6;HH /- `6$;HH /- %`60;HH /- 1`6< ;HH *)   =`6L ;HH (0   M`6X ;HH 66>   Y`6h ;HH 1  i`6r ;HH 2  s`6|;HH 1 }`6F;HH 2 G`6P;HH %8$  Q`6^ ;HH &%   _`6j;HH %8  k`6v ;HH &  w`6@;HH + A`6L%;HH 88 ) M`6X%;HH 88 ) Y`6d%;HH 8 )e`6n%;HH 8 )o`6z ;HH '  {`6D ;HH (  E`6N;HH c  O`6^,;HH ? 0_`6h,;HH ? 0i`6t&;HH 9 *u`6~ ;HH *  `6H);HH D -I`6R);HH D -S`6\);HH D -]`6f);HH D -g`6p(;HH C ,q`6z$1;HH L 5${`6D$1;HH L 5$E`6N$1;HH L 5$O`6X$1;HH L 5$Y`6b';HH B +c`6l(;HH C ,m`6v(;HH C ,w`62`F -#-">*Z  NNN ,XXv #;#;L5 +; T  k5c!1223 5l'/=0 1l^ < <p3Wgo.p3f   U"U"pm ~m ;s f