
    tiA                    Z   U d dl mZ d dlZd dlZd dlmZ d dlmZmZ d dl	m
Z
 d dlmZ d dlZd dlZd dlmZ d dlmc mZ d dlmZ d dlmZ d d	lmZmZmZmZmZmZmZm Z m!Z! d d
l"m#Z#m$Z$ d dl%m&Z&m'Z' d dl(m)Z) d dl*m+Z+ d dl,Zd dl-m.Z.m/Z/ d dl0m1Z1 d dl0m2Z2 ejf                  Z3dZ4d Z5d Z6 e7ddddd ddejp                         e7dddddd dde!xr' d ejr                  ju                  d       jv                  v  ejp                  !
       e7d"d#d$ejp                  %       e7d"d&d'd(d)d$ejp                  *       e7d+d,d- ejp                  .       e7d/d0d1d2d$d$ejp                  3      gZ<d4 Z=d5 Z>d6 Z?d7 Z@d8 ZAd9 ZBd: ZCd; ZDd< ZEd= ZFd> ZGd? ZHd@ ZIdA ZJdB ZKdC ZLdD ZMdE ZNdF ZOdG ZPdH ZQdI ZRdJ ZSdK ZTdL ZUdM ZVdN ZWdO ZXdP ZYdQ ZZdR Z[dS Z\dT Z]dU Z^dV Z_dW Z`dX ZadY ZbdZ Zcd[ Zdd\ Zed] Zfd^ Zgd_ Zhd` Zida Zjdb Zkdc Zldd Zmde Zndf Zodg Zpdh Zqdi ZrddjZs	 	 ddkZt	 	 ddlZu	 	 ddmZv	 	 ddnZw	 	 ddoZxddpZyddqZzdr Z{ddsZ|ddtZ}dduZ~dv ZddwZddxZ	 	 ddyZddzZdd{Zes eesd|      exeteyeze|e}e~eeeeeewd}Ze7d~e1f   ed<   g Zd Zg dZg dZ e
ee      D ]9  \  ZZ e7e de edddded$ejp                        Zej                  e       ; eD ]3  Z e7de d d d ed$ejp                        Zej                  e       5 dd d fdd d fdd d fdd d fdd d fdd d fdd d fdd d fdd d fdd d fg
Zddd$iiZe7ee7f   ed<   d$d$d$d$d$dZg dZ e
ee      D ]o  \  \  ZZZZ e7e de eddefdefdedej+                  ed            Zej+                  ei       Zej1                  e       ej                  e       q  G d de      Z G d d      Z G d de      Z G d d      Z G d dee      Z G d dee      ZddZddZd Zy)    )abstractmethodN)deepcopy)reducepartial)product)mul)
_reduction)common_utils)	TestCaseto_gpufreeze_rng_stateis_iterable	gradcheckgradgradcheckset_default_dtypeskipIfTorchDynamoTEST_WITH_ROCM)	TEST_CUDASM90OrLater)_get_numerical_jacobian_iter_tensors)Variable)_TensorOrTensors)UnionAny)Callable)Sequenceh㈵>c                 r    t        | dd       }|#t        j                  t        | dd       dd      }|J |S )N	reductionsizeAverageTF)emit_warning)getattr
_Reductionlegacy_get_stringmresults     g/home/ubuntu/crypto_trading_bot/.venv/lib/python3.12/site-packages/torch/testing/_internal/common_nn.pyget_reductionr*   #   sD    QT*F~--ga.MtbghM    c                 >    t        | dd       }||S t        | dd       S )Nweightweights)r#   r&   s     r)   
get_weightr/   +   s*    Q$'F1i&&r+   Linear)
      ztorch::nn::LinearOptions(10, 8))   r1   c                     t        j                  | |d   j                               |d   j                  dd      j	                  dd      z   S )Nr      r3   r2   )torchmmtviewexpandip_s      r)   <lambda>r@   n   s>    UXXa1%:QqTYYq"=M=T=TUVXY=Z%Z r+   T{Gzt?)module_nameconstructor_argscpp_constructor_args
input_sizereference_fn	with_tf32tf32_precisiondefault_dtype)r1   r2   Fz+torch::nn::LinearOptions(10, 8).bias(false)no_biasc                 P    t        j                  | |d   j                               S Nr   )r7   r8   r9   r<   s      r)   r@   r@   y   s    UXXa1%: r+   gfx94)
rB   rC   rD   rE   descrF   rG   rH   	test_cudarI   RReLU)r5      rQ   F)rB   rE   rO   rI   皙?g?/torch::nn::RReLUOptions().lower(0.1).upper(0.9))r3   r3      with_up_downrB   rC   rD   rE   rN   rO   rI   FlattenrQ      r3   rU   c                 .    t        j                  | d      S Nr5   )r7   flatten)r=   r?   s     r)   r@   r@      s    5==A#6 r+   )rB   rE   rF   rI   CrossMapLRN2d)rU   rA   MbP?rQ   z>torch::nn::CrossMapLRN2dOptions(5).alpha(5e-3).beta(1e-3).k(2)rQ   rZ      ra   )rB   rC   rD   rE   check_gradgradcheck_batched_gradrI   c                      t        t        | d      } t        j                  |      j                  |  j                         S r\   )r   r   r7   randpermr:   double)sizetotals     r)   _rand_tensor_non_equalri      s5    3a E%5>>% %%t,3355r+   c                 B      G  fddt         j                        }|S )Nc                       e Zd Z fdZy))wrap_functional.<locals>.FunctionalModulec                      |i S N )selfargsfnkwargss     r)   forwardz1wrap_functional.<locals>.FunctionalModule.forward   s    t&v&&r+   N)__name__
__module____qualname__rt   )rr   rs   s   r)   FunctionalModulerl      s    	'r+   rx   )nnModule)rr   rs   rx   s   `` r)   wrap_functionalr{      s    '299 ' r+   c            
           t        j                  dd       t        dt         fd      dd d d fdd	t         j                  
      S )Nr1   PoissonNLLLoss_no_reducec                 R    t        j                  | j                  |       d      S Nnoner    )Fpoisson_nll_losstype_asr=   r9   s    r)   r@   z/poissonnllloss_no_reduce_test.<locals>.<lambda>       a((AIIaLFK r+   zaF::poisson_nll_loss(i, t.to(i.options()), F::PoissonNLLLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S Nr1   r7   randro   r+   r)   r@   z/poissonnllloss_no_reduce_test.<locals>.<lambda>   s    B+ r+   _get_input()r   c                 H    | j                         j                  |       z
  S rn   )expr   r=   r?   r9   s     r)   r@   z/poissonnllloss_no_reduce_test.<locals>.<lambda>   s    1557QUU1X#5 r+   Ffullnameconstructorcpp_function_callinput_fncpp_var_maprF   picklerI   r7   randndictr{   rf   r9   s   @r)   poissonnllloss_no_reduce_testr      sM    BA+#KMj+(q15ll
$ 
$r+   c                      t        t        j                  dd      j                  d      j	                  t        j
                               t        dt         fd      dd d d	 fd
ddt        j
                  	      S )N   r1   r   BCELoss_no_reducec                 R    t        j                  | j                  |       d      S r   r   binary_cross_entropyr   r   s    r)   r@   z(bceloss_no_reduce_test.<locals>.<lambda>       a,,Q		!O r+   iF::binary_cross_entropy(i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S Nr   r1   y&1?v?r7   r   clamp_ro   r+   r)   r@   z(bceloss_no_reduce_test.<locals>.<lambda>       B+226:F r+   r   r   c                 `    | j                         z  dz
  d| z
  j                         z  z    S r\   logr   s     r)   r@   z(bceloss_no_reduce_test.<locals>.<lambda>   ,    Q[AEa!e[[]3J%J#K r+   FgǺF?	r   r   r   r   r   rF   r   	precisionrI   )r   r7   r   gttorf   r   r{   r   s   @r)   bceloss_no_reduce_testr      so    R$''*--ell;<A$#OQnF(q1Kll$ $r+   c            
           t        j                  d      j                  d      j                  t         j                         t        dt         fd      dd d d fd	d
t         j                        S )Nro   r   BCELoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r   r   r   s    r)   r@   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>   r   r+   r   c                  L    t        j                  d      j                  dd      S Nro   r   r   r   ro   r+   r)   r@   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>       B..vzB r+   r   r   c                 `    | j                         z  dz
  d| z
  j                         z  z    S r\   r   r   s     r)   r@   z/bceloss_no_reduce_scalar_test.<locals>.<lambda>   r   r+   Fr   )r7   r   r   r   rf   r   r{   r   s   @r)   bceloss_no_reduce_scalar_testr      sd    B1  .A+#OQnB(q1Kll
$ 
$r+   c                  v    t        t        j                  ddt        j                        j	                  d      j                  t        j                               t        j                  dt        j                        t        dt         fd      dd d	 d
 fdddt        j                  	      S )Nr   r1   dtyper   BCELoss_weights_no_reducec                 r    t        j                  | j                  |       j                  |       d      S Nr   r-   r    r   r=   r9   r.   s    r)   r@   z0bceloss_weights_no_reduce_test.<locals>.<lambda>   .    a,,Q		!4;OOA4FRXZ r+   zF::binary_cross_entropy(i, t.to(i.options()), F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S r   r   ro   r+   r)   r@   z0bceloss_weights_no_reduce_test.<locals>.<lambda>   r   r+   r   r   c                 f    | j                         z  dz
  d| z
  j                         z  z    z  S r\   r   )r=   r>   r'   r9   r.   s      r)   r@   z0bceloss_weights_no_reduce_test.<locals>.<lambda>   s2    q1557{a!eA{{}5L'L%MPW%W r+   Fa2U0*3?r   )	r   r7   r   rf   r   r   r   r   r{   r9   r.   s   @@r)   bceloss_weights_no_reduce_testr      s    R5<<8;;A>AA%,,OPAjj5<<0G,#Z[x G(qWEWll r+   c            
      @    t        j                  d      j                  d      j                  t         j                         t        j
                  dt         j                        t        dt         fd      dd dd	  fd
dt         j                        S )Nro   r   r    BCELoss_weights_no_reduce_scalarc                 r    t        j                  | j                  |       j                  |       d      S r   r   r   s    r)   r@   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>   r   r+   zF::binary_cross_entropy(
            i, t.to(i.options()),
            F::BinaryCrossEntropyFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))r   r   c                  L    t        j                  d      j                  dd      S r   r   ro   r+   r)   r@   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>   r   r+   c                 f    | j                         z  dz
  d| z
  j                         z  z    z  S r\   r   r=   r?   r9   r.   s     r)   r@   z7bceloss_weights_no_reduce_scalar_test.<locals>.<lambda>  s1    Q[AEa!e[[]3J%J#Kg#U r+   F)r   r   r   r   r   rF   r   rI   )r7   r   r   r   rf   r   r   r{   r   s   @@r)   %bceloss_weights_no_reduce_scalar_testr      s    B1  .Ajj5<<0G3#Z[k )qWEBUll r+   c                  0    t        t        j                  dd      j                  d      j	                  t        j
                              t        j                          t        dt        fd      dd dd	 fd
ddt        j
                  	      S )Nr   r1   r   BCEWithLogitsLoss_legacy_enumc                 R    t        j                  | j                  |       d      S )NF)r   r    binary_cross_entropy_with_logitsr   r   s    r)   r@   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>  s    a88AIIaLQVW r+   F::binary_cross_entropy_with_logits(
            i, t.to(i.options()), F::BinaryCrossEntropyWithLogitsFuncOptions().reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S r   r   ro   r+   r)   r@   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>  r   r+   r   r   c                 x     |       j                         z  dz
  d |       z
  j                         z  z    S r\   r   r=   r?   sigmoidr9   s     r)   r@   z4bce_with_logistic_legacy_enum_test.<locals>.<lambda>  :    Q)9%9QUq7ST:~FZFZF\<\%\#] r+   F	r   r   r   r   r   rF   rb   r   rI   
r   r7   r   r   r   rf   ry   Sigmoidr   r{   r   r9   s   @@r)   "bce_with_logistic_legacy_enum_testr     sz    R$''*--ell;<AjjlG0#WYkF(q1]ll r+   c                  0    t        t        j                  dd      j                  d      j	                  t        j
                              t        j                          t        dt        fd      dd dd	 fd
ddt        j
                  	      S )Nr   r1   r   BCEWithLogitsLoss_no_reducec                 R    t        j                  | j                  |       d      S r   r   r   s    r)   r@   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>      a88AIIaLTZ[ r+   r   c                  N    t        j                  dd      j                  dd      S r   r   ro   r+   r)   r@   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>"  r   r+   r   r   c                 x     |       j                         z  dz
  d |       z
  j                         z  z    S r\   r   r   s     r)   r@   z2bce_with_logistic_no_reduce_test.<locals>.<lambda>$  r   r+   Fr   r   r   s   @@r)    bce_with_logistic_no_reduce_testr     sz    R$''*--ell;<AjjlG.#[]kF(q1]ll r+   c                      t        j                  d      j                  d      j                  t         j                        t        j                          t        dt        fd      dd dd fd	d
d
t         j                  	      S )Nro   r   "BCEWithLogitsLoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r   r   r   s    r)   r@   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>1  r   r+   r   c                  L    t        j                  d      j                  dd      S r   r   ro   r+   r)   r@   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>4  r   r+   r   r   c                 x     |       j                         z  dz
  d |       z
  j                         z  z    S r\   r   r   s     r)   r@   z9bce_with_logistic_no_reduce_scalar_test.<locals>.<lambda>6  r   r+   Fr   )	r7   r   r   r   rf   ry   r   r   r{   r   s   @@r)   'bce_with_logistic_no_reduce_scalar_testr   +  sr    B1  .AjjlG5#[]kB(q1]ll r+   c                       t        j                  ddt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nr1   r   KLDivLoss_with_target_no_reducec                 R    t        j                  | j                  |       d      S r   r   kl_divr   r   s    r)   r@   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>B      ahhq!))A,&A r+   NF::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(torch::kNone))c                  J    t        j                  dd      j                         S r   r7   r   r   ro   r+   r)   r@   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>D      B+//1 r+   r   r   c                 D    t        d   | j                  |       d      S N	KLDivLossr   r   loss_reference_fnsr   r   s     r)   r@   z6kldivloss_with_target_no_reduce_test.<locals>.<lambda>F  !    {+Aqyy|vN r+   TF	r   r   r   r   r   rF   supports_forward_adr   rI   r7   r   rf   r   r{   r   s   @r)   $kldivloss_with_target_no_reduce_testr   =  sV    

2r.A2#ACj1(q1O ll$ $r+   c                       t        j                  ddt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nr1   r   KLDivLoss_no_reducec                 R    t        j                  | j                  |       d      S r   r   r   s    r)   r@   z*kldivloss_no_reduce_test.<locals>.<lambda>R  r   r+   r   c                  J    t        j                  dd      j                         S r   r   ro   r+   r)   r@   z*kldivloss_no_reduce_test.<locals>.<lambda>T  r   r+   r   r   c                 D    t        d   | j                  |       d      S r   r   r   s     r)   r@   z*kldivloss_no_reduce_test.<locals>.<lambda>V  r   r+   TFr   r   r   s   @r)   kldivloss_no_reduce_testr  M  sV    

2r.A&#ACj1(q1O ll r+   c                       t        j                  dt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nro   r   KLDivLoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r   r   r   s    r)   r@   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>c  r   r+   r   c                  H    t        j                  d      j                         S Nro   r   ro   r+   r)   r@   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>e      B++- r+   r   r   c                 D    t        d   | j                  |       d      S r   r   r   s     r)   r@   z1kldivloss_no_reduce_scalar_test.<locals>.<lambda>g  r   r+   TFr   r   r   s   @r)   kldivloss_no_reduce_scalar_testr  ^  sT    

2U\\*A-#ACj-(q1O ll$ $r+   c                       t        j                  ddt         j                        j                          t	        dt         fd      dd d d fd	d
dt         j                  	      S )Nr1   r   #KLDivLoss_with_log_target_no_reducec                 T    t        j                  | j                  |       dd      S Nr   T)r    
log_targetr   r   s    r)   r@   z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>s      ahhq!))A,&TR r+   _F::kl_div(i, t.to(i.options()), F::KLDivFuncOptions().reduction(torch::kNone).log_target(true))c                  J    t        j                  dd      j                         S r   r   ro   r+   r)   r@   z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>u  r   r+   r   r   c                 D    t        d   | j                  |       d      S NKLDivLoss_log_targetr   r   r   r   s     r)   r@   z:kldivloss_with_log_target_no_reduce_test.<locals>.<lambda>w  #    56q!))A,RXY r+   TFr   r7   r   rf   r   r   r{   r   s   @r)   (kldivloss_with_log_target_no_reduce_testr  n  s_    

2r.224A6#RT{1(q1Z ll$ $r+   c                       t        j                  ddt         j                        j                          t	        dt         fd      dd d d fd	d
dt         j                  	      S )Nr1   r   KLDivLoss_no_reduce_log_targetc                 T    t        j                  | j                  |       dd      S r  r   r   s    r)   r@   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>  r  r+   r  c                  J    t        j                  dd      j                         S r   r   ro   r+   r)   r@   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>  r   r+   r   r   c                 D    t        d   | j                  |       d      S r  r   r   s     r)   r@   z5kldivloss_no_reduce_log_target_test.<locals>.<lambda>  r  r+   TFr   r  r   s   @r)   #kldivloss_no_reduce_log_target_testr"  ~  s_    

2r.224A1#RT{1(q1Z ll r+   c                       t        j                  dt         j                        j                          t	        dt         fd      dd d d fd	d
dt         j                  	      S )Nro   r   %KLDivLoss_no_reduce_scalar_log_targetc                 T    t        j                  | j                  |       dd      S r  r   r   s    r)   r@   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>  r  r+   r  c                  H    t        j                  d      j                         S r  r   ro   r+   r)   r@   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>  r  r+   r   r   c                 D    t        d   | j                  |       d      S r  r   r   s     r)   r@   z<kldivloss_no_reduce_scalar_log_target_test.<locals>.<lambda>  r  r+   TFr   r  r   s   @r)   *kldivloss_no_reduce_scalar_log_target_testr(    s]    

2U\\*..0A8#RT{-(q1Z ll$ $r+   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrQ   rZ   r3   r   L1Loss_no_reducec                 R    t        j                  | j                  |       d      S r   r   l1_lossr   r   s    r)   r@   z'l1loss_no_reduce_test.<locals>.<lambda>      aii199Q<6B r+   PF::l1_loss(i, t.to(i.options()), F::L1LossFuncOptions().reduction(torch::kNone))c                  0    t        j                  ddd      S NrQ   rZ   r3   r7   r   ro   r+   r)   r@   z'l1loss_no_reduce_test.<locals>.<lambda>      Q1- r+   r   r   c                 H    | j                  |       z
  j                         S rn   r   absr   s     r)   r@   z'l1loss_no_reduce_test.<locals>.<lambda>      A		!$4#9#9#; r+   TFr   r7   r   rf   r   r{   r   s   @r)   l1loss_no_reduce_testr9    sU    Aq!5<<0A##BDl-(q1; ll
$ 
$r+   c            
           t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fddd      S )NrQ   rZ   r3   r   L1Loss_no_reduce_complexc                 R    t        j                  | j                  |       d      S r   r,  r   s    r)   r@   z/l1loss_no_reduce_complex_test.<locals>.<lambda>  r.  r+   r/  c                  P    t        j                  dddt         j                        S )NrQ   rZ   r3   r   )r7   r   cdoublero   r+   r)   r@   z/l1loss_no_reduce_complex_test.<locals>.<lambda>  s    Q1EMMB r+   r   r   c                 H    | j                  |       z
  j                         S rn   r5  r   s     r)   r@   z/l1loss_no_reduce_complex_test.<locals>.<lambda>  r7  r+   TF)r   r   r   r   r   rF   r   r   )r7   r   r>  r   r{   r   s   @r)   l1loss_no_reduce_complex_testr@    sN    Aq!5==1A+#BDlB(q1; 	 	r+   c                       t        j                  dt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nro   r   L1Loss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r   r,  r   s    r)   r@   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>  r.  r+   r/  c                  ,    t        j                  d      S r  r2  ro   r+   r)   r@   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>      R r+   r   r   c                 H    | j                  |       z
  j                         S rn   r5  r   s     r)   r@   z.l1loss_no_reduce_scalar_test.<locals>.<lambda>  r7  r+   TFr   r8  r   s   @r)   l1loss_no_reduce_scalar_testrG    sQ    Bell+A*#BDl((q1; ll
$ 
$r+   c                      d} t        j                  | dt         j                  it        dt	        fd      d| ddfdd	d
t         j                  	      S )NrY   r   MSELoss_no_reducec                 R    t        j                  | j                  |       d      S r   r   mse_lossr   r=   targets    r)   r@   z(mseloss_no_reduce_test.<locals>.<lambda>      ajjFNN1$5H r+   WF::mse_loss(i, target.to(i.options()), F::MSELossFuncOptions().reduction(torch::kNone))r   rM  c                 ,    | z
  j                  d      S NrQ   powr=   r?   rN  s     r)   r@   z(mseloss_no_reduce_test.<locals>.<lambda>      AJ#3#3A#6 r+   TF	r   r   r   rE   r   rF   r   r   rI   r8  rE   rN  s    @r)   mseloss_no_reduce_testrY    sY    J[[*9ELL9F$#HJs(F;6 ll
$ 
$r+   c                      d} t        j                  | t         j                        t        dt	        fd      d| ddfdd	d
t         j                  	      S )Nro   r   MSELoss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r   rK  rM  s    r)   r@   z/mseloss_no_reduce_scalar_test.<locals>.<lambda>  rO  r+   rP  r   rM  c                 ,    | z
  j                  d      S rR  rS  rU  s     r)   r@   z/mseloss_no_reduce_scalar_test.<locals>.<lambda>  rV  r+   TFrW  r8  rX  s    @r)   mseloss_no_reduce_scalar_testr^    sV    J[[5<<8F+#HJs(F;6 ll
$ 
$r+   c            
      (    t        t        j                  d      j                         j	                  d      j                         j                               ddi t        dt         fd      dd d	d
 fddt        j                        S )Nr   r1   r    r   NLLLoss_no_reducec                 t    t        j                  | j                  |       j                         d         S Nr    r   r   nll_lossr   longr=   rs   r9   s    r)   r@   z(nllloss_no_reduce_test.<locals>.<lambda>  *    ajjAIIaL$5$5$76+CVW r+   pF::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().reduction(torch::kNone))c                  J    t        j                  dd      j                         S Nr   r1   r   ro   r+   r)   r@   z(nllloss_no_reduce_test.<locals>.<lambda>  r   r+   r   r   c                 ^    t        d   | j                  |       j                         fi S NNLLLossr   r   re  r=   r?   rs   r9   s     r)   r@   z(nllloss_no_reduce_test.<locals>.<lambda>  -    y)!QYYq\->->-@KFK r+   Fr   
r   r7   emptyuniform_r   floorre  r   r{   rf   rs   r9   s   @@r)   nllloss_no_reduce_testrv    s    R))+//399;@@BCA6"F$#WYg1(q1Lll$ $r+   c            
      *    t        t        j                  d      j                         j	                  d      j                         j                               ddd t        dt         fd      dd	 d
d fddt        j                        S )Nr   r1   rQ   r   ignore_indexr    NLLLoss_no_reduce_ignore_indexc           	          t        j                  | j                  |       j                         t	        d         t        d               S Nry  r    rx  r   rd  r   re  intstrrf  s    r)   r@   z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>  @    ajjAIIaL$5$5$7c&Q_J`Fa+.vk/B+CE r+   zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().ignore_index(2).reduction(torch::kNone))c                  J    t        j                  dd      j                         S rj  r   ro   r+   r)   r@   z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>  r   r+   r   r   c                 ^    t        d   | j                  |       j                         fi S rl  rn  ro  s     r)   r@   z5nllloss_no_reduce_ignore_index_test.<locals>.<lambda>  rp  r+   Fr   rq  ru  s   @@r)   #nllloss_no_reduce_ignore_index_testr    s    R))+//399;@@BCA:;&)QF1#EFw1(q1Lll$ $r+   c            
      X    t        t        j                  d      j                         j	                  d      j                         j                               t        j                  d      fd t        dt         fd      dd dd	 fd
dt        j                        S )Nr   r1   c                 ,    j                  |       ddS r   r   r=   r-   s    r)   rs   z.nllloss_no_reduce_weights_test.<locals>.kwargs       ..+&AAr+   NLLLoss_no_reduce_weightsc                 x    t        j                  | j                  |       j                         fi  |       S rn   rc  rf  s    r)   r@   z0nllloss_no_reduce_weights_test.<locals>.<lambda>  ,    ajjAIIaL$5$5$7E6!9E r+   F::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone))c                  h    t        j                  dd      j                  d      j                         S Nr   r1   {Gz?r7   r   addr   ro   r+   r)   r@   z0nllloss_no_reduce_weights_test.<locals>.<lambda>  %    B+//599; r+   r   r=   r9   r-   c                 j    t        d   | j                  |       j                         fi  |       S rl  rn  ro  s     r)   r@   z0nllloss_no_reduce_weights_test.<locals>.<lambda>  1    y)!QYYq\->->-@NF1IN r+   Fr   r   r7   rr  rs  r   rt  re  r   r   r{   rf   rs   r9   r-   s   @@@r)   nllloss_no_reduce_weights_testr    s    R))+//399;@@BCAZZ^FB ,#EG_ <(qFCOll$ $r+   c            
      X    t        t        j                  d      j                         j	                  d      j                         j                               t        j                  d      fd t        dt         fd      dd dd	 fd
dt        j                        S )Nr   r1   c                 .    j                  |       dddS )Nr   rQ   r-   r    ry  r  r  s    r)   rs   z;nllloss_no_reduce_weights_ignore_index_test.<locals>.kwargs(  s     ..+& !# 	#r+   &NLLLoss_no_reduce_weights_ignore_indexc                     t        j                  | j                  |       j                         fi  | j                        S rn   )r   rd  r   re  datarf  s    r)   r@   z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>/  s0    ajjAIIaL$5$5$7J6!&&>J r+   zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone).ignore_index(2))c                  h    t        j                  dd      j                  d      j                         S r  r  ro   r+   r)   r@   z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>3  r  r+   r   r  c                 j    t        d   | j                  |       j                         fi  |       S rl  rn  ro  s     r)   r@   z=nllloss_no_reduce_weights_ignore_index_test.<locals>.<lambda>5  r  r+   Fr   r  r  s   @@@r)   +nllloss_no_reduce_weights_ignore_index_testr  $  s    R))+//399;@@BCAZZ^F# 9#JLo <(qFCOll$ $r+   c            
          t        t        j                  d      j                         j	                  d      j                         j                               t        j                  d      fd t        dt         fd      dt        j                  ddt        j                        j                  d      j                         d	d
 fddt        j                        S )Nr   r1   c                 .    j                  |       dddS )Nr   r6   r  r  r  s    r)   rs   z?nllloss_no_reduce_weights_ignore_index_neg_test.<locals>.kwargs?  s     ..+& "$ 	$r+   *NLLLoss_no_reduce_weights_ignore_index_negc                 x    t        j                  | j                  |       j                         fi  |       S rn   rc  rf  s    r)   r@   zAnllloss_no_reduce_weights_ignore_index_neg_test.<locals>.<lambda>F  r  r+   zF::nll_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::NLLLossFuncOptions().weight(weight.to(i.options())).reduction(torch::kNone).ignore_index(-1))r   r  r   r  c                 j    t        d   | j                  |       j                         fi  |       S rl  rn  ro  s     r)   r@   zAnllloss_no_reduce_weights_ignore_index_neg_test.<locals>.<lambda>L  r  r+   F)r   r   r   inputr   rF   r   rI   )r   r7   rr  rs  r   rt  re  r   r   r{   rf   r  r   r  s   @@@r)   /nllloss_no_reduce_weights_ignore_index_neg_testr  ;  s    R))+//399;@@BCAZZ^F$ =#EGp jjRu||488>BBD(qFCOll$ $r+   c            
          t        t        j                  ddd      j                  d      j	                         j                               ddi t        dt         fd      dd	 d
d fddt        j                        S )NrQ   rU   rZ   r    r   NLLLoss2d_no_reducec                 t    t        j                  | j                  |       j                         d         S rb  rc  rf  s    r)   r@   z*nllloss2d_no_reduce_test.<locals>.<lambda>X  rg  r+   rh  c                  N    t        j                  dddd      j                         S NrQ   rZ   rU   r   ro   r+   r)   r@   z*nllloss2d_no_reduce_test.<locals>.<lambda>[      Aq!Q/335 r+   r   r   c                 ^    t        d   | j                  |       j                         fi S N	NLLLossNdrn  ro  s     r)   r@   z*nllloss2d_no_reduce_test.<locals>.<lambda>]  -    {+Aqyy|/@/@/BMfM r+   Fr   	r   r7   r   r   rt  re  r   r{   rf   ru  s   @@r)   nllloss2d_no_reduce_testr  R  s~    Aq!$((+11388:;A6"F&#WYg5(q1Nll$ $r+   c            
          t        t        j                  ddd      j                  d      j	                         j                               ddd t        dt         fd      d	d
 dd fddt        j                        S )NrQ   rU   rZ   r5   r   rx   NLLLoss2d_no_reduce_ignore_indexc           	          t        j                  | j                  |       j                         t	        d         t        d               S r|  r}  rf  s    r)   r@   z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>i  r  r+   F::nll_loss(
            i, t.to(i.options()).to(torch::kLong), F::NLLLossFuncOptions().ignore_index(1).reduction(torch::kNone))c                  N    t        j                  dddd      j                         S r  r   ro   r+   r)   r@   z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>m  r  r+   r   r   c                 ^    t        d   | j                  |       j                         fi S r  rn  ro  s     r)   r@   z7nllloss2d_no_reduce_ignore_index_test.<locals>.<lambda>o  r  r+   Fr   r  ru  s   @@r)   %nllloss2d_no_reduce_ignore_index_testr  c  s    Aq!$((+11388:;A:;&)QF3#EFw5(q1Nll$ $r+   c            
      @    t        t        j                  ddd      j                  d      j	                         j                               t        j                  d      fd t        dt         fd      dd d	d
 fddt        j                        S )NrQ   rU   rZ   c                 ,    j                  |       ddS r   r  r  s    r)   rs   z0nllloss2d_no_reduce_weights_test.<locals>.kwargsy  r  r+   NLLLoss2d_no_reduce_weightsc                 x    t        j                  | j                  |       j                         fi  |       S rn   rc  rf  s    r)   r@   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>  r  r+   r  c                  N    t        j                  dddd      j                         S r  r   ro   r+   r)   r@   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>  r  r+   r   r  c                 j    t        d   | j                  |       j                         fi  |       S r  rn  ro  s     r)   r@   z2nllloss2d_no_reduce_weights_test.<locals>.<lambda>  1    {+Aqyy|/@/@/BPfQiP r+   Fr   r  r  s   @@@r)    nllloss2d_no_reduce_weights_testr  u  s    Aq!$((+11388:;AZZ]FB .#EG_ 6(qFCQll$ $r+   c            
          t        t        j                  ddddd      j                  d      j	                         j                               ddi t        dt         fd      dd	 d
d fddt        j                        S )NrQ   rU   rZ   r    r   NLLLossNd_no_reducec                 t    t        j                  | j                  |       j                         d         S rb  rc  rf  s    r)   r@   z*nlllossNd_no_reduce_test.<locals>.<lambda>  rg  r+   rh  c                  R    t        j                  dddddd      j                         S r  r   ro   r+   r)   r@   z*nlllossNd_no_reduce_test.<locals>.<lambda>  "    Aq!Q1599; r+   r   r   c                 ^    t        d   | j                  |       j                         fi S r  rn  ro  s     r)   r@   z*nlllossNd_no_reduce_test.<locals>.<lambda>  r  r+   Fr   r  ru  s   @@r)   nlllossNd_no_reduce_testr    s    Aq!Q*..q1779>>@AA6"F&#WYg;(q1Nll$ $r+   c            
          t        t        j                  ddddd      j                  d      j	                         j                               ddd t        dt         fd      d	d
 dd fddt        j                        S )NrQ   rU   rZ   r5   r   rx   NLLLossNd_no_reduce_ignore_indexc           	          t        j                  | j                  |       j                         t	        d         t        d               S r|  r}  rf  s    r)   r@   z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>  r  r+   r  c                  R    t        j                  dddddd      j                         S r  r   ro   r+   r)   r@   z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>  r  r+   r   r   c                 ^    t        d   | j                  |       j                         fi S r  rn  ro  s     r)   r@   z7nlllossNd_no_reduce_ignore_index_test.<locals>.<lambda>  r  r+   Fr   r  ru  s   @@r)   %nlllossNd_no_reduce_ignore_index_testr    s    Aq!Q*..q1779>>@AA:;&)QF3#EFw;(q1Nll$ $r+   c            
      D    t        t        j                  ddddd      j                  d      j	                         j                               t        j                  d      fd t        dt         fd      dd d	d
 fddt        j                        S )NrQ   rU   rZ   c                 ,    j                  |       ddS r   r  r  s    r)   rs   z0nlllossNd_no_reduce_weights_test.<locals>.kwargs  r  r+   NLLLossNd_no_reduce_weightsc                 x    t        j                  | j                  |       j                         fi  |       S rn   rc  rf  s    r)   r@   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>  r  r+   r  c                  R    t        j                  dddddd      j                         S r  r   ro   r+   r)   r@   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>  r  r+   r   r  c                 j    t        d   | j                  |       j                         fi  |       S r  rn  ro  s     r)   r@   z2nlllossNd_no_reduce_weights_test.<locals>.<lambda>  r  r+   Fr   r  r  s   @@@r)    nlllossNd_no_reduce_weights_testr    s    Aq!Q*..q1779>>@AAZZ]FB .#EG_ <(qFCQll$ $r+   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrQ   rZ   r3   r   SmoothL1Loss_no_reducec                 R    t        j                  | j                  |       d      S r   r   smooth_l1_lossr   r   s    r)   r@   z-smoothl1loss_no_reduce_test.<locals>.<lambda>      a&&q!))A,&I r+   jF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone))c                  0    t        j                  ddd      S r1  r2  ro   r+   r)   r@   z-smoothl1loss_no_reduce_test.<locals>.<lambda>  r3  r+   r   r   c                 D    t        d   | j                  |       d      S NSmoothL1Lossr   r   r   r   s     r)   r@   z-smoothl1loss_no_reduce_test.<locals>.<lambda>  !    ~.q!))A,&Q r+   TFr   r8  r   s   @r)   smoothl1loss_no_reduce_testr    s[    Aq!5<<0A)#IK[-(q1R ll$ $r+   c                       t        j                  dt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )Nro   r   SmoothL1Loss_no_reduce_scalarc                 R    t        j                  | j                  |       d      S r   r  r   s    r)   r@   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>  r  r+   r  c                  ,    t        j                  d      S r  r2  ro   r+   r)   r@   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>  rE  r+   r   r   c                 D    t        d   | j                  |       d      S r  r   r   s     r)   r@   z4smoothl1loss_no_reduce_scalar_test.<locals>.<lambda>  r  r+   TFr   r8  r   s   @r)   "smoothl1loss_no_reduce_scalar_testr    sW    Bell+A0#IK[((q1R ll$ $r+   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrQ   rZ   r3   r   SmoothL1Loss_betac                 T    t        j                  | j                  |       dd      S )Nr         ?r    betar  r   s    r)   r@   z(smoothl1loss_beta_test.<locals>.<lambda>  s     a&&q!))A,&sS r+   zoF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone), 0.5)c                  0    t        j                  ddd      S r1  r2  ro   r+   r)   r@   z(smoothl1loss_beta_test.<locals>.<lambda>  r3  r+   r   r   c                 F    t        d   | j                  |       dd      S )Nr  r   r  r  r   r   s     r)   r@   z(smoothl1loss_beta_test.<locals>.<lambda>  s$    ~.q!))A,&WZ[ r+   TFr   r8  r   s   @r)   smoothl1loss_beta_testr    s[    Aq!5<<0A$#SU`-(q1\ ll$ $r+   c                       t        j                  dddt         j                         t        dt	         fd      dd d	 d
 fdddt         j                  	      S )NrQ   rZ   r3   r   SmoothL1Loss_zero_betac                 T    t        j                  | j                  |       dd      S )Nr   r   r  r  r   s    r)   r@   z-smoothl1loss_zero_beta_test.<locals>.<lambda>  s     a&&q!))A,&qQ r+   zmF::smooth_l1_loss(
            i, t.to(i.options()), F::SmoothL1LossFuncOptions().reduction(torch::kNone), 0)c                  0    t        j                  ddd      S r1  r2  ro   r+   r)   r@   z-smoothl1loss_zero_beta_test.<locals>.<lambda>  r3  r+   r   r   c                 F    t        d   | j                  |       dd      S )Nr  r   r   r  r   r   s     r)   r@   z-smoothl1loss_zero_beta_test.<locals>.<lambda>  s$    ~.q!))A,&WXY r+   TFr   r8  r   s   @r)   smoothl1loss_zero_beta_testr    s[    Aq!5<<0A)#QS^-(q1Z ll$ $r+   c                       t        j                  ddd       t        dt         fd      dd d d	 fd
ddt         j                  	      S )NrQ   rZ   r3   HuberLoss_deltac                 T    t        j                  | j                  |       dd      S )Nr   r  r    delta)r   
huber_lossr   r   s    r)   r@   z&huberloss_delta_test.<locals>.<lambda>  s    all1aiilfCP r+   znF::huber_loss(
            i, t.to(i.options()), F::HuberLossFuncOptions().reduction(torch::kNone).delta(0.5))c                  0    t        j                  ddd      S r1  r2  ro   r+   r)   r@   z&huberloss_delta_test.<locals>.<lambda>  r3  r+   r   r   c                 F    t        d   | j                  |       dd      S )N	HuberLossr   r  r  r   r   s     r)   r@   z&huberloss_delta_test.<locals>.<lambda>  s$    {+Aqyy|vUXY r+   TFr   r   r   s   @r)   huberloss_delta_testr    sU    Aq!A"#PRc-(q1Z ll$ $r+   c                       t        j                  d      j                          t        dt	         fd      dd d d fdd	d
d
	      S )Nro   !MultiLabelMarginLoss_0d_no_reducec                 n    t        j                  | j                  |       j                         d      S r   r   multilabel_margin_lossr   re  r   s    r)   r@   z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>  (    a..q!))A,2C2C2EQWX r+   F::multilabel_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultilabelMarginLossFuncOptions().reduction(torch::kNone))c                  ,    t        j                  d      S r  r2  ro   r+   r)   r@   z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>!  rE  r+   r   r   c                 t    t        d   | j                  j                  |       j                         d      S NMultiLabelMarginLossr   r   r   r  r   re  r   s     r)   r@   z8multilabelmarginloss_0d_no_reduce_test.<locals>.<lambda>#  1    56q!&&..:K:P:P:R^de r+   TF)	r   r   r   r   r   rF   check_sum_reductionrb   r   )r7   zerosre  r   r{   r   s   @r)   &multilabelmarginloss_0d_no_reduce_testr    sU    BA4#XZt((q1f  r+   c                      t        t        j                  d      j                  d      j	                         j                                t        dt         fd      dd d d fdd	d
d
t        j                  
      S )Nr1   !MultiLabelMarginLoss_1d_no_reducec                 n    t        j                  | j                  |       j                         d      S r   r  r   s    r)   r@   z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>/  r	  r+   r
  c                  ,    t        j                  d      S r   r2  ro   r+   r)   r@   z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>2  rE  r+   r   r   c                 t    t        d   | j                  j                  |       j                         d      S r  r  r   s     r)   r@   z8multilabelmarginloss_1d_no_reduce_test.<locals>.<lambda>4  r  r+   TF
r   r   r   r   r   rF   r  rb   r   rI   r  r   s   @r)   &multilabelmarginloss_1d_no_reduce_testr  *  su    B##B'--/4467A4#XZt((q1f ll$ $r+   c                  L    t        t        j                  t        j                  dd      j	                  d      j                  d      j                         j                         d             t        dt         fd      d	d
 d d fddddt        j                  
      S )NrU   r1   g         r6   minMultiLabelMarginLoss_index_negc                 n    t        j                  | j                  |       j                         d      S r   r  r   s    r)   r@   z5multilabelmarginloss_index_neg_test.<locals>.<lambda>A  r	  r+   r
  c                  .    t        j                  dd      S NrU   r1   r2  ro   r+   r)   r@   z5multilabelmarginloss_index_neg_test.<locals>.<lambda>D      Q+ r+   r   r   c                 t    t        d   | j                  j                  |       j                         d      S r  r  r   s     r)   r@   z5multilabelmarginloss_index_neg_test.<locals>.<lambda>F  r  r+   TFr  )r   r7   clampr   r  r   rt  re  r   r{   rf   r   s   @r)   #multilabelmarginloss_index_neg_testr&  <  s    UZZ2.2237;;B?EEGLLNTVWXA1#XZt+(q1f ll$ $r+   c                      t        t        j                  dd      j                  d      j	                         j                                t        dt         fd      dd d d fd	d
ddt        j                  
      S )NrU   r1   MultiLabelMarginLoss_no_reducec                 n    t        j                  | j                  |       j                         d      S r   r  r   s    r)   r@   z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>S  r	  r+   r
  c                  .    t        j                  dd      S r"  r2  ro   r+   r)   r@   z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>V  r#  r+   r   r   c                 t    t        d   | j                  j                  |       j                         d      S r  r  r   s     r)   r@   z5multilabelmarginloss_no_reduce_test.<locals>.<lambda>X  r  r+   TFr  r  r   s   @r)   #multilabelmarginloss_no_reduce_testr,  N  sx    Ar"&&r*002779:A1#XZt+(q1f ll$ $r+   c                  >    t        t        j                  d      j                  d      j	                  t        j
                        j                  d      j                  d             t        dt         fd      dd d	 d
 fdddt        j
                  	      S )Nr1   r   rQ   r5   HingeEmbeddingLoss_no_reducec                 R    t        j                  | j                  |       d      S r   r   hinge_embedding_lossr   r   s    r)   r@   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>e  r   r+   zvF::hinge_embedding_loss(
            i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().reduction(torch::kNone))c                  ,    t        j                  d      S r   r2  ro   r+   r)   r@   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>h  rE  r+   r   r   c                 D    t        d   | j                  |       d      S )NHingeEmbeddingLossr   r   r   r   s     r)   r@   z3hingeembeddingloss_no_reduce_test.<locals>.<lambda>j  s#    34Q		!PVW r+   TF	r   r   r   r   r   rF   r  r   rI   
r   r7   r   r   r   rf   mul_subr   r{   r   s   @r)   !hingeembeddingloss_no_reduce_testr9  `  s    R##A&))%,,7<<Q?CCAFGA/#OQa((q1X ll$ $r+   c                  >    t        t        j                  d      j                  d      j	                  t        j
                        j                  d      j                  d             t        dt         fd      dd d	 d
 fdddt        j
                  	      S )Nr1   r   rQ   r5   #HingeEmbeddingLoss_margin_no_reducec                 T    t        j                  | j                  |       dd      S Nr  r   marginr    r0  r   s    r)   r@   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>v  s!    a,,Q		!STZ[ r+   zF::hinge_embedding_loss(
            i, t.to(i.options()), F::HingeEmbeddingLossFuncOptions().margin(0.5).reduction(torch::kNone))c                  ,    t        j                  d      S r   r2  ro   r+   r)   r@   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>y  rE  r+   r   r   c                 F    t        d   | j                  |       dd      S )Nr4  r  r   r>  r   r   s     r)   r@   z:hingeembeddingloss_margin_no_reduce_test.<locals>.<lambda>{  s%    34Q		!S\bc r+   TFr5  r6  r   s   @r)   (hingeembeddingloss_margin_no_reduce_testrB  q  s    R##A&))%,,7<<Q?CCAFGA6#[]m((q1d ll$ $r+   c                       t        j                  ddt         j                         t        dt	         fd      dd d d fd	d
dt         j                  	      S )NrU   r   SoftMarginLoss_no_reducec                 R    t        j                  | j                  |       d      S r   )r   soft_margin_lossr   r   s    r)   r@   z/softmarginloss_no_reduce_test.<locals>.<lambda>  r   r+   znF::soft_margin_loss(
            i, t.to(i.options()), F::SoftMarginLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S )NrU   r2  ro   r+   r)   r@   z/softmarginloss_no_reduce_test.<locals>.<lambda>  s    Q* r+   r   r   c                 D    t        d   | j                  |       d      S )NSoftMarginLossr   r   r   r   s     r)   r@   z/softmarginloss_no_reduce_test.<locals>.<lambda>  s"    /0AIIaLFS r+   TFr   r8  r   s   @r)   softmarginloss_no_reduce_testrJ    sY    Aq-A+#KM]*(q1T ll$ $r+   c                       t        j                  dd      j                  d      j                          t	        dt         fd      dd d d	 fd
ddt         j                  	      S )NrU   r1   rQ   "MultiLabelSoftMarginLoss_no_reducec                 R    t        j                  | j                  |       d      S r   r   multilabel_soft_margin_lossr   r   s    r)   r@   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>  s    a33Aqyy|vV r+   zF::multilabel_soft_margin_loss(
            i, t.to(i.options()), F::MultilabelSoftMarginLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S r"  r2  ro   r+   r)   r@   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>  r#  r+   r   r   c                     | j                         j                         z  dz
  |  j                         j                         z  z    j                  d      | j                  d      z  S Nr5   dimr   r   sumrg   r   s     r)   r@   z9multilabelsoftmarginloss_no_reduce_test.<locals>.<lambda>  s`    199;??$$A1"1C1C1E'EEFKKPQKRUVU[U[\]U^^ r+   Fr   r7   r   r   rt  r   r{   rf   r   s   @r)   'multilabelsoftmarginloss_no_reduce_testrX    sg    

1ba &&(A5#VXg+(q1_ll$ $r+   c                      t        j                  dd      j                  d      j                          t        j                  d      t	        dt         fd      dd d d	 fd
dddt         j                  
      S )NrU   r1   rQ   *MultiLabelSoftMarginLoss_weights_no_reducec                 r    t        j                  | j                  |       j                  |       d      S r   rN  r   s    r)   r@   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>  s.    a33Aqyy|;B??1;MY_a r+   zF::multilabel_soft_margin_loss(
            i, t.to(i.options()),
            F::MultilabelSoftMarginLossFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                  .    t        j                  dd      S r"  r2  ro   r+   r)   r@   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>  r#  r+   r   r   c                     | j                         j                         z  dz
  |  j                         j                         z  z    z  j                  d      | j                  d      z  S rR  rU  r   s     r)   r@   zAmultilabelsoftmarginloss_weights_no_reduce_test.<locals>.<lambda>  se    199;??$$A1"1C1C1E'EEFPUUZ[U\_`_e_efg_hh r+   TFr  rW  r   s   @@r)   /multilabelsoftmarginloss_weights_no_reduce_testr^    s~    

1ba &&(AjjnG=#abq ,(qWEi ll$ $r+   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )NrU   r2   MultiMarginLoss_no_reducec                 n    t        j                  | j                  |       j                         d      S r   r   multi_margin_lossr   re  r   s    r)   r@   z0multimarginloss_no_reduce_test.<locals>.<lambda>  '    a))!QYYq\->->-@FS r+   F::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultiMarginLossFuncOptions().reduction(torch::kNone))c                  .    t        j                  dd      S r"  r2  ro   r+   r)   r@   z0multimarginloss_no_reduce_test.<locals>.<lambda>  r#  r+   r   r   c                 t    t        d   | j                  j                  |       j                         d      S NMultiMarginLossr   r   r  r   s     r)   r@   z0multimarginloss_no_reduce_test.<locals>.<lambda>  1    01!QVV^^A5F5K5K5MY_` r+   TFr  r7   r   r   rt  re  r   r{   rf   r   s   @r)   multimarginloss_no_reduce_testrl    sp    

1!""$))+A,#SUo+(q1a ll$ $r+   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )Nr5   r2   MultiMarginLoss_1d_no_reducec                 n    t        j                  | j                  |       j                         d      S r   rb  r   s    r)   r@   z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>  rd  r+   re  c                  ,    t        j                  d      S r   r2  ro   r+   r)   r@   z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>  rE  r+   r   r   c                 t    t        d   | j                  j                  |       j                         d      S rh  r  r   s     r)   r@   z3multimarginloss_1d_no_reduce_test.<locals>.<lambda>  rj  r+   TFr  rk  r   s   @r)   !multimarginloss_1d_no_reduce_testrr    sp    

1!""$))+A/#SUo((q1a ll$ $r+   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )Nro   r2   ,multimarginloss_1d_input_0d_target_no_reducec                 n    t        j                  | j                  |       j                         d      S r   rb  r   s    r)   r@   zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>  rd  r+   re  c                  ,    t        j                  d      S r   r2  ro   r+   r)   r@   zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>  rE  r+   r   r   c                 t    t        d   | j                  j                  |       j                         d      S rh  r  r   s     r)   r@   zCmultimarginloss_1d_input_0d_target_no_reduce_test.<locals>.<lambda>  rj  r+   TFr  rk  r   s   @r)   1multimarginloss_1d_input_0d_target_no_reduce_testrx    sp    

21##%**,A?#SUo((q1a ll$ $r+   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )NrU   r2   MultiMarginLoss_p_no_reducec                 p    t        j                  | j                  |       j                         dd      S )NrQ   r   r>   r    rb  r   s    r)   r@   z2multimarginloss_p_no_reduce_test.<locals>.<lambda>  s*    a))!QYYq\->->-@AQWX r+   zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong), F::MultiMarginLossFuncOptions().p(2).reduction(torch::kNone))c                  N    t        j                  dd      j                  dd      S )NrU   r1   r  gGz?)r7   r   r   ro   r+   r)   r@   z2multimarginloss_p_no_reduce_test.<locals>.<lambda>  s    Q+224B r+   r   r   c                 v    t        d   | j                  j                  |       j                         dd      S )Nri  rQ   r   r|  r  r   s     r)   r@   z2multimarginloss_p_no_reduce_test.<locals>.<lambda>  s4    01!QVV^^A5F5K5K5MQR^de r+   TFr  rk  r   s   @r)    multimarginloss_p_no_reduce_testr    sp    

1!""$))+A.#XZtB(q1f ll$ $r+   c                       t        j                  d      j                  d      j                         j	                          t        dt         fd      dd d d fd	d
ddt         j                  
      S )NrU   r2    MultiMarginLoss_margin_no_reducec                 p    t        j                  | j                  |       j                         dd      S r=  rb  r   s    r)   r@   z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>  s*    a))!QYYq\->->-@X^_ r+   zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::MultiMarginLossFuncOptions().margin(0.5).reduction(torch::kNone))c                  .    t        j                  dd      S r"  r2  ro   r+   r)   r@   z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>
  r#  r+   r   r   c                 v    t        d   | j                  j                  |       j                         dd      S )Nri  r  r   r>  r  r   s     r)   r@   z7multimarginloss_margin_no_reduce_test.<locals>.<lambda>  s6    01!QVV^^A5F5K5K5M9<P r+   TFr  rk  r   s   @r)   %multimarginloss_margin_no_reduce_testr    st    

1!""$))+A3#_aT ,(q1P !ll$ $r+   c                  B    t        j                  d      j                  d      j                         j	                          t        j                  dt         j
                        t        dt         fd      dd d	 d
 fddddt         j
                  
      S )NrU   r2   r1   r   !MultiMarginLoss_weights_no_reducec                     t        j                  | j                  |       j                         j                  |       d      S r   rb  r   s    r)   r@   z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>  s6    a))!QYYq\->->-@YZI[4:< r+   zF::multi_margin_loss(
            i, t.to(i.options()).to(torch::kLong),
            F::MultiMarginLossFuncOptions().weight(weights.to(i.options())).reduction(torch::kNone))c                  .    t        j                  dd      S r"  r2  ro   r+   r)   r@   z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>   r#  r+   r   r   c                 v    t        d   | j                  j                  |       j                         d      S )Nri  r   r   r  r   s     r)   r@   z8multimarginloss_weights_no_reduce_test.<locals>.<lambda>"  s6    01!QVV^^A5F5K5K5M9@FT r+   TFr  )r7   r   r   rt  re  rf   r   r{   r   s   @@r)   &multimarginloss_weights_no_reduce_testr    s    

1!""$))+Ajj5<<0G4#<=h ,(qWET !ll!$ $r+   c                     d } ||       }t        |t        j                        r|gn|}t               5   || j	                  d      cddd       S # 1 sw Y   yxY w)zReference function for modules supporting no batch dimensions.

    The module is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    c                     t        | t        t        f      r| D cg c]  }|j                  d       c}S | j                  d      S c c}w rL   
isinstancelisttuple	unsqueezeinpr9   s     r)   unsqueeze_inpz0single_batch_reference_fn.<locals>.unsqueeze_inp1  =    cD%=),/0qAKKN00}}Q 1   Ar   N)r  r7   Tensorr   squeeze)r  
parametersmoduler  single_batch_inputs        r)   single_batch_reference_fnr  +  sa     
 'u-1;<NPUP\P\1],-cu		 6)*22156 6 6s   AAc                  =   t        j                          g t               t               t	               t               t               t               t               t               t               t               t               t               t               t               t!               t#               t%               t'               t)               t+               t-               t/               t1               t3               t5               t7               t9               t;               t=               t?               tA               tC               tE               tG               tI               tK               tM               tO               tQ               tS               tU               tW               tY               t[               t]               t_               ta               tc               te               tg               ti        dddddddtj        jl                        ti        ddd	ddd
ddtj        jl                  	      ti        ddddddddtj        jl                  	      ti        ddddddddtj        jl                  	      ti        ddddddddtj        jl                  	      ti        ddddddddtj        jl                  	      ti        dddddddd      ti        dd ddddtj        jl                         ti        d!d" d#d$dddtj        jl                  %      ti        d&d' d(ddddtj        jl                  %      ti        d)d* d+ddddtj        jl                  %      ti        d,d- d.ddddtj        jl                  %      ti        d/d0 d1ddddtj        jl                  %      ti        d2d3 d4dd5ddtj        jl                  6      ti        d2d7d8d9dd:ddtj        jl                  	      ti        d2d;d<d9dd=ddtj        jl                  	      ti        d>d? d@ddAddtj        jl                  6      ti        dBdCdDdEddddtj        jl                  F	      ti        dBdGdHdIddJdddtj        jl                  K
      ti        dBdLdMdIddNdddtj        jl                  K
      ti        dBdOdPdQdd=dddtj        jl                  K
      ti        dBdRdSdTdd:dddUtj        jl                  K
      ti        dBdCdDdVddddW      ti        dXdY dZd[ddddUtj        jl                  \	      ti        d]d^ dZd[dddUtj        jl                  _      ti        d`da dbdcdddtj        jl                  %      ti        ddde dfdcdddtj        jl                  %      ti        dgdh didcdddtj        jl                  %      ti        djdkdlddmdddtj        jl                  n	      ti        djdodpdqdd=dddtj        jl                  K
      ti        djdrdsdqdd:dddtj        jl                  K
      ti        dtdu dvdwddddtj        jl                  \	      ti        dxdy dzd{ddtj        jl                         ti        d|d} d~d{ddtj        jl                         ti        dd dd{ddtj        jl                         ti        dd dd{ddtj        jl                         ti        dd ddddtj        jl                         ti        ddddddddtj        jl                  F	      ti        dddddd:dddtj        jl                  K
      ti        dddddddddtj        jl                  K
      ti        dddddd
dddtj        jl                  K
      ti        dddddddddtj        jl                  K
      ti        dddddddd      ti        dd ddddddtj        jl                  \	      ti        dd ddddtj        jl                         ti        dd ddddtj        jl                         ti        dd dddddtj        jl                  %      ti        dd dddddtj        jl                  %      ti        dd dddddtj        jl                  %      ti        dddddddtj        jl                        ti        dddddd=ddtj        jl                  	      ti        ddddtj        jl                        ti        ddddtn        dtj        jl                        ti        dddd dd      ti        ddddń dtj        jl                  tq        dƫ      ǫ      ti        ddddȄ ddtj        jl                  tq        dƫ      ʫ      ti        dddd̈́ ddtj        jl                  ϫ      ti        ddddЄ ddtj        jl                  ϫ      ti        ddddӄ ddtj        jl                  ϫ      ti        ddddׄ ddtj        jl                  ϫ      ti        ddڄ dd܄ dtj        jl                  ݫ      ti        dd߄ dd dtj        jl                  ݫ      ti        dd dd dtj        jl                  ݫ      ti        dd dd dd      ti        d dd ddd      ti        ddddtj        jl                        ti        ddddtj        jl                        ti        ts        tt        jv                  dd d      ddddtj        jl                        ti        ts        tt        jv                  dd d      dddd       ti        ts        tt        jv                  dd d      ddddtj        jl                        ti        ts        tt        jv                  d dd      ddddtj        jl                        ti        ts        tt        jv                  dd dd      d	dd
dtj        jl                        ti        ts        tt        jv                  dd dd      ddddtj        jl                        ti        ts        tt        jv                  d ddd      ddddtj        jl                        ti        ts        tt        jv                  dd dd      d	ddd       ti        ts        tt        jv                  dd dd      ddddtj        jl                        ti        ts        tt        jv                  d ddd      ddddtj        jl                        ti        ts        tt        jv                  dd d      ddddtj        jl                        ti        ts        tt        jv                  dd d      ddddtj        jl                        ti        ts        tt        jv                  dd d      ddddtj        jl                        ti        ts        tt        jv                  d dd      d dd!dtj        jl                        ti        ts        tt        jv                  dd d      dd"d#d       ti        ts        tt        jv                  dd d$d      d%dd&dtj        jl                        ti        ts        tt        jv                  dd d$d      d%d"d'd       ti        ts        tt        jv                  d(d d$d      d)d*d+dtj        jl                        ti        ts        tt        jv                  d dd$d      d,dd-dtj        jl                        ti        ts        tt        jv                  d d.d$d      d/dd0dtj        jl                        ti        ts        tt        jv                  d d1d$d      d2dd3dtj        jl                        ti        ts        tt        jv                  d(d d$d      d4dd5dtj        jl                        ti        ts        tt        jv                  d d1d$d      d6dd7dtj        jl                        ti        ts        tt        jv                  dd d8d      d9dd:dtj        jl                        ti        ts        tt        jv                  dd d8d      d9d"d;d       ti        ts        tt        jv                  d(d d8d      d<d*d=dtj        jl                        ti        ts        tt        jv                  d dd8d      d>dd?dtj        jl                        ti        ts        tt        jv                  d d.d8d      d@ddAdtj        jl                        ti        ts        tt        jv                  d d1d8d      dBddCdtj        jl                        ti        ts        tt        jv                  d(d d8d      dDddEdtj        jl                        ti        ts        tt        jv                  d d1d8d      dFddGdtj        jl                        ti        ts        tt        jv                  dd d      dHdIdJdtj        jl                        ti        ts        tt        jv                  dd d      dHdKdLd       ti        ts        tt        jv                  dMd d      dNdOdPdtj        jl                        ti        ts        tt        jv                  d dd      dQdIdRdtj        jl                        ti        ts        tt        jv                  dd dSd      dTdIdUdtj        jl                        ti        ts        tt        jv                  dd dSd      dTdKdVd       ti        ts        tt        jv                  dWd dSd      dXdYdZdtj        jl                        ti        ts        tt        jv                  d d[dSd      d\dd]d^dtj        jl                  _      ti        ts        tt        jv                  dWd dSd      d`dYdadtj        jl                        ti        ts        tt        jv                  d d[dSd      dbdOdcd^dtj        jl                  _      ti        ts        tt        jx                  dde      dfdgdhdtj        jl                        ti        ts        tt        jx                  ditj        jz                  j      dkdgdlddtj        jl                  m      ti        ts        tt        jx                  die      dndodpdtj        jl                        ti        ts        tt        jx                  die      dndqdrdtj        jl                        ti        ts        tt        jx                  ditj        jz                  j      dkdqdsddtj        jl                  m      ti        ts        tt        jx                  dte      dudvdwddtj        jl                  x      ti        ts        tt        jx                  dye      dzdvd{ddtj        jl                  x      ti        ts        tt        jx                  dde      dfd|d}dd~      ti        ts        tt        j|                  dde      ddgddtj        jl                        ti        ts        tt        j|                  die      ddoddtj        jl                        ti        ts        tt        j|                  die      ddqddtj        jl                        ti        ts        tt        j|                  dte      ddvddtj        jl                        ti        ts        tt        j|                  dye      ddvddtj        jl                        ti        ts        tt        j|                  dte      dd|dd       ti        dd ddddtj        jl                        ti        dd ddddtj        jl                        ti        dd dddtn        dtj        jl                        ti        dd ddddtj        jl                        ti        dd ddddtj        jl                        ti        dd ddtn        ddtj        jl                        ti        dddd|ddtj        jl                        ti        dd tj        jl                        ti        dd dtj        jl                        ti        dd dtj        jl                        ti        dddd dtj        jl                        ti        dd tn        dtj        jl                        ti        dddddddddtj        jl                  
      ti        dddddtt        j~                  fdddddt        rdndtj        jl                  	      ti        dddÐdĄ ddddtj        jl                  ū	      ti        dddddtt        j~                  fdƐdǄ ddddtj        jl                  ū	      ti        dȐddddddtt        j                  fdɐdʄ dddt        rdndtj        jl                  ū	      ti        d͐dΐdϐdЄ dф dddtj        jl                  ҫ	      ti        dӐdԐdՐdtn        dtj        jl                  ׫      ti        dؐdِdtk        j                  ddg      fdtn        dtj        jl                  ׫      ti        dg dܢddfdސdddddd	      } t        g dg dd      D ]  \  }}dD ]  }|dyk(  r|dk(  rt        t        di|diz               }ddj                  t        t        |            z   dz   }dd|z  z   }dt        d |D              z   }| j                  ti        d| dddydyd|didid|f	d| d| d| d||d| dddtj        jl                  
               g d}i ddtj        jl                  dddd1idddtj        jl                  ddddtj        jl                  dddtj        jl                  dddtj        jl                  id dtj        jl                  iddtj        jl                  iddtj        jl                  iddtj        jl                  iddtj        jl                  iddtj        jl                  iddtj        jl                  iddtj        jl                  iddtj        jl                  id	dtj        jl                  id
dtj        jl                  idtj        jl                  idtj        jl                  idtj        jl                  idtj        jl                  idtj        jl                  idtj        jl                  idtj        jl                  id}	|D ]L  }
ti        |
dtn        dd      }|	j                  |
i       }|j                  |       | j                  |       N | S (  NConv1d)r3   rU   rZ   z!torch::nn::Conv1dOptions(4, 5, 3))rQ   r3   r1   TrA   )rB   rC   rD   rE   cudnnrG   rH   rI   )r3   rU   rZ   rQ   z+torch::nn::Conv1dOptions(4, 5, 3).stride(2)stride)	rB   rC   rD   rE   r  rN   rG   rH   rI   )r3   rU   rZ   r5   r5   z6torch::nn::Conv1dOptions(4, 5, 3).stride(1).padding(1)pad1r  )r3   rU   rU   r5   rQ   z6torch::nn::Conv1dOptions(4, 5, 5).stride(1).padding(2)pad2)r3   r3   rZ   r5   r5   z6torch::nn::Conv1dOptions(4, 4, 3).stride(1).padding(1))r5   r3   r5   	pad1size1)r3   r3   rU   r5   rQ   z6torch::nn::Conv1dOptions(4, 4, 5).stride(1).padding(2)	pad2size1)r   r3   r1   
zero_batch)rB   rC   rD   rE   r  rN   rG   rH   Conv1d_dilatedc                  4    t        j                  dddd      S )Nr3   rU   rZ   rQ   kernel_sizedilationry   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>      		!QA J r+   z-torch::nn::Conv1dOptions(4, 5, 3).dilation(2))r   r   rD   rE   rG   rH   rI   Conv1d_groupsc                  4    t        j                  dddd      S )Nr3   ra   rZ   rQ   r  groupsr  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>      		!QAa H r+   z+torch::nn::Conv1dOptions(4, 6, 3).groups(2))rQ   r3   ra   )r   r   rD   rE   r  rG   rH   rI   Conv1d_pad_validc                  4    t        j                  dddd      S )Nr3   rU   rZ   validpaddingr  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    		!Q7 C r+   z8torch::nn::Conv1dOptions(4, 5, 3).padding(torch::kValid)Conv1d_pad_samec                  4    t        j                  dddd      S )Nr3   rU   rZ   samer  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>      		!Q6 B r+   z7torch::nn::Conv1dOptions(4, 5, 3).padding(torch::kSame)Conv1d_pad_same2c                  4    t        j                  dddd      S )Nr3   rU   r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r  r+   z7torch::nn::Conv1dOptions(4, 5, 4).padding(torch::kSame)Conv1d_pad_same_dilatedc                  6    t        j                  ddddd      S )Nr3   rU   r  rQ   r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    		!Q6A N r+   zCtorch::nn::Conv1dOptions(4, 5, 3).padding(torch::kSame).dilation(2)ConvTranspose1dc                  8    t        j                  dddddd      S )NrZ   r3   rZ   r5   r5   )r  r  r  output_paddingry   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s     2 21aQt]^os t r+   zQtorch::nn::ConvTranspose1dOptions(3, 4, 3).stride(3).padding(1).output_padding(1))r5   rZ      )r   r   rD   r  rE   rG   rH   rI   )rZ   r3   rZ   rQ   r5   r5   r5   Fztorch::nn::ConvTranspose1dOptions(3, 4, 3)
                                    .stride(2).padding(1).output_padding(1).groups(1).bias(false))r5   rZ   ra   rJ   )	rZ   r3   rZ   rQ   r5   r5   r5   TrQ   ztorch::nn::ConvTranspose1dOptions(3, 4, 3)
                                    .stride(2).padding(1).output_padding(1).groups(1).bias(true).dilation(2)dilatedConvTranspose1d_groupsc            	      :    t        j                  ddddddd      S )	Nr3   ra   rZ   r  r5   r  rQ   )r  r  r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s      2 21a4QRcgpq r r+   ztorch::nn::ConvTranspose1dOptions(4, 6, 3)
                                    .stride(3).padding(1).output_padding(1).groups(2))rQ   r3   r  Conv2d)rZ   r3   rZ   rQ   z&torch::nn::Conv2dOptions(3, 4, {3, 2}))rQ   rZ   r  rU   )	rB   rC   rD   rE   r  check_with_long_tensorrG   rH   rI   )rZ   r3   rZ   rZ   rQ   rQ   z5torch::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2})r`   strided)
rB   rC   rD   rE   r  rN   r  rG   rH   rI   )rZ   r3   r  r  r5   r5   zEtorch::nn::Conv2dOptions(3, 4, {3, 3}).stride({2, 2}).padding({1, 1})r  )rZ   rQ   r  r  r  r  zVtorch::nn::Conv2dOptions(3, 2, {3, 3}).stride({2, 2}).padding({1, 1}).dilation({2, 2}))rQ   rZ   r2   r2   )rZ   r3   r  r5   r   r5   r5   Fztorch::nn::Conv2dOptions(3, 4, {3, 2})
                                    .stride(1).padding(0).dilation(1).groups(1).bias(false))rQ   rZ   ra   rU   gQ?)r   rZ   r  rU   )rB   rC   rD   rE   r  rN   r  rG   Conv2d_groupsc                  4    t        j                  dddd      S Nr3   ra   r  rQ   r  ry   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>l      		!Qq A r+   z0torch::nn::Conv2dOptions(4, 6, {3, 2}).groups(2))rQ   r3   ra   rU   )	r   r   rD   rE   r  r  rG   rH   rI   Conv2d_groups_thnnc                  4    t        j                  dddd      S r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>w  r  r+   )r   r   rD   rE   r  rG   rH   rI   Conv2d_pad_validc                  4    t        j                  dddd      S )NrQ   r3   rZ   r3   r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    		!Q H r+   z=torch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kValid))rQ   rQ   ra   rU   Conv2d_pad_samec                  4    t        j                  dddd      S )NrQ   r3   r  r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    		!Q G r+   z<torch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kSame)Conv2d_pad_same_dilatedc                  6    t        j                  ddddd      S )NrQ   r3   r  r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    		!QQR S r+   zHtorch::nn::Conv2dOptions(2, 4, {3, 4}).padding(torch::kSame).dilation(2)ConvTranspose2d)rZ   r3   rZ   r  r5   r  ztorch::nn::ConvTranspose2dOptions(3, 4, 3)
                                    .stride({3, 2}).padding(1).output_padding({1, 1}))r5   rZ   r  ra   )	rB   rC   rD   r  rE   r  rG   rH   rI   )	rZ   r3   rZ   rQ   rZ   r5   r  r5   Fr  a`  torch::nn::ConvTranspose2dOptions(3, 4, 3)
                                    .stride({2, 3})
                                    .padding(1)
                                    .output_padding({1, 1})
                                    .groups(1)
                                    .bias(false)
                                    .dilation({2, 2}))r5   rZ   ra   r  )rZ   r3   rZ   r  r5   r  r5   Fztorch::nn::ConvTranspose2dOptions(3, 4, 3)
                                    .stride({2, 3}).padding(1).output_padding({1, 1}).groups(1).bias(false)ConvTranspose2d_groupsc                  4    t        j                  dddd      S )NrQ   r3   r  r  )ry   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s     2 21a J r+   z9torch::nn::ConvTranspose2dOptions(2, 4, {2, 3}).groups(2))r5   rQ   r3   rU   Conv2d_depthwisec                  4    t        j                  dddd      S )Nr3   r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r  r+   z0torch::nn::Conv2dOptions(4, 4, {3, 3}).groups(4))rQ   r3   ra   ra    Conv2d_depthwise_with_multiplierc                  4    t        j                  dddd      S )Nr3   r2   r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r  r+   z0torch::nn::Conv2dOptions(4, 8, {3, 3}).groups(4)Conv2d_depthwise_stridedc                  6    t        j                  ddddd      S )Nr3   r  r  )r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    		!Qva P r+   z?torch::nn::Conv2dOptions(4, 4, {3, 3}).stride({2, 2}).groups(4)Conv2d_depthwise_paddedc                  6    t        j                  ddddd      S )Nr3   r  r  )r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    		!Qq Q r+   z@torch::nn::Conv2dOptions(4, 4, {3, 3}).padding({1, 1}).groups(4)Conv2d_depthwise_dilatedc                  6    t        j                  ddddd      S )Nr3   r  )r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    		!QPQ R r+   zAtorch::nn::Conv2dOptions(4, 4, {2, 2}).dilation({2, 2}).groups(4))rQ   r3   rU   rU   Conv3d)rQ   rZ   rQ   rZ   rQ   z)torch::nn::Conv3dOptions(2, 3, {2, 3, 2}))r5   rQ   r3   rU   r3   g?)rQ   rZ   rQ   rZ   r3   r5   r   r5   r5   Fztorch::nn::Conv3dOptions(2, 3, {2, 3, 4})
                                    .stride(1).padding(0).dilation(1).groups(1).bias(false))r5   rQ   rZ   r3   rU   )rQ   rZ   )r5   r5   r5   r5   r   r5   r5   F1x1x1_no_biasF)rZ   r3   rQ   rQ   z+torch::nn::Conv3dOptions(3, 4, 2).stride(2))rQ   rZ   rU   rU   rU   )rZ   r3   rQ   rQ   r5   z6torch::nn::Conv3dOptions(3, 4, 2).stride(2).padding(1)stride_padding)rZ   r3   r  z)torch::nn::Conv3dOptions(3, 4, {2, 3, 4}))r   rZ   rZ   r3   rU   )rB   rC   rD   rE   r  r  rN   rG   Conv3d_groupsc                  4    t        j                  dddd      S )NrQ   r3   rZ   r  ry   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>I  r  r+   z+torch::nn::Conv3dOptions(2, 4, 3).groups(2)Conv3d_dilatedc                  4    t        j                  dddd      S )NrZ   r3   rQ   r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>T  r  r+   z-torch::nn::Conv3dOptions(3, 4, 2).dilation(2)Conv3d_dilated_stridedc                  6    t        j                  ddddd      S )NrZ   r3   rQ   )r  r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>]  s    		!QARS T r+   z7torch::nn::Conv3dOptions(3, 4, 2).dilation(2).stride(2)Conv3d_pad_validc                  4    t        j                  dddd      S )NrZ   r3   r  r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>f  s    		!Q	7 K r+   z@torch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kValid))rQ   rZ   ra   rU   r3   Conv3d_pad_samec                  4    t        j                  dddd      S )NrZ   r3   r  r  r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>p  s    		!Q	6 J r+   z?torch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kSame)Conv3d_pad_same_dilatedc                  6    t        j                  ddddd      S )NrZ   r3   r  r  rQ   r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>z  s    		!Q	6TU V r+   zKtorch::nn::Conv3dOptions(3, 4, {2, 3, 4}).padding(torch::kSame).dilation(2)ConvTranspose3dz2torch::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2}))rB   rC   rD   r  rE   rG   rH   rI   )	rQ   rZ   r  r5   r   r   r5   T)rQ   rQ   rQ   ztorch::nn::ConvTranspose3dOptions(2, 3, {2, 3, 2})
                                    .stride(1).padding(0).output_padding(0).groups(1).bias(true).dilation({2, 2, 2}))	rB   rC   rD   r  rE   rN   rG   rH   rI   ReplicationPad3d))r5   rQ   rZ   rZ   rQ   r5   z6torch::nn::ReplicationPad3dOptions({1, 2, 3, 3, 2, 1}))rQ   rZ   rQ   rQ   rQ   )rB   rC   rD   rE   rI   )rZ   rQ   rQ   rQ   no_batch_dim)rB   rC   rD   rE   rF   rN   rI   c            	      V    t        j                  dddddt         j                  d      S )NrQ   rZ   T)r   requires_grad)r7   r   
complex128ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s"    UZZ1aAU=M=M]ab r+   complex)rB   rC   rD   r   	skip_halfrN   	Embedding)r3   rZ   z!torch::nn::EmbeddingOptions(4, 3)c                  l    t        j                  ddt         j                        j                  d      S NrQ   rZ   r   r3   r7   rr  re  random_ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  "    U[[AUZZ@HHK r+   z0https://github.com/pytorch/pytorch/issues/117971)rB   rC   rD   r   rb   rI   	decoratorc                      t        j                  ddt         j                        j                  d      j	                  dd      S Nr5   i   r   r3   r  r7   rr  re  r  r;   ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  1    U[[CuzzBJJ1MTTUVX[\ r+   discontiguous)rB   rC   rD   r   rb   rN   rI   r  EmbeddingBagz$torch::nn::EmbeddingBagOptions(4, 3)c                  l    t        j                  ddt         j                        j                  d      S r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r  r+   mean)rB   rC   rD   r   rb   rN   rI   c                      t        j                  ddt         j                        j                  d      j	                  dd      S r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r  r+   )r3   rZ   N       @FrV  ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kSum)c                  l    t        j                  ddt         j                        j                  d      S r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r  r+   rV  )r3   rZ   Nr  Fmaxztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kMax)c                  l    t        j                  ddt         j                        j                  d      S r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r  r+   r  EmbeddingBag_mean_padding_idxc                  2    t        j                  ddd      S )Nr3   rZ   r5   padding_idxry   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    1! D r+   z3torch::nn::EmbeddingBagOptions(4, 3).padding_idx(1)c                  |    t        j                  t        j                  d      t        j                  d      g      S NrZ   r7   stackre   ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  &    U[[%..*;U^^A=N)OP r+   )r   r   rD   r   rb   rI   EmbeddingBag_sum_padding_idxc            	      :    t        j                  ddd dddd      S )Nr3   rZ   r  FrV  r5   r  r!  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>      1dBuZ[ \ r+   ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kSum).padding_idx(1)c                  |    t        j                  t        j                  d      t        j                  d      g      S r#  r$  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r&  r+   EmbeddingBag_max_padding_idxc            	      :    t        j                  ddd dddd      S )Nr3   rZ   r  Fr  r5   r  r!  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r)  r+   ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .max_norm(std::nullopt).norm_type(2.).scale_grad_by_freq(false).mode(torch::kMax).padding_idx(1)c                  |    t        j                  t        j                  d      t        j                  d      g      S r#  r$  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r&  r+   EmbeddingBag_sparsec                  P    t        j                  dddt        j                        S )Nr3   rZ   T)sparser   )ry   r  r7   rf   ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    1T V r+   ztorch::nn::EmbeddingBagOptions(4, 3)
                                    .sparse(true)._weight(torch::rand({4, 3}).to(torch::kFloat64))c                  L    t        j                  d      j                  dd      S NrQ   r5   r7   re   repeatro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>      U^^A.55a; r+   )r   r   rD   r   rb   has_sparse_gradientsc                  P    t        j                  ddt        j                  d      S )Nr3   rZ   T)r   r0  )ry   r	  r7   rf   ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  s    Qd S r+   z_torch::nn::EmbeddingOptions(4, 3).sparse(true)._weight(torch::rand({4, 3}).to(torch::kFloat64))c                  L    t        j                  d      j                  dd      S r2  r3  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>  r5  r+   Embedding_sparse)r   rD   r   r   rb   r6  PixelShuffler  z!torch::nn::PixelShuffleOptions(3))r5   	   r3   r3   PixelUnshufflez#torch::nn::PixelUnshuffleOptions(3))r5   r5      r=  r=  nearest)rg   scale_factormodezF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12})).scale_factor(std::nullopt).mode(torch::kNearest))r5   rQ   r3   interpolate_nearest_1d)r   cpp_options_argsrE   r   r   rI   )r   rQ   r3   interpolate_nearest_1d_zero_dim)r   rB  rE   r   r   )r=  )r5   rQ   rZ   interpolate_nearest_tuple_1dg      @zF::InterpolateFuncOptions()
                                .size(std::nullopt).scale_factor(std::vector<double>({4.})).mode(torch::kNearest)interpolate_nearest_scale_1dlinear)rg   r?  r@  align_cornersa  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kLinear)
                                .align_corners(false)interpolate_linear_1d)r3   a  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kLinear)
                                .align_corners(false)interpolate_linear_tuple_1da  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4.}))
                                .mode(torch::kLinear)
                                .align_corners(false)interpolate_linear_scale_1dinterpolate_linear_1d_zero_dima  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kLinear)
                                .align_corners(true)#interpolate_linear_1d_align_cornersa  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4.}))
                                .mode(torch::kLinear)
                                .align_corners(true))interpolate_linear_scale_1d_align_cornersrQ   zF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({2, 2}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r5      r5   r5   %interpolate_nearest_2d_launch_configszF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r5   rQ   r3   r3   interpolate_nearest_2d)r=     zF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 16}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r5   rQ   rZ   r3   interpolate_nearest_tuple_2dzF::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4.}))
                                .mode(torch::kNearest)interpolate_nearest_scale_2d)r   rQ   r3   r3   interpolate_nearest_2d_zero_dimbilineara  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBilinear)
                                .align_corners(false)interpolate_bilinear_2d interpolate_bilinear_2d_zero_dim)r3   ra   a	  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBilinear)
                                .align_corners(false))r5   rQ   rQ   rZ   interpolate_bilinear_tuple_2da
  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4.}))
                                .mode(torch::kBilinear)
                                .align_corners(false)interpolate_bilinear_scale_2d)r  r  a
  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 2.}))
                                .mode(torch::kBilinear)
                                .align_corners(false)*interpolate_bilinear_scale_tuple_shared_2d)r        ?a
  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBilinear)
                                .align_corners(false)*interpolate_bilinear_scale_tuple_skewed_2da  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBilinear)
                                .align_corners(true)+interpolate_bilinear_tuple_2d_align_cornersa	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBilinear)
                                .align_corners(true)8interpolate_bilinear_scale_tuple_skewed_2d_align_cornersbicubica
  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBicubic)
                                .align_corners(false)interpolate_bicubic_2dinterpolate_bicubic_2d_zero_dima  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBicubic)
                                .align_corners(false)interpolate_bicubic_tuple_2da	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4.}))
                                .mode(torch::kBicubic)
                                .align_corners(false)interpolate_bicubic_scale_2da	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 2.}))
                                .mode(torch::kBicubic)
                                .align_corners(false))interpolate_bicubic_scale_tuple_shared_2da	  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBicubic)
                                .align_corners(false))interpolate_bicubic_scale_tuple_skewed_2da  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kBicubic)
                                .align_corners(true)*interpolate_bicubic_tuple_2d_align_cornersa  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({2., 1.}))
                                .mode(torch::kBicubic)
                                .align_corners(true)7interpolate_bicubic_scale_tuple_skewed_2d_align_cornerszF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r5   rQ   r3   r3   r3   interpolate_nearest_3d)r   rQ   r3   r3   r3   interpolate_nearest_3d_zero_dim)r=  rQ  rQ  zF::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 16, 16}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kNearest))r5   rQ   rZ   r3   r3   interpolate_nearest_tuple_3dzF::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({4., 4., 4.}))
                                .mode(torch::kNearest)interpolate_nearest_scale_3d	trilineara  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({12, 12, 12}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kTrilinear)
                                .align_corners(false)interpolate_trilinear_3d!interpolate_trilinear_3d_zero_dim)r3   ra   ra   a  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kTrilinear)
                                .align_corners(false))r5   rQ   rQ   rZ   rZ   interpolate_trilinear_tuple_3dg      @a  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({3., 3., 3.}))
                                .mode(torch::kTrilinear)
                                .align_corners(false)interpolate_trilinear_scale_3dr   )r   rB  rE   r   r   r   rI   a  F::InterpolateFuncOptions()
                                .size(std::vector<int64_t>({4, 6, 6}))
                                .scale_factor(std::nullopt)
                                .mode(torch::kTrilinear)
                                .align_corners(true),interpolate_trilinear_tuple_3d_align_cornersa  F::InterpolateFuncOptions()
                                .size(std::nullopt)
                                .scale_factor(std::vector<double>({3., 3., 3.}))
                                .mode(torch::kTrilinear)
                                .align_corners(true),interpolate_trilinear_scale_3d_align_cornersr6   rS  zF::SoftmaxFuncOptions(-1))rQ   rN  softmax_lastdimr5   )rT  r   z/F::SoftmaxFuncOptions(1).dtype(torch::kFloat64)softmax_lastdim_dtype)r   rB  rE   r   r   rO   rI   zF::SoftmaxFuncOptions(1))rQ   rN  rQ   rQ   softmax_spatial_special)rQ   rQ   r3   r3   softmax_spatialsoftmax_spatial_dtyper   zF::SoftmaxFuncOptions(0)rY   softmax_functional_dim0)r   rB  rE   r   rO   r   rI   rZ   zF::SoftmaxFuncOptions(3)softmax_functional_dim3ro   softmax_functional_scalar)r   rB  rE   r   rO   r   zF::LogSoftmaxFuncOptions(-1)log_softmax_lastdimzF::LogSoftmaxFuncOptions(1)log_softmax_spatial_speciallog_softmax_spatialzF::LogSoftmaxFuncOptions(0)log_softmax_dim0zF::LogSoftmaxFuncOptions(3)log_softmax_dim3log_softmax_scalarUnfoldc                  2    t        j                  dddd      S )Nr  r  r   r   ry   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>v	  s    		&&&& I r+   zPtorch::nn::UnfoldOptions({2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1}))rQ   r3   rZ   rZ   )r   r   rD   rE   rb   rO   rI   Foldc                  4    t        j                  ddddd      S Nr  r  r  r  ry   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	       O r+   zVtorch::nn::FoldOptions({3, 3}, {2, 2}).dilation({1, 1}).padding({0, 0}).stride({1, 1}))rQ   rQ  r3   Fold_no_batch_dim_inputc                  4    t        j                  ddddd      S r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	  r  r+   )rQ  r3   )r   r   rD   rE   rb   refrO   rI   Unfold_int_inputc                  2    t        j                  dddd      S )NrQ   r5   r   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	  s    		!Q1 5 r+   z<torch::nn::UnfoldOptions(2).dilation(1).padding(0).stride(1)Fold_int_inputc                  4    t        j                  ddddd      S NrZ   rQ   r5   r   r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	      1aA 6 r+   z=torch::nn::FoldOptions(3, 2).dilation(1).padding(0).stride(1)Fold_no_batch_dim_int_inputc                  4    t        j                  ddddd      S r  r  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	  r  r+   )r   r   rD   rE   r  rb   rO   rI   rP   rR   rT   with_up_down_scalarrW   PairwiseDistancec                  Z    t        j                  dd      t        j                  dd      fS Nr1   r2   r2  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	  !    ekk"a0%++b!2DE r+   )rB   r   rI   c                  Z    t        j                  dd      t        j                  dd      fS )Nr1   r5   r2   r2  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	  r  r+   broadcast_lhs)rB   r   rN   rI   c                  Z    t        j                  dd      t        j                  dd      fS )Nr1   r2   r5   r2  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	  s!    ekk"a0%++a2CD r+   broadcast_rhs)g      ?r   TzDtorch::nn::PairwiseDistanceOptions().p(1.5).eps(1e-05).keepdim(true)c                  Z    t        j                  dd      t        j                  dd      fS r  r2  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	  r  r+   with_non_default_args)rB   rC   rD   r   rN   rI   c                  V    t        j                  d      t        j                  d      fS )Nr2   r2  ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	  s    ekk!nekk!n= r+   )rB   r   rF   rN   rI   TransformerEncoderLayer)r3   rQ   rQ          ztorch::nn::TransformerEncoderLayerOptions(4, 2)
                                    .dim_feedforward(16)
                                    .dropout(0.0)r  relu_activationrS   )
rB   rC   rD   rE   rN   rG   rH   rc   rb   rI   r3   r2   r  ztorch::nn::TransformerEncoderLayerOptions(4, 2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)
                                    .activation(torch::kGELU)gelu_activationg{Gz?)	rB   rC   rD   rE   rb   rN   rG   rH   rI   TransformerDecoderLayer)r3   rQ   r2   r  ztorch::nn::TransformerDecoderLayerOptions(4, 2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)c                  ^    t        j                  ddd      t        j                  ddd      fS NrZ   r3   rQ   r   ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>	  %    ejjAq15::aA3FG r+   )	rB   rC   rD   r   rb   rN   rG   rH   rI   ztorch::nn::TransformerDecoderLayerOptions(4, 2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)
                                    .activation(torch::kGELU)c                  ^    t        j                  ddd      t        j                  ddd      fS r  r   ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>
  r  r+   Transformera  torch::nn::TransformerOptions()
                                    .d_model(4)
                                    .nhead(2)
                                    .num_encoder_layers(2)
                                    .num_decoder_layers(2)
                                    .dim_feedforward(8)
                                    .dropout(0.0)
                                    .activation(torch::kReLU)c                      t        j                  ddd      t        j                  ddd      t        j                  dd      fS r  r   ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>
  s4    ejjAq15::aA3F

STVWHXY r+   multilayer_codergQ?r0   )rZ   rU   ztorch::nn::LinearOptions(3, 5)c                  ,    t        j                  d      S r#  r   ro   r+   r)   r@   z&get_new_module_tests.<locals>.<lambda>%
  s    UZZ] r+   c                     t        j                  | j                  dd      |d   j                               j                  d      |d   z   S )Nr5   r6   r   )r7   r8   r:   r9   r<   s      r)   r@   z&get_new_module_tests.<locals>.<lambda>&
  s=    !&&B-1)J)O)OPR)SVWXYVZ)Z r+   )	rB   rC   rD   r   rF   rN   rG   rH   rI   rX   z5torch::nn::FlattenOptions().start_dim(-3).end_dim(-1))r6   )rZ   r3   rU   )rB   rD   rC   rE   rF   rN   rI   	Unflattenz'torch::nn::UnflattenOptions(-2, {2, 2})	LayerNorm)8   r  r  r   zMtorch::nn::LayerNormOptions({56, 56, 56}).eps(1e-5).elementwise_affine(false))r3   r  r  r  3d_no_affine_large_feature)	rB   rC   rD   rE   r  
check_evalgradcheck_fast_mode
check_halfrN   )reflectcircular	replicater  )ztorch::kReflectztorch::kCircularztorch::kReplicateztorch::kZerosstrictr  {z, }r  r  c              3   &   K   | ]	  }|d z     ywr5   Nro   .0r>   s     r)   	<genexpr>z'get_new_module_tests.<locals>.<genexpr>[
  s     (@1Q(@s   Convdztorch::nn::Convz~dOptions(2, 3, 3)
                                            .stride(2)
                                            .padding(z)
                                            .dilation(1)
                                            .groups(1)
                                            .bias(true)
                                            .padding_mode()_stride2_pad2)
rB   rC   rD   rE   output_sizer  rN   rG   rH   rI   )ELU
HardshrinkHardsigmoidHardtanh	Hardswish	LeakyReLU
LogSigmoidPReLUReLUReLU6rP   SELUCELUGELUGLUr   SiLUMishSoftplus
SoftshrinkSoftsignTanh
Tanhshrink	Thresholdr  )r  )rC   rI   r  rC   r  )rb   	check_jitrI   r  )rO   rI   r  rI   r  r  r  r  r  r  r  r  r  r  r  )r  r   r  r  r  r  r  )rB   rE   rF   rN   test_cpp_api_parity)Lr
   set_rng_seedr   r   r   r   r   r   r   r   r   r  r  r  r"  r(  r9  r@  rG  rY  r^  rv  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r&  r,  r9  rB  rJ  rX  r^  rl  rr  rx  r  r  r  r   r7   rf   r  r   r{   r   interpolatesoftmaxfloat64log_softmaxgelur   reluSizezipr  rangejoinmapr  appendgetupdate)new_module_testspadding_modecpp_padding_moder  r  cpp_paddingrE   r  non_linear_activations_no_batch!non_linear_activations_extra_infonon_linear_activationactivation_test_info
extra_infos                r)   get_new_module_testsr  <  s5#   K%'K K 	'(K 	+,	K
 	)*K 	&'K 	./K 	01K 	-.K 	!"K 	()K 	12K 	,-K 	34K 	K  	&'!K" 	%&#K$ 	 %K& 	&''K( 	 )K* 	,-+K, 	'(-K. 	45/K0 	891K2 	!"3K4 	)*5K6 	./7K8 	!"9K: 	)*;K< 	./=K> 	$%?K@ 	+,AKB 	 CKD 	$%EKF 	GKH 	/0IKJ 	/0KKL 	,-MKN 	,-OKP 	*+QKR 	12SKT 	&'UKV 	01WKX 	89YKZ 	'([K\ 	*+]K^ 	:;_K` 	)*aKb 	./cKd 	/0eKf 	 &!D! ,,		
gKz 	 )!N! ,,
	
{KP 	 ,!Y!,,
	
QKf 	 ,!Y! ,,
	
gK| 	 ,!Y  ,,
	
}KR 	 ,!Y  ,,
	
SKh 	 &!D! 		
iK| 	%J!P! ,,	
}KN 	$H!N  ,,		
OKb 	'C![! ,,		
cKv 	&B!Z! ,,		
wKJ 	'B!Z! ,,		
KK^ 	.N!f! ,,		
_Kr 	&t!t  ,,		
sKF 	)9"e  ,,	
GK^ 	);"p  ,,	
_Kv 	-r"Y  ,,
	
wKL 	 +!I##' ,,
	
MKb 	 3!X##' ,,	
cKz 	 ;!h##' ,,	
{KR 	 C!y##' ,,	
SKj 	 >"_##' ,,	
kKD	 	 +!I##'		
E	KX	 	$A!S##' ,,
	
Y	Kn	 	)A!S##' ,,		
o	KB
 	'H!`# ,,		
C
KV
 	&G!_#,,		
W
Kj
 	.S!k#,,		
k
K~
 	)9"Y##',,	

KV 	)K"9 $#',,!	
WKz 	)C"o##',,	
{KT 	-J!\##',,
	
UKj 	'A!S# ,,	
kK| 	7A!S# ,,	
}KN 	/P!b# ,,	
OK` 	.Q!c# ,,	
aKr 	/R!d# ,,	
sKD 	 .!L&#',,
	
EKZ 	 A"_&#',,	
[Kt 	 A"_& #(,,	
uKN 	 )!N&#',,	
OKf 	 ,!Y&!#',,	
gK~ 	 .!L&#'		
KR 	$H!N&#' ,,
	
SKh 	%J!P&,,	
iKz 	-T!Z&,,	
{KL 	'K!c&,,		
MK` 	&J!b&,,		
aKt 	.V!n&,,		
uKH 	).!U&,,		
IK\ 	)K"x&,,	
]Kt 	*2!Y&,,	
uKB 	*2!Y#2,,	
CKT 	*2!Yb	
UKd 	##!DK ,,'(Z[	
eKv 	##!D\  ,,'(Z[		
wKJ 	&#!GK ,,	
KK\ 	&#!G\  ,,	
]Kn 	&;"yK ,,		
oKB 	&;"yK ,,		
CKV 	4D!VP ,,	
WKf 	3\"HP ,,	
gKx 	3\"HP ,,	
yKJ 	*V"f; !%	
KK\ 	S "C;' !%	
]Kl 	&!!D#,,	
mKz 	(!!F%,,	
{KH 	'BTXabv -,,	
IKZ 	'BTXabv 6	
[Kj 	'FQU\efv 3,,	
kK| 	'DrXabu 3,,	
}KN 	'BTX`puv9
 !,,,	
OKf 	'EPT[csxy9
 !2,,	
gK~ 	'DrX`puv9
 !2,,	
KV 	'BTX`puv9
 !5
	
WKl 	'BTX`ptu8
 !:,,	
mKD 	'DrX`ptu8
 !@,,	
EK\ 	'ADW`a: &<,,
	
]Kr 	'BTXab: $-,,
	
sKH 	'HSW^gh: $3,,
	
IK^ 	'DrXab: $3,,
	
_Kt 	'BTXab: $6		
uKH 	'BTXbrwx9
 $.,,	
IK` 	'BTXbrwx9
 $7
	
aKv 	'FQU-7uN9
 $4,,	
wKP 	'Dr-7uN9
 $4,,	
QKj 	'Dx-7uN9
 $A,,	
kKD 	'Dx-7uN9
 $A,,	
EK^ 	'FQU\fvz{8
 $B,,	
_Kv 	'Dx-7tM8
 $O,,	
wKP 	'BTXaqvw9
 $-,,	
QKh 	'BTXaqvw9
 $6
	
iK~ 	'FQU-6eM9
 $3,,	
KX  	'DrXaqvw9
 $3,,	
Y Kp  	'Dx-6eM9
 $@,,	
q KJ! 	'Dx-6eM9
 $@,,	
K!Kd! 	'FQU\euyz8
 $A,,	
e!K|! 	'Dx-6dL8
 $N,,	
}!KV" 	'BTXab: '-,,
	
W"Kl" 	'BTXab: '6		
m"K@# 	'LW[bkl: '3,,
	
A#KV# 	'DrXab: '3,,
	
W#Kl# 	'BTXcsxy9
 '/,,	
m#KD$ 	'BTXcsxy9
 '8
	
E$KZ$ 	'I59[`b9
 '5,,	
[$Kt$ 	'DrXcsxy9
 '5,,	
u$KP% 	'ITX-8N8
 'C,,	
Q%Kj% 	'DrXcswx8
 'C,,	
k%KF& 	'		r:8&,,	
G&KV& 	'		qNN,,,	
W&Kh& 	'		q97%.,,	
i&Kx& 	'		q97#&,,	
y&KH' 	'		qNN#,,,	
I'KZ' 	'		q97#.,,	
['Kl' 	'		q97#.,,	
m'K~' 	'		r:80	
'KN( 	'2>;*,,	
O(K^( 	'1=:%2,,	
_(Kn( 	'1=:#*,,	
o(K~( 	'1=:#',,	
(KN) 	'1=:#',,	
O)K^) 	'1=:)	
_)Kl) 	I!s# ,,	
m)K~) 	O!y! ,,	
)KP* 	.O!y ),,		
Q*Kd* 	'5!_# ,,	
e*Kv* 	%6!`! ,,	
w*KH+ 	26!`) ,,		
I+K\+ 	'!R&,,	
]+Kn+ 	*E,,	
o+Kx+ 	*E ,,		
y+KD, 	*D ,,		
E,KP, 	*/!gE(,,	
Q,K`, 	*=2,,	
a,Kn, 	1,"5 !"  % ,,	
o,KP- 	1AsAFF3"A ! "#.4D,,	
Q-Kl- 	1+"5 H ",,	
m-KF. 	1AsAFF3"A H ",,	
G.Kb. 	%Aq!S!&&9"A Z ##.4D,,!	
c.KF/ 	 #!A*Z ,,
	
G/K\/ 	!!X% 2,,	
]/Kn/ 	#!J %**aV"45 2,,	
o/K@0 	#*D%8!p& $-
	
A0K\0 +.9Ybf+h #&&  	AAv,)3 E!QUO,G		#c7*; <<sBK$(*J 5(@(@#@@K##"&qc&'Aq'1a|%T-<QC @66A] C; <L:LA*Q  * +(>7"#'"',,!	#L'#:UU\\J:((3: 	%eV[VbVbc: 	ETYT`T`a	: 	uu||D: 	.: 	%,,/: 	.: 	5: 	_ell3: 	ou||4: 	5: 	%,,/:  	/5<<0!:" 	/5<<0#:$ 	%,,/%:& 	%,,/':( !%,,/#U\\2$ell3&5$ell3 %,,/&55:%8 "A 
6#-2 % 
 7::;PRTU
##J/ 45
6 r+   c                 6   |rt        j                  |      || z
  z  }n||j                         | z
  z  }|dk(  r|j                         S |dk(  r|j	                         S |dk(  r5|j                         dk7  r"|j	                         |j                  d      z  S |S )Nr  rV  	batchmeanr   )r7   r   r   r  rV  rT  rg   )r  rN  r    r  r(   s        r)   kldivloss_referencer  
  s    6"fun56::<%/0F{{}	e	zz|	k	!fjjla&7zz|fkk!n,,Mr+   c                    | j                         dk\  sJ | j                  d      }| j                  d      }|f| j                         dd  z   }t        j                  |      j	                  |       }|$t        j
                  |      j	                  |       }d}	t        |D 
cg c]  }
t        |
       c}
 D ]X  }||   }||k(  rdn||   j                         }t        |      }|j                  d|       | t        |          |z  ||<   |	|z  }	Z |dk(  r|j                         |	z  S |dk(  r|j                         S |S c c}
w )NrZ   r   r5   rQ   r  r  rV  )rT  rg   r7   r  r   onesr   r  itemr  insertr  rV  )r  rN  r-   ry  r    NCout_sizeoutputtotal_weightrg   tupt_nxnorminput_indexs                  r)   nlllossNd_referencer
  
  sB   99;!

1A

1Atejjl12&&H[["**51F~A&&u-L:t:; c{!T)rvd|/@/@/B3i1d#U;/0047s Fzz|l**	e	zz|M ;s   $Ec                    | j                         dk\  sJ t        j                  | d      } | j                  d      }|$t        j                  |      j                  |       } |j                  d|gd | j                  dd  D         }|dkD  r|dk  sJ |d|z
  z  ||z  z   }| |z  |z  j                  d       }|dk(  r|j                         S |dk(  r|j                         S |S )	NrQ   r5   c              3       K   | ]  }d   ywr  ro   r  r?   s     r)   r  z;cross_entropy_loss_prob_target_reference.<locals>.<genexpr>
  s      <q <   r  r[  rS  r  rV  )
rT  r7   r  rg   r  r   r:   shaperV  r  )r  rN  r-   r    label_smoothingr  r  s          r)   (cross_entropy_loss_prob_target_referencer  
  s    99;!eQ'E

1A~A&&u-V[[A= <EKKO <=F#%%%A/0?Q3FFv~&+++22FF{{}	e	zz|Mr+   c                 (   t        j                  | d      }t        j                  |||||      }|dk(  r|S d|cxk  rdk  sJ  J t        j                  | d      } | j	                  d      }|+|  |j
                  d|gd | j                  dd  D         z  } t        j                  | d       }	||k(  }
|	j                  |
d       |dk(  r|bt        j                  |	      |j                  d|j                  |
j                               j                               j                         z  }nPt        j                  |	j                  |
j                                     }n|d	k(  rt        j                  |	      }n|	}d|z
  |z  |||z  z  z   S )
Nr5   rx  r  r[  c              3       K   | ]  }d   ywr  ro   r  s     r)   r  z>cross_entropy_loss_indices_target_reference.<locals>.<genexpr>
  s     +G!A+Gr  rQ   r  r   rV  )r7   r  r   rd  rg   r:   r  rV  masked_fill_gathermasked_selectlogical_notr]   r  )r  rN  r-   ry  r    r  log_softmax_inputnlllossr  smooth_lossignore_maskrets               r)   +cross_entropy_loss_indices_target_referencer  
  s   ))%3jj!G #'C'''''eQ'E

1AAqH+Gu{{12+GHH99UA&&KL(K[#.F ))K(6==F<P<PQ\QhQhQj<k<s<s<u+v+z+z+||C**[66{7N7N7PQRC	e	ii$7*SOa4G-HHHr+   c                 v    | j                   |j                   k(  rt        | ||||      S t        | |||||      S )N)r-   r    r  )r-   r    ry  r  )r  r  r  )r  rN  r-   ry  r    r  s         r)   cross_entropy_loss_referencer  
  sK    {{fll"7+- 	- ;6&I%
 	
r+   c           
          d }t        | |d      D cg c]  \  }} |||||       }}}t        |ddi\  }	}
| j                  |	      }|dk(  rt        |      t        |
      z  S |dk(  rt        |      S |S c c}}w )Nc                 :    ||k(  ry|dn||   }| |    |z  }||fS )Nr  r5   ro   )r  rN  r-   ry  r  r(   s         r)   nll_loss_helperz*nllloss_reference.<locals>.nll_loss_helper  s6    \!Nqv-$&~r+   Tr  r  r  rV  )r  
new_tensorrV  )r  rN  r-   ry  r    r"  r=   r9   losses_and_weightslossesr.   losses_tensors               r)   nllloss_referencer'    s     '*%&EG"a *!QE G G-;d;OFG$$V,MF=!CL00	e	=!!Gs   A:c                    | |z
  j                         }||k\  j                  |      }||k  j                  |      }|dk(  r|}n||d|z  z
  z  |dz  |dz  z  |z  z   }|dk(  r|j                         S |dk(  r|j                         S |S )Nr   r  rQ   r  rV  )r6  r   r  rV  )r  rN  r    r  abs_diffge_beta_masklt_beta_maskr  s           r)   smoothl1loss_referencer,  0  s    ##%H$--h7LtO,,X6LqyC$J!67,:LPX\]P]:^ae:eeF{{}	e	zz|Mr+   c                     | |z
  j                         }||k\  }||k  }||z  |d|z  z
  z  |dz  |dz  z  z   }|dk(  r|j                         S |dk(  r|j                         S |S )Nr  rQ   r  rV  )r6  r  rV  )r  rN  r    r  r)  ge_delta_masklt_delta_maskr  s           r)   huberloss_referencer0  @  s    ##%H&M%MU"hu&<=PS@SW_cdWd@eeFF{{}	e	zz|Mr+   c           	          g }|D ]  }|dk  r n|j                  |        d}|D ];  }t        t        |             D ]"  }||vs|t        dd| |   z
  | |   z         z  }$ = |S )Nr   r5   )r  r  lenr  )r  rN  targetstarget_indexrV  r=   s         r)   _multilabelmarginloss_referencer5  L  s    G %!|$%
 C Bs5z" 	BAs1a%"55a@AA	BB
 Jr+   c                    | j                         }| j                         dk  r|j                         dk  sJ | j                         dk(  r| j                  d      n| j                  d      j                  d      } |j                         dk(  r|j                  d      n|j                  d      j                  d      }| j                  d      }| j                  d      }| j                  |      j	                         }t        |      D ]  }t        | |   ||         ||<    |dk(  r|j                         |z  S |dk(  r|j                         |z  S |dk  r|j                         |z  S ||z  S NrQ   r5   r   r  rV  )
rT  r  rg   newzero_r  r5  r  rV  r  )r  rN  r    	input_dimnrT  r  r=   s           r)   multilabelmarginloss_referencer<  \  sM   		Iyy{Qzz|a&+iikQ&6"EOOA<N<X<XYZ<[(.

(9!!!$v?O?OPQ?R?\?\]^?_

1A
**Q-CYYq\!F1X I3E!HfQiHq	I F{{}s""	e	zz|c!!	Q ~~#%%|r+   c                     || z
  j                  d      j                  |       }t        j                  |dk(  | |      }|dk(  r|j	                         S |dk(  r|j                         S |S )Nr   r  r5   r  rV  )r%  r   r7   wherer  rV  )r  rN  r?  r    margin_clampr  s         r)   hingeembeddingloss_referencer@  v  sg    UN))a)088?L[[1e\:FF{{}	e	zz|Mr+   c                     d|  |z  j                         z   j                         }|dk(  r|j                         S |dk(  r|j                         S |S )Nr5   r  rV  )r   r   r  rV  )r  rN  r    r  s       r)   softmarginloss_referencerB    sQ    E6F?''))..0FF{{}	e	zz|Mr+   c           	          |)| j                  t        |             j                  d      }d}t        t        |             D ],  }||k7  s	|||   t	        d|| |   z
  | |   z         |z  z  z  }. |S )Nr5   r   )r8  r2  fill_r  r  )r  
target_idxr>   r?  r-   r  r=   s          r)   _multimarginloss_referencerF    s    ~3u:&,,Q/F3u: b
?fZ(CFU:=N4NQVWXQY4Y,[_`,`aaFb Mr+   c                 j   | j                         dk  rD| j                         dk(  r| j                  d      n| j                  d      j                  d      } |j                         }|j                         dk(  r|j                  d      }| j                  d      }| j                  d      }| j                  |      }	t	        |      D ]  }
t        | |
   ||
   |||      |	|
<    |dk(  r|	j                         |z  S |dk(  r|	j                         |z  S |dk(  r|	j                  d      |z  S |	|z  S r7  )	rT  r  rg   r8  r  rF  r  rV  r  )r  rN  r>   r?  r-   r    
target_dimr;  rT  r  xs              r)   multimarginloss_referencerJ    s   yy{Q&+iikQ&6"EOOA<N<X<XYZ<[Jzz|q!!!$

1A
**Q-CYYq\F1X W.uQxAvvVq	W F{{}s""	e	zz|c!!	q~~a 3&&C<r+   c                     d }t        j                  |dk(  d || |      z
   || |      |z
  j                  d            }|dk(  r|j                         S |dk(  r|j	                         S |S )Nc                 6   | j                  | j                  d            }t        | j                  d            D ][  }| |   ||   z  j                         | |   | |   z  j                         dz   ||   ||   z  j                         dz   z  dz  z  ||<   ] |S )Nr   g-q=r  )r8  rg   r  rV  )abcosr=   s       r)   _cosz+cosineembeddingloss_reference.<locals>._cos  s    eeAFF1Iqvvay! 	tAdQqTk&&(qtad{.?.?.AE.IqQRtVWXYVZ{N_N_NadiNi-jor,rsCF	t
r+   r5   r   r  r  rV  )r7   r>  r%  r  rV  )input1input2rN  r?  r    rP  r  s          r)   cosineembeddingloss_referencerS    st     [[1a$vv*>&>ffAUX^A^@e@ejk@e@lmFF{{}	e	zz|Mr+   c                 T   t        j                  | |||      }t        j                  | |||      }	|r.t        j                  ||||      }
t        j                  |	|
      }	t        j                  ||z   |	z
  d      }|dk(  r|j	                         S |dk(  r|j                         S |S )Nr  r  r  rV  )r7   pairwise_distancer  r%  r  rV  )anchorpositivenegativer?  r>   epsswapr    d_pd_nd_sr  s               r)   tripletmarginloss_referencer^    s    

!
!&(As
;C

!
!&(As
;C%%h!SAiiS![[#+5FF{{}	e	zz|Mr+   c                     | | |z
  z  |z   j                  d      }|dk(  r|j                         S |dk(  r|j                         S |S )Nr   r  r  rV  )r%  r  rV  )rQ  rR  rN  r?  r    r  s         r)   marginrankingloss_referencer`    sR    g&)F299a9@FF{{}	e	zz|Mr+   c                 @   t        j                  |t         j                        }t        j                  |t         j                        }| j                  }| j	                         } |j                         }|j                  d      }g }t        | j                  d            D ]|  }	||	   j                         }
||	   j                         }||	   j                         }|j                  d|z  dz   f|      }|j                         dk(  r||	d |f   |dd d<   n|||z
  | |dd d<   | d |
|	f   j                         }| j                  |dz  dz   f      }|d|f   |d<   |d|d   f   |d<   |d d |dd  k7  }t        d|
      D ]b  }|j                         }|dd xxx |d d z  ccc |dd xxx t        j                  ||d d |j                  d            z  ccc |||f   |z  }d |j                  |dd  j!                         j#                         d             t        j$                  |d      }|dk(  r9||j'                  |j                  |j(                        z  j+                         }n|d	k(  r|j!                         }|j'                  |      }|S )
Nr   r   r5   rQ   r  r6   r  r   devicerV  )r7   	as_tensorre  r   rf   cumsumr  rg   r  new_fullrT  r   	new_zeroscloner>  r  rV  r   catr   rc  r  )	log_probsr3  input_lengthstarget_lengthsblankr    dtcum_target_lengthsr%  r=   input_lengthtarget_lengthcum_target_lengthtargets_primeprobsalpha
mask_thirdr9   
alpha_nextr  s                       r)   ctcloss_referencerx    s   OOMDM__^5::FN	B  "IllnG'..q1F9>>!$% 5$Q',,.&q)..0.q1668((!m*;a*?)A5I;;=A")!^m^*;"<M!$Q$")*;m*KL]"^M!$Q$-<-*+//1##]Q%6%:$<=E?aM!,,-a#CR(M!",==
q,' 	9AJqrNeCRj(NqrNekk*eCRj%//RSBTUUN!]*+j8E		9
 	uRSz~~'++-d334'5( YYvq!FF>,,6<<,VV\\^	e	YYr]FMr+   )r  )r   r  rm  r  r  r  r  r4  rI  ri  CosineEmbeddingLossTripletMarginLossMarginRankingLossCTCLossCrossEntropyLossr  r   c                      | d   }d }fd | dd D cg c]
  } ||       c}      } || }t        |      }|dk(  r|j                  d      S |S c c}w )zReference function for criterion supporting no batch dimensions.

    The criterion is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    r6   c                     t        | t        t        f      r| D cg c]  }|j                  d       c}S | j                  d      S c c}w rL   r  r  s     r)   r  z:single_batch_reference_criterion_fn.<locals>.unsqueeze_inp  r  r  c                     g }t        | t        t        f      r | D ]  }|j                   |              |S |j	                  |        |S rn   )r  r  r  extendr  )xsr(   rI  r]   s      r)   r]   z4single_batch_reference_criterion_fn.<locals>.flatten  sO    b4-( *gaj)*  MM"r+   Nr   r   )r*   r  )rq   	criterionr  r  single_batch_input_argsr  r    r]   s          @r)   #single_batch_reference_criterion_fnr    sr     RI 
 &cr&S}U';&ST/0Fi(IF~~a  M 'Ts   A)L1LossMSELossPoissonNLLLossr  r  )r   r  rV  _no_batch_dim_)namec                 8     t        t        |       t              S Nr   r#   ry   r    r  rq   s     r)   r@   r@   ;      ->WR->-S r+   r  )r   r   rE   target_sizerF   r  rI   KLDivLoss_no_batch_dim_c                  6    t        j                  t              S r  )ry   r   r    ro   r+   r)   r@   r@   H  s    BLL9= r+   c                  H    t        j                  d      j                         S Nr  r   ro   r+   r)   r@   r@   I  s    D)--/ r+   c                  ,    t        j                  d      S r  r   ro   r+   r)   r@   r@   J  s    %**T* r+   )r   r   r   	target_fnrF   r  rI   BCELossc                  r    t        j                  t        j                  dt         j                              S Nr;  r   )r7   r   r   rf   ro   r+   r)   r@   r@   W  s    ekk!5<<@A r+   c                      t        j                  dt         j                        j                  d      j	                  t         j                        S )Nr;  r   r   )r7   r   rf   r   r   ro   r+   r)   r@   r@   X  s/    AU\\255a8;;ELLI r+   BCEWithLogitsLossc                  L    t        j                  dt         j                        S r  r7   r   rf   ro   r+   r)   r@   r@   Z  s    %++au||"D r+   c                  L    t        j                  dt         j                        S r  r  ro   r+   r)   r@   r@   Z  s    ekkZ[chcocoNp r+   r4  c                  L    t        j                  dt         j                        S r  r  ro   r+   r)   r@   r@   [  s    5;;q#E r+   c                  6    t        j                  g ddz        S N)r6   r5   r5   rZ   r7   tensorro   r+   r)   r@   r@   [  s    u||\fij\jOk r+   r  c                  L    t        j                  dt         j                        S )Nr3   r   r  ro   r+   r)   r@   r@   \  s    U[[%,,%G r+   c                  0    t        j                  g d      S )N)rZ   r   r6   r5   r  ro   r+   r)   r@   r@   \  s    QVQ]Q]^kQl r+   rI  c                  L    t        j                  dt         j                        S r  r  ro   r+   r)   r@   r@   ]  s    u{{1ELLA r+   c                  6    t        j                  g ddz        S r  r  ro   r+   r)   r@   r@   ]  s    5<<XbefXfKg r+   rm  c                  v    t        j                  t        j                  dt        j                        d      S )NrZ   r   r   rS  )r   r  r7   r   rf   ro   r+   r)   r@   r@   ^  s     ekk!5<<&HaP r+   c                  ,    t        j                  d      S r\   r  ro   r+   r)   r@   r@   ^  s    Z_ZfZfghZi r+   ry  c                      t        j                  dt         j                        t        j                  dt         j                        fS r  r  ro   r+   r)   r@   r@   a  )    Qell3U[[%,,5WX r+   c                  L    t        j                  dt         j                        S )Nr5   r   )r7   r  rf   ro   r+   r)   r@   r@   b  s    Qell3 r+   r{  c                  V    t        j                  d      t        j                  d      fS r  r2  ro   r+   r)   r@   r@   e  s    5;;r?EKKO"D r+   c                  H    t        j                  d      j                         S r  )r7   r   signro   r+   r)   r@   r@   e  s    ekkZ\oNbNbNd r+   rz  c                      t        j                  dt         j                        t        j                  dt         j                        fS r  r  ro   r+   r)   r@   r@   i  r  r+   c                  L    t        j                  dt         j                        S r  r  ro   r+   r)   r@   r@   j  s    AU\\2 r+   MultiLabelSoftMarginLossc                  L    t        j                  dt         j                        S r  r  ro   r+   r)   r@   r@   l  s    Qell)K r+   c                  ,    t        j                  d      S )Nr;  r2  ro   r+   r)   r@   r@   l  s    UZU`U`abUc r+   rb   ,classification_criterion_no_batch_extra_info)r  r  r4  rm  rI  c                 8     t        t        |       t              S r  r  r  s     r)   r@   r@   ~  r  r+   c                      |        S rn   ro   fs    r)   r@   r@     s    AC r+   c                      |        S rn   ro   r  s    r)   r@   r@     s    ac r+   )r   r   r   r  rF   r  
has_parityc                   |   e Zd Zed        Zedej                  deeej                     eej                     f   fd       Z
edej                  ddfd       Ze	 ddej                  dedej                  d	eej                  eej                     f   d
ef
d       Zd Zd Zd ZddefdZddefdZddefdZy)
NNTestCasec                     t         rn   NotImplementedError)rp   rq   rs   s      r)   _forwardzNNTestCase._forward      !!r+   r  returnc                     t         rn   r  rp   r  s     r)   _get_parameterszNNTestCase._get_parameters  r  r+   Nc                     t         rn   r  r  s     r)   _zero_grad_parametersz NNTestCase._zero_grad_parameters  r  r+   r  r  grad_outputcreate_graphc                     t         rn   r  )rp   r  r  r  r  r  s         r)   	_backwardzNNTestCase._backward  s
    
 "!r+   c                     t        |t              rt         fd|D              S t        |t              r |D cg c]  } j                  |       c}S t	        j
                  |j                               S c c}w )Nc              3   B   K   | ]  }j                  |        y wrn   )	_jacobian)r  elemnum_outrp   s     r)   r  z'NNTestCase._jacobian.<locals>.<genexpr>  s     I4g6Is   )r  r  r  r  r7   r  nelement)rp   r  r  r  s   ` ` r)   r  zNNTestCase._jacobian  sb    eU#I5IIIt$>CDdDNN41DD;;u~~/99 Es   A<c                      t        |t        j                        r<|j                  r|j	                         j                  d      S |j                  d      S t         fd|D              S )Nr6   c              3   @   K   | ]  }j                  |        y wrn   )_flatten_tensors)r  rM  rp   s     r)   r  z.NNTestCase._flatten_tensors.<locals>.<genexpr>  s     =a..q1=   )r  r7   r  	is_sparseto_denser:   r  )rp   rI  s   ` r)   r  zNNTestCase._flatten_tensors  sN    a&{{zz|((,,vvbz!=1===r+   c                    t        |t        j                        rO|j                  rB|j                  5|j                  j                          |j                  j                          y y y |D ]  }| j                  |        y rn   )r  r7   r  r  gradr9  detach__zero_grad_input)rp   r  r=   s      r)   r  zNNTestCase._zero_grad_input  si    eU\\*""uzz'=

  "

""$ (>"  )%%a()r+   c                    | j                  ||      }|j                         }|r&| j                  ||      }t        t	        |            }|r:t        d | j                  |      d   D              }	t        j                  |	|      }
t        |      D ])  }| j                  |      \  }}t        ||d      D cg c]  \  }}|t        j                  |      n|  }}}t        j                  |      }|j                  d      }d||<   |r| j                  |       |r| j                  |       | j                  ||||      }|rEt        t	        |      d      D ]+  \  }}|j!                         j                  d      |d d |f<   - |st        j"                  | j%                  |      d      
d d |f<   , d}|r|fz  }|r|
fz  }|S c c}}w )Nc              3   <   K   | ]  }|j                           y wrn   )numelr  s     r)   r  z2NNTestCase._analytical_jacobian.<locals>.<genexpr>  s     O!AGGIO   r   Tr  r6   r5   ro   )r  r  r  r  r   rV  r  r7   r  r  r  
zeros_liker:   r  r  r  
contiguousri  r  )rp   r  r  jacobian_inputjacobian_parametersr  r  jacobian_inpflat_jacobian_input	num_paramjacobian_paramr=   paramd_paramr>   r  d_out
flat_d_outd_input
jacobian_xd_xress                         r)   _analytical_jacobianzNNTestCase._analytical_jacobian  s   vu-oo'>>%=L"&}\'B"COt/C/CF/KA/NOOI"[[K@N{# 	TA!11&9NE7MPQVX_hlMmn6Aqaiu''*Q>nGn$$V,EBJJqM"**62%%e,nnVUFEBG'*+>g@V_c'd AOJ'*~~'7'<'<R'@Jq!t$A"',yy1F1Fw1OQR'Sq!t$)	T, )+<= C>?"C
3 os   ;#Gc                       fd}d}|r|t        ||d      fz  }|r] j                        \  }}g }	|D ](  }
t        |||
d      }|	j                  |d   d          * |t        j                  |	d      fz  }|S )Nc                  D    j                  |       j                         S rn   )r  detach)r  r  rp   s    r)   fwz*NNTestCase._numerical_jacobian.<locals>.fw  s    ==/6688r+   ro   ư>)rY  )rN  rY  r   )r   r  r  r7   ri  )rp   r  r  r  r  r  r  r  r?   to_catr>   jacobians   ``          r)   _numerical_jacobianzNNTestCase._numerical_jacobian  s    	9 )+*2u$?@@C++F3HE1F .22uQDQhqk!n-. EIIfa(**C
r+   c                    t        | j                  |      d         }| j                  ||||      }| j                  ||||      }t	        t        |            }t	        t        |            }g }	t        ||d      D ]W  \  }
}|
j                         dk7  s|	j                  |
j                  |d      j                         j                                Y t        |	      dkD  r | j                  t        |	      t               y y )Nr   Tr  r6   )ru  )boolr  r  r  r  r   r  r  r  r  r6  r  r2  assertLessEqual	PRECISION)rp   r  r  r  r  
analytical	numericalanalytical_tnumerical_tdifferencesrM  r;  s               r)   check_jacobianzNNTestCase.check_jacobian  s    "4#7#7#?#BC..vunNab
,,VUNL_`	M*56=34k$? 	CDAqwwyA~""155"5#5#9#9#;#?#?#AB	C {a  [!19=  r+   F)TTT)ru   rv   rw   r   r  ry   rz   r  r  	Parameterr  r  r   r7   r  r   r   r  r  r  r  r  r  r  r  ro   r+   r)   r  r    s    " " "bii "E$r||:LdSUS_S_N`:`4a " " "BII "$ " "  (-"		 ")"38<<"$U\\8ELL3I%IJ" !%" ":>)(2B (T1A ">,< >r+   r  c                   \    e Zd Zh dZddZd Zd Zed        Zed        Z	d Z
dd	Zd
 Zy)TestBase>   r  
extra_argsrC   Nc                     || _         || _        || _        || _        | j                  D ]?  }||vs|dz   |vs|dz   |vs|dv rd||<   "t        | j                          d| d       || _        i | _        y )N_fn_size>   r	  rC   ro   z
: Specify z5 by a value, a function to generate it, or it's size!)	rN   r   r   rF   _required_arg_names
ValueErrorget_name_extra_kwargs
_arg_cache)rp   r   rN   rF   r   rs   r  s          r)   __init__zTestBase.__init__	  s    	 &(,, 	@D6!dUl&&@TG^[aEa==#%F4L$'8
4&H}%~	@ $r+   c                     | j                   d| j                   z   S d| j                  j                  z   }| j                  r|d| j                  z   z  }|S )Ntest_r?   )r   r   ru   rN   )rp   	test_names     r)   r  zTestBase.get_name  sP    ==$T]]**d..777	99tyy(Ir+   c                      t        |t        j                        r|S t        |      r t	        |       fd|D              S |S )Nc              3   @   K   | ]  }j                  |        y wrn   )_unpack)r  vrp   s     r)   r  z#TestBase._unpack.<locals>.<genexpr>$  s     >1t||A>r  )r  r7   r  r   type)rp   values   ` r)   r  zTestBase._unpack   s:    eU\\*L4;>>>>Lr+   c                 &    | j                  dd      S )NrC   T_get_argrp   s    r)   rC   zTestBase.constructor_args(  s    }}/66r+   c                 &    | j                  dd      S )Nr	  Tr  r  s    r)   r	  zTestBase.extra_args,  s    }}\400r+   c           
         || j                   v sJ || j                  vr|dz   }|dz   }|| j                  v r| j                  |   | j                  |<   n|| j                  v r" | j                  |          | j                  |<   nV|| j                  v s!J d| d| d| d| j                                 fd | j                  |         | j                  |<   |r| j	                  | j                  |         S | j                  |   S )Nr  r  z	Missing `z`, `z` or `z` for c                     t        | t              r| D cg c]
  } |       c}S t        | t        j                        r| j	                         S t        j
                  |       S c c}w rn   )r  r  r7   r  rf   r   )sizessmap_tensor_sizess     r)   r%  z+TestBase._get_arg.<locals>.map_tensor_sizes?  sS    !%.=BC 0 3CC#E5<<8$||~-${{511	  Ds   A')r  r  r  r  r  )rp   r  unpackfn_name	size_namer%  s        @r)   r  zTestBase._get_arg0  s   t/////t&UlGwIt)))(,(:(:4(@%D...(C(:(:7(C(E% D$6$66 \vT)F7)6$--/IZ[\62 )99K9KI9V(W%6<t||DOOD12W$//RVBWWr+   c                 &    | j                  d|      S )Nr  r  )rp   r&  s     r)   
_get_inputzTestBase._get_inputK  s    }}Wf--r+   c                     t         rn   r  )rp   	test_cases     r)   __call__zTestBase.__call__N  s    !!r+   ) NNr  )ru   rv   rw   r  r  r  r  propertyrC   r	  r  r*  r-  ro   r+   r)   r  r    sO    E 7 7 1 1X6."r+   r  c            	       l     e Zd Zededej                  dedefd       Z fdZd Z	d Z
d	 Zd
 Z xZS )
ModuleTestr,  r  r  r  c                     t         rn   r  )rp   r,  r  r  s       r)   _do_testzModuleTest._do_testT  r  r+   c                    t        |   |i | |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd	      | _        |j                  d
d      | _	        |j                  d      | _
        | j                  t        j                         | _
        y y )Nr  TrO   r   rb   !FIXME_no_cuda_gradgrad_comparisonFr   g-C6*?check_forward_onlyrI   )superr  r  r  should_test_cudashould_test_picklerb   r5  r   r6  rI   r7   get_default_dtyperp   rq   rs   	__class__s      r)   r  zModuleTest.__init__X  s    $)&)$jj)94@ &

; ="(**Xt"<$jj)94@JJ:EB 	.K6"(**-A5"I#ZZ8%!&!8!8!:D &r+   c           	      j   t        | j                        5   | j                  | j                   }| j	                         }| j
                  a|j                  ||      }t        |      }t        |      }| j                  ||j                  |      d   |      }|j                  ||d       | j                  r
	 d d d        y | j                  |||       | j                  rt        j                         5 }|j                  ||       t        j                   ||       |j#                  d       t        j$                  |d      }	|j                  |j                  ||      |j                  |	|             d d d        | j'                  |||       d d d        y # 1 sw Y   %xY w# 1 sw Y   y xY w)Nr   F)exact_dtype)weights_only)r   rI   r   rC   r*  rF   r  r   r  assertEqualr6  test_noncontigr9  tempfileTemporaryFiler7   saveseekloadr3  )
rp   r,  r  r  out	ref_input
ref_moduleexpected_outr  module_copys
             r)   r-  zModuleTest.__call__f  s   t112 	4%T%%t'<'<=FOO%E  ,((7$UO	%f-
#00I<U<UV\<]^_<`blm%%c<U%K&&	4 	4 	659&&++- u&&vu5JJvq)FF1I"'**QU"CK)))*<*<VU*KYM_M_`kmrMstu MM)VU31	4 	4 u u!	4 	4s+   B$F)3F)6BF9F)F&	"F))F2c                 Z    t        |t              r|D cg c]  } j                  |       c}S t        |t              rt         fd|D              S |}|j	                         }|}t        |      D ]  }|j                  |      dkD  s|dz   } n t        j                  t        j                  |      |g|      j                  |d      j                         }|j                         dk(  s%|j                         dk(  s|j                         rJ |j                  |_        |S c c}w )Nc              3   @   K   | ]  }j                  |        y wrn   )noncontiguize)r  orp   s     r)   r  z+ModuleTest.noncontiguize.<locals>.<genexpr>  s     <1++A.<r  r5   r   )r  r  rN  r  rT  r  rg   r7   r%  
empty_likeselectr  r  is_contiguousr  )rp   objrO  r  ndimrT  r  	noncontigs   `       r)   rN  zModuleTest.noncontiguize  s   c4 367aD&&q)77U#<<<<zz|
 t 	A{{1~!!e	 KK!1!1&!96 BCHOOPSUVW^^`	 A%):a)?yG^G^G```"("6"6	# 8s   D(c           	      X   t        |t        j                        r|j                         dk(  ry t	        d |D              ry |j                  |       |j                  |       t               5  |j                  ||      }t        |dd      r|d   }|j                  |j                        j                         }|j                         }t        |j                  ||||            }t        |j!                  |      d         }d d d        | j#                  |      }| j#                        }	t%        dd      D ]  \  }
}|
r|n|}t        |r|n|	      }|j                  |       |j                  |       t               5  |j                  ||      }t        |dd      r|d   }|j                  ||||      }|j'                  |       |j'                  |d	d
       |j'                  |j!                  |      d          d d d         y # 1 sw Y   xY w# 1 sw Y   xY w)Nr   c              3   x   K   | ]2  }t        |t        j                        s|j                         d k(   4 yw)r   N)r  r7   r  rT  )r  r=   s     r)   r  z,ModuleTest.test_noncontig.<locals>.<genexpr>  s'     JjELL.Iquuw!|Js   ::return_indicesFr5   TFrQ   )r4  g-C6?atolrtol)r  r7   r  rT  anyr  r  r   r  r#   r8  r  normal_rh  r   r  r  rN  r   r@  )rp   r,  r  r  r  r  r  r  nc_inputnc_grad_outputcontig_icontig_gr=   gorG  r  s                   r)   rA  zModuleTest.test_noncontig  s   eU\\*uyy{a/?JUJJ''/""5) 	E''6Fv/7 **V\\2::<K\\^Fy2265&+VWGy88@CDG	E %%e,++K8")-"B 	UHh!xA +~FB++F3&&q)!# U((36#3U;a&C **61c2>%%c62%%dG$Q%G%%i&?&?&G&JGTU U	U	E 	E$U Us   .BHBH H H)	c           	      
   t         r| j                  st        j                  d      t	        | j
                        5  | j                         }t        j                  t        j                  i}t        |t              r|n|f}t        d |D              }t        ||      } | j                  | j                   } | j                  | j                   j                         j!                         }|j#                  |      }	|j#                  |      }
t%        |	d   |
d   d      D ]   \  }}|j&                  j)                  |       " |j+                  |       |j+                  |       |j-                  |       |j-                  |       |j/                  ||      }|j/                  ||      }t1        |dd      r
|d   }|d   }|j3                  ||| j4                  dd	       t7        d
      D ]  }|j9                         j;                         }|j=                  |      }|j?                  ||||      }|j?                  ||||      }|j3                  ||| j4                  dd	       t%        |	d   |
d   d      D ]$  \  }}|j3                  ||| j4                  d       &  | j@                  r| jB                  s || } || }t1        |dd      r
|d   }|d   }t        jD                  |d      }|j=                  |      jG                         }d|_$        t        jJ                  jM                  ||t        |jO                               z   |d      }t        jJ                  jM                  ||t        |jO                               z   |d      }t%        ||d      D ]%  \  }}|j3                  ||| j4                  dd	       ' |rc|jQ                         jS                         tQ        d |D              z   }|jQ                         jS                         tQ        d |D              z   }nF|jQ                         tQ        d |D              z   }|jQ                         tQ        d |D              z   }t        jJ                  jM                  |||fz   t        |jO                               z   d      }t        jJ                  jM                  |||fz   t        |jO                               z   d      }|j3                  | j4                  dd	       t%        ||d      D ]%  \  }}|j3                  ||| j4                  dd	       ' | jU                  |||       d d d        y # 1 sw Y   y xY w)NExcluded from CUDA testsc              3      K   | ]6  }t        |t        j                        xr |j                  j                   8 y wrn   )r  r7   r  r   
is_complexr  r9   s     r)   r  z'ModuleTest.test_cuda.<locals>.<genexpr>  s.     &s^_z!U\\'B'YqwwGYGY'Y&ss   <>)type_mapr   Tr  rX  Fr[  r\  r>  rU   r5   rZ  )r  )r  c              3   X   K   | ]"  }|j                         j                          $ y wrn   rV  r6  r  rI  s     r)   r  z'ModuleTest.test_cuda.<locals>.<genexpr>       >eQRquuw{{}>e   (*c              3   X   K   | ]"  }|j                         j                          $ y wrn   rl  rm  s     r)   r  z'ModuleTest.test_cuda.<locals>.<genexpr>  rn  ro  c              3   <   K   | ]  }|j                           y wrn   rV  rm  s     r)   r  z'ModuleTest.test_cuda.<locals>.<genexpr>       8YQ8Yr  c              3   <   K   | ]  }|j                           y wrn   rr  rm  s     r)   r  z'ModuleTest.test_cuda.<locals>.<genexpr>  rs  r  )retain_graph)+r   r8  unittestSkipTestr   rI   r*  r7   rf   floatr  r  r]  r   r   rC   cudar  r  r  copy_r  r  r  r#   r@  r   r  rh  r^  r   r  rb   r5  
randn_liker  r  autogradr  r  rV  r6  rA  )rp   r,  	cpu_inputri  cpu_input_tupleis_any_input_complexgpu_input_tuple
cpu_module
gpu_module	cpu_param	gpu_paramcpu_pgpu_p
cpu_output
gpu_outputr?   cpu_gradOutputgpu_gradOutputcpu_gradInputgpu_gradInputcpu_d_pgpu_d_pcpu_gradInputsgpu_gradInputscpu_d_igpu_d_ioutputs_cpuoutputs_gpucpu_gggpu_ggs                                 r)   rO   zModuleTest.test_cuda  sA    5 5##$>??t112 W	H)Iekk2H+5i+Gii\O#&&scr&s#s $_xHO)))4+@+@AJ)))4+@+@AGGINNPJ!11*=I!11*=I #IaL)A,t L (u

  '( &&7&&7++J7++J7"++JHJ"++JHJz#3U;']
']
!!*jt~~TUch!i 1X Y!+!1!1!3!;!;!=!/!7!7
!C ) 3 3JQ[]k l ) 3 3JQ[]k l%%m]^_mr%s(+IaL)A,t(T Y$GW))'7VW)XYY ""4+Q+Q'9
'9
:'7?!+AJ!+AJ!&!1!1*D!Q!/!7!7
!C!J!J!L/3,!&!4!4#eJ,A,A,C&DD"!%	 "5 "'
 "'!4!4#eJ,A,A,C&DD"!%	 "5 "' ),NNSW(X l$GW))'7VWej)kl (",.."2"6"6"83>eVd>e;e"eK",.."2"6"6"83>eVd>e;e"eK",.."2S8Y.8Y5Y"YK",.."2S8Y.8Y5Y"YK,,#~&77%
@U@U@W:XX!% - ' ,,#~&77%
@U@U@W:XX!% - ' %%m]^_mr%s(+FF4(H l$GW))'7VWej)kl 	:GoW	H W	H W	Hs   T)U//U8)ru   rv   rw   r   r   ry   rz   r3  r  r-  rN  rA  rO   __classcell__r<  s   @r)   r1  r1  R  sR    "# "ryy " " " ";46*#UJ[Hr+   r1  c                       e Zd Zd Zy)InputVariableMixinc                 J    t         j                  | d      }fd |      S )NFc                     t        | t        j                        r)| j                         s| j	                         rd| _        | S  t        |       fd| D              S )NTc              3   .   K   | ]  } |        y wrn   ro   )r  r  map_variabless     r)   r  zGInputVariableMixin._get_input.<locals>.map_variables.<locals>.<genexpr>#  s     At}T2A   )r  r7   r  is_floating_pointrg  r  r  )r=   r  s    r)   r  z4InputVariableMixin._get_input.<locals>.map_variables  sH    !U\\*&&(ALLN&*AOtAwAqAAAr+   )r  r*  )rp   r  r  s     @r)   r*  zInputVariableMixin._get_input  s(    ##D%0	B U##r+   N)ru   rv   rw   r*  ro   r+   r)   r  r    s    $r+   r  c                   @     e Zd Z fdZd Zd Zd Zed        Z xZ	S )NewModuleTestc                 x   t        |   |i | |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  d	d
      | _	        |j                  dd      | _
        |j                  dd      | _        |j                  dd      | _        |j                  d      | _        |j                  dd      | _        |j                  dd      | _        y )Nr  Fcheck_inplacerb   Tskip_doubler  rG   rH   r_   test_cpur6  rc   r  r   supports_fwgrad_bwgrad)r7  r  r  r  r  rb   r  r  rG   rH   r  r6  rc   r  r   r  r;  s      r)   r  zNewModuleTest.__init__)  s   $)&)ZZ/
#ZZ?$jj)94@!::mU;K7K7$jj)95A

:t4$*JJ/Eu$M!"(**-A4"H#)::.C#D #)::.CU#K &,jj1I5&Q#r+   c           	         t        d j                         D              }t        |      fd}| j                  r6dk(  sJ t	        j
                  |d         }j                  |d   |       n@j                  t        |||z   | j                  | j                  | j                               | j                  rAj                  t        |||z   | j                  | j                  | j                               y y )Nc              3       K   | ]  }|  y wrn   ro   rm  s     r)   r  z1NewModuleTest._check_gradients.<locals>.<genexpr>:  s     6Qq6r  c                  6    |rJ j                  | d        S rn   )r  )inputs_and_paramsrs   r  
num_inputsr,  s     r)   fn_to_gradcheckz7NewModuleTest._check_gradients.<locals>.fn_to_gradcheck=  s&    :%%f.?.LMMr+   r5   r   )rc   	fast_modecheck_forward_ad)rc   r  check_fwd_over_rev)r  r  r2  r6  r7   r  r  
assertTruer   rc   r  r   rb   r   r  )rp   r,  r  input_tupleparamsr  test_input_jacobianr  s    ``    @r)   _check_gradientszNewModuleTest._check_gradients9  s    6&"3"3"566%
	N $$?"?"'"9"9+a."I$$V[^=PQ  ?K&<P>B>U>U595M5M<@<T<T"V W
   f@TBFBYBY9=9Q9QBFB]B]"_ ` r+   c                    t        j                         }t        j                  d       t        |t              r|n|f}| j                  |       j                          | j                  rt        |      dk(  sJ |d   } | j                  | j                  ddi}|j                  }t               5   |      }d d d        j                  |j                  |       t        |      }	|	j                         }
t               5   ||
      }d d d        j!                  |
j                  |       j                         |j"                  j                         j%                         }|j&                  7t        j(                         5  |j&                  j+                          d d d        |	j&                  7t        j(                         5  |	j&                  j+                          d d d        |j-                  |       |j-                  |       j                  |j&                  |	j&                         dfd	}t/        d |D              rt0        rt	        d |D              }j3                         j5                           |   |t         j4                  j6                  d       t         j4                  j9                         dkD  rt	        d |D              }j5                  d       t         j4                  j;                  d      5   |  d d d         |t         j4                  j6                  d       n@d	 fd
fdfdt	        fd|D              }j3                           |   |t         j6                         t	        fd|D              }j=                           |   |t         j>                         t0        r| j@                  rt	        fd|D              }j3                         j5                           |   |t         j4                  j6                  d       t	        d |D              }jC                           |   |t         j6                         t	        d |D              }j5                           |   |t         j4                  j6                  d       | jD                  rYt         jF                  jD                  jI                  d      5   |   |t         j4                  j6                  d       d d d        t         j4                  j9                         dk\  rqt	        d |D              }j5                  d       t         j4                  j;                  d      5   |  d d d         |t         j4                  j6                  d       | jJ                  sXt	        fd|D              }j=                         j5                           |   |t         j4                  j>                  d       | jL                  sXt	        fd|D              }jO                         j5                           |   |t         j4                  jP                  d       t        j                  |       y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   WxY w)Nr5   r   inplaceTc                     j                         D ]7  }j                  ||        |j                  |j                         |       9 y rn   )r  assertIsInstancer@  
get_device)tensor_type	device_idr>   r  r,  s      r)   assert_module_parameters_arez<NewModuleTest._do_test.<locals>.assert_module_parameters_are  sI    &&( E**1k:())!,,.)DEr+   c              3   P   K   | ]  }t        |t        j                           y wrn   )r  r7   
LongTensorrh  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     D1z!U--.Ds   $&c              3   <   K   | ]  }|j                           y wrn   ry  rh  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     >Q>r  c              3   >   K   | ]  }|j                  d         ywr  r  rh  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #C!AFF1I#C   c                     | j                         r| j                  |      S | j                         r| j                  |      S | S rn   )rg  r   r  )r  realr  s      r)   to_typez'NewModuleTest._do_test.<locals>.to_type  s<    $$&!99W----/!99T?*!Mr+   c                 4     | t         j                  d       S rn   )r7   float16rI  r  s    r)   to_halfz'NewModuleTest._do_test.<locals>.to_half  s    q%--66r+   c                 P     | t         j                  t         j                        S rn   )r7   float32	complex64r  s    r)   	to_singlez)NewModuleTest._do_test.<locals>.to_single  s    q%--AAr+   c                 P     | t         j                  t         j                        S rn   )r7   r  r  r  s    r)   	to_doublez)NewModuleTest._do_test.<locals>.to_double  s    q%--1A1ABBr+   c              3   .   K   | ]  } |        y wrn   ro   r  r9   r  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>       B	!Br  c              3   .   K   | ]  } |        y wrn   ro   r  r9   r  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  r  r  c              3   J   K   | ]  } |      j                           y wrn   r  r  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #MAIaL$5$5$7#M    #c              3   <   K   | ]  }|j                           y wrn   )cpurh  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #AAEEG#Ar  c              3   <   K   | ]  }|j                           y wrn   r  rh  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     #BAFFH#Br  F)enabledrQ   c              3   >   K   | ]  }|j                  d         ywr  r  rh  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     'Gaq	'Gr  c              3   J   K   | ]  } |      j                           y wrn   r  r  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     'Q	!(9(9(;'Qr  c              3   J   K   | ]  } |      j                           y wrn   r  )r  r9   r  s     r)   r  z)NewModuleTest._do_test.<locals>.<genexpr>  s     'Oa
(9'Or  rn   ))r7   get_num_threadsset_num_threadsr  r  r  __repr__r  r2  r   rC   _versionr   r@  r   rh  assertNotEqualr  r^  r  no_gradr9  backwardallr   rx  ry  FloatTensordevice_countrc  rf   DoubleTensorr8  r  r  backendsflagsr  r  half
HalfTensor)rp   r,  r  r  num_threadsr  	module_ipinput_versionr  input_ipinput_ip_clone	output_ipr  r  r  r  r  r  s    ``           @@@@r)   r3  zNewModuleTest._do_testU  sU   ++-a )%7eeXi= 	 {#q(((NE((($*?*?NNI!NNM!# ''!!%..-@H%^^-N!# 6%n5	6$$^%<%<mL!!&)4;;$$&..0Dzz%]]_ 'JJ$$&'}}(]]_ *MM'')*OOD!t$!!%**hmm<	E DDD  >+>>KLLN!K ()?)?Czz&&(1,##C{#CCAZZ&&q) )K(),UZZ-C-CQG"7BC  BkBBKLLNK ():):;  BkBBKMMOK (););<T22
 $#M#MM##%$,UZZ-C-CQG $#A[#AA

$,U->->? $#Bk#BB$,UZZ-C-CQG ::--33E3B P,4UZZ5K5KQOP ::**,1 #('G;'G"GKKKN**1- -,-01G1GK''"''Q['Q"QKMMO((*K(01H1H!L ~~"''O;'O"OKKKM&&(K(01F1FJk*w' '6 6' '* *.) )vP P- -sT   <	Z#	Z0Z=[
['[$[1#Z-0Z:=[
[[!$[.1[;c                 &    | j                  dd      S NrN  Fr  r  s    r)   _get_targetzNewModuleTest._get_target      }}Xu--r+   c                 &    | j                  dd      S NrC   Fr  r  s    r)   rC   zNewModuleTest.constructor_args      }}/77r+   )
ru   rv   rw   r  r  r3  r  r/  rC   r  r  s   @r)   r  r  (  s/    R `8Q+f. 8 8r+   r  c                        e Zd Zej                  j                  dh      Z fdZd ZddZd Z	e
d        Ze
d        Z xZS )	CriterionTestrN  c                 h   t        |   |i | |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  dd      | _        |j                  d	d      | _	        |j                  d
d      | _
        |j                  dd      | _        |j                  dd      | _        |j                  d      | _        | j                  t        j                         | _        y y )NrO   Tr6  Frb   r  check_bfloat16check_complexr  rG   rH   r_   rc   rI   )r7  r  r  r8  r6  rb   r  r  r  r  rG   rH   rc   rI   r7   r:  r;  s      r)   r  zCriterionTest.__init__  s   $)&) &

; ="(**-A5"I$jj)94@ **\48$jj)95A#ZZ?

:t4K6$jj)95A"(**-A4"H#ZZ8%!&!8!8!:D &r+   c                 (  
 t        | j                        5   | j                  | j                   
| j	                         }
j                          t        
       | j                         }| j                  g|j                  
||| j                        }t        |      t        |      f| j                  z   
fz   } | j                  | }|j                  ||       | j                  r
	 d d d        y t        d 
j                         D              }t!        |t              s|f|z   |fz   }
fd}	n||z   |fz   }
fd}	t#        |	|| j$                         | j&                  rt)        |	|| j$                         d d d        y # 1 sw Y   y xY w)Nr	  c              3       K   | ]  }|  y wrn   ro   rm  s     r)   r  z)CriterionTest.__call__.<locals>.<genexpr>  s     :1:r  c                      | |      S rn   ro   )r  rN  r  r  s      r)   apply_fnz(CriterionTest.__call__.<locals>.apply_fn  s    !%00r+   c                      | ||      S rn   ro   )rQ  rR  rN  r  r  s       r)   r  z(CriterionTest.__call__.<locals>.apply_fn"  s    !&&&99r+   )rc   )r   rI   r   rC   r*  r  r  r  rF   _forward_criterionr	  r   r@  r6  r  r  r  r   rc   rb   r   )rp   r,  r  rN  rG  ref_argsrJ  r  inputsr  r  s             @r)   r-  zCriterionTest.__call__  s~   t112 "	\%T%%t'<'<=FOO%E OOK%%'F  ,2265&UYUdUd2e$UOXf-=>PTZS\\0t00(;%%c<8&&#"	\ "	\& :f&7&7&9::FeU+F*fY61 6)3: h4;R;RS""h4CZCZ[E"	\ "	\ "	\s   CF5B
FFc                    d
fd	t         r| j                  st        j                  d      t	        | j
                        5  | j                         }| j                         } | j                  | j                   } | j                  | j                   } ||d      }|j                         s|j                         r	 ||      }|j                  |       |j                  |       t        |      }t        |      }	|j                          |t        j                   t        j"                  hv r9| j                         }| j                         } | j                  | j                   }|j%                  ||||      }
|j%                  |||	|      }|j'                  |
||t        j                   t        j"                  hv rdnddd	       |j)                  |||
||      }|j)                  ||||	|      }|j'                  |||t        j                   t        j"                  hv rdnddd	       d d d        y # 1 sw Y   y xY w)NFc                     t        | t        j                        r/| j                         j	                        j                        S t        | t              rt        fd| D              S | S )Nr   c              3   2   K   | ]  } |        y wrn   ro   )r  rO  convert_dtyper   r  s     r)   r  zACriterionTest.test_cuda.<locals>.convert_dtype.<locals>.<genexpr>/  s     Q]1e]CQs   )r  r7   r  r  r   requires_grad_r  )rS  r   r  r  s    ``r)   r  z.CriterionTest.test_cuda.<locals>.convert_dtype+  sS    #u||,zz|U3BB=QQC'QSQQQ
r+   re  Tr  rS   g-C6:?r   rj  r  )r   r8  rv  rw  r   rI   r*  r  r   rC   r  rg  r  r   ry  r7   r  bfloat16r  r@  _backward_criterion)rp   r,  r   r	  r}  
cpu_targetr  r  	gpu_input
gpu_targetr  r  r  r  r  s                 @r)   rO   zCriterionTest.test_cuda*  s7   	  5 5##$>??t112 %	{)I))+J)))4+@+@AJ)))4+@+@AJ &i=I++-1F1F1H*:u=
OOE"OOE" y)I
+JOO U^^44 OO-	!--/
-T--t/D/DE
"55j)Zdn5oJ"55j)Zdn5oJ!!*j/4U^^8T/TtZ^efty " { &99Iz:* : VM%99Iz:* : VM !!-/4U^^8T/TtZ^efty " {I%	{ %	{ %	{s   G3I  I	c                 &    | j                  dd      S r  r  r  s    r)   r  zCriterionTest._get_target]  r  r+   c                 &    | j                  dd      S r  r  r  s    r)   rC   zCriterionTest.constructor_args`  r  r+   c                 &    | j                  dd      S )Nr	  Fr  r  s    r)   r	  zCriterionTest.extra_argsd  s    }}\511r+   rn   )ru   rv   rw   r  r  unionr  r-  rO   r  r/  rC   r	  r  r  s   @r)   r  r    s^     #66<<hZH; #\J1{f. 8 8 2 2r+   r  c                    t        j                  |t         j                  |d      }|Et        j                  |t         j                  |      |z  j                         j                         } ||      }t        j                  ||      }|j                  |       |j	                         }	|j                         j	                         j                         }
|j	                         } |	|
      }|j                  |       | j                  ||||d       | j                  |j                  j                  |
j                  j                  ||d       y )NT)r   rc  r  rb  )rc  Frj  )r7   r   r  r   r  rx  r  r{  r  r  r@  r  r  )r,  oprc  inp_dimsprecr?  rQ  out1grad_input1op_bfp16rR  grad_input2out2s                r)   _test_bfloat16_opsr   i  s
   [[vUYZF**XU^^FKlZaacrrtf:D""47KMM+ {{}H]]_%%'668F&&(KFDMM+$4dN&++**FKK,<,<4d`efr+   c                    |s|j                  d        ||      }|s&t        j                  |      }|j                  |       |r.| j	                  |j                         |j                                |s|j                         D ]H  }|j                  s| j	                  |j                  t        j                  |j                               J | j	                  |j                  t        j                  |             y y )NT)
r  r7   	rand_liker  r@  rg   r  r  r  r  )r,  r  r  
check_size	inferencerG  gOr>   s           r)   _test_module_empty_inputr&  |  s    4 
+C__S!Rchhj#((*5""$ 	HA%%affe.>.>qvv.FG	H 	chh(8(8(=>	 r+   c                       G d dt         j                         G fddt         j                        }         } |        }t        j                  ||      }|||fS )Nc                         e Zd Zd fdZ xZS ) _create_basic_net.<locals>.Layerc                     t         |           t        j                  t	        j
                  dd            | _        t        j                  t	        j                  dddd            | _	        y )NrZ   rU   r5   r  )
r7  r  ry   r  r7   rr  layer_dummy_paramBufferr  layer_dummy_buf)rp   r<  s    r)   r  z)_create_basic_net.<locals>.Layer.__init__  sJ    G%'\\%++a2C%DD"#%99U[[Aq!-D#ED r+   r  Nru   rv   rw   r  r  r  s   @r)   Layerr)    s    	F 	Fr+   r0  c                   $     e Zd Zd fdZ xZS )_create_basic_net.<locals>.Netc                     t         |                   | _        t        j                  t        j                  dd            | _        t        j                  t        j                  dddd            | _
        y )NrZ   rU   r  r5   )r7  r  l1ry   r  r7   rr  dummy_paramr,  r  	dummy_buf)rp   r0  r<  s    r)   r  z'_create_basic_net.<locals>.Net.__init__  sR    GgDG!||EKK1,=>DYYu{{1aA'>?DNr+   r.  r/  )r<  r0  s   @r)   Netr2    s    	@ 	@r+   r7  )ry   rz   
Sequential)r7  lr;  r$  r0  s       @r)   _create_basic_netr:    sR    F		 F@bii @ 	AA
aAa7Nr+   )r  F)Nr  )Nr  r  )Nr;  r  r  )r  r[  )r  )r[  r  )r5   r5   Nr  )r   r  )r[  rQ   r  Fr  )ro   r  NrY  )abcr   rB  rv  copyr   	functoolsr   r   	itertoolsr   operatorr   r7   
torch.cudatorch.nnry   torch.nn.functional
functionalr   r	   r$   torch.testing._internalr
   $torch.testing._internal.common_utilsr   r   r   r   r   r   r   r   r   #torch.testing._internal.common_cudar   r   torch.autograd.gradcheckr   r   torch.autogradr   torch.typesr   torch.backends.cudnntypingr   r   collections.abcr   r   rC  r  r*   r/   r   rf   ry  get_device_propertiesgcnArchNamemodule_testsri   r{   r   r   r   r   r   r   r   r   r   r  r  r  r"  r(  r9  r@  rG  rY  r^  rv  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r&  r,  r9  rB  rJ  rX  r^  rl  rr  rx  r  r  r  r  r  r  r
  r  r  r  r'  r,  r0  r5  r<  r@  rB  rF  rJ  rS  r^  r`  rx  r   __annotations__criterion_testsr  regression_criterion_no_batch
reductionsr  r    regression_test_infor  !classification_criterion_no_batchr  r  classification_cpp_parityr   r  r  classification_test_infor  r  r  r  r1  r  r  r  r   r&  r:  ro   r+   r)   <module>rY     s       %        - 0S S S F K # (   $ $&&	'| 	 >Zll	 	'J:%d'UZZ5U5UVW5X5d5d*dell 	ll	 	#Nll 	6ll	 	#+] ll	]8|6
$$ $(&$$$$ "$ $ "$ $$$ $ $"$$$,$.$.$"$$$,$"$$$,$"$"$"$"$""$$$$$$$"$"$"$"$*$$$$$$$$$($,6"eP3 BF"(4 TZ=@, Z^RU%IP [a14
  @D &, 	 4.  ]b*0 "L %#$7DI $*$:6.0844 4- D) & D!  %
<jI 
1OD)6	{3'+S8!ll /0
1  
1I*9+6=/*8!ll /0
1" 	AI
 DFpqEGklGIlmACghPRijX3 DFde 	X2
  !KMcd1% !6 -u5A ,d39o 
   %
.56W6@/B 5*T8Y#6	{3'+S"'$)8 ,00t<  >AA$KJ##J/345 x> x>vJ" J"ZDH DHN$ $E8&
 E8Pv2& v2rg&? r+   