
    9jy                    ~M   U d dl Z d dlZd dlmZ d dlmZ d dlmZmZ d dl	m
Z
mZ d dl	Z	d dlZd dlmc mZ d dlmZ d dlmZ d dlmZ d d	lmZmZmZ d d
lmZmZmZmZm Z m!Z!m"Z"m#Z# d dl$m%Z% d dl&m'Z'm(Z(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4 d dl5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z;m<Z< d dl=m>Z> d dl?Z?e j                  j                  e j                  j                  j                  j                  e j                  j                  j                  j                  e j                  j                  j                  j                  e j                  j                  j                  j                  gZEeFe>   eGd<   e j                  j                  e j                  j                  e j                  j                  e j                  j                  j                  j                  e j                  j                  j                  j                  hZLeMeN   eGd<   g  e
j                  eED  cg c]#  } | j                  D cg c]  } eQ| |       c}% c}}       ZReFeN   eGd<   eRD cg c]	  }|eLvs| c}ZRi ZSeTeNeUf   eGd<   eED ]W  ZVeVj                  D ]F  ZW eQeVeW      ZXeVj                  j                  dd      j                  dd      Z[eXeSvs=e[ deW eSeX<   H Y  edd      Z\ G d de      Z@d Z] G d d      Z^ G d d      Z_ G d  d!e      Z` G d" d#      Za G d$ d%      Zbd& Zcd' Zdd( Zed) Zfd* Zgd+ Zhd, Zid- Zjd. Zkd/ Zld0 Zmd1 Znd2 Zod3 Zpd4 Zqd5 Zrd6 Zsd7 Ztd8 Zud9 Zvd: Zwd; Zxd< Zyd= Zzd> Z{d? Z|d@ Z}dA Z~dB ZdC ZdD ZdE ZdF ZdG ZdH ZdI ZdJ ZdK ZdL ZdM ZdN ZdO ZdP ZdQ ZdR ZdS ZdT ZdU ZdV ZdW ZdX ZdY ZdZ Zd[ Zd\ Zd] Zd^ Zd_ Zd` Zda Zdb Zdc Zdd Zde Zdf Zdg Zdh Zdi Zdj Zdk Zdl Zdm Zdn Zdo Zdp Zdq Zdr Zds Zdt Zdu Zdv Zdw Zdx Zdy Zdz Zd{ Zd| Zd} Zd~ Zd Zd Zd Zd Zd ZddZd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Zd Z e%ej                  ddexr e9 d       e%ej                  ddexr e9 d       e%ej                  ddexr e9 d       e%ej                  ddexr e9e j                  fd      fZd Zd Zd Zd Zd Zd Zd Ze j                  j                  j                         xr) e j                  j                  j                  dd       Zg  ebe j                  j                  es e%e#      f       ebe j                  j                  e8et e%e#       e%ej                  dd e?j                  d            f       ebe j                  j                  e8eu e% ej                  d      dd       e%e#      f       ebe j                  j                  ev       ebe j                  j                  e8ew       ebe j                  j                  e8ex e% ej                  d      dd       e%e#      f       ebe j                  j                  ep       ebe j                  j                  eq e%ej                  dd e?j                  d      d      f       ebe j                  j                  ere8 e% ej                  d      dd       e%e#dd      f       ebe j                  j                  deye| e%ej                  dd e?j                  d             e%ej                  dd e?j                  d            f       ebe j                  j                  deze| e%edd e?j                  d             e%ej                  dd e?j                  d             e%ej                  dd e?j                  d            f       ebe j                  j                  de{e| e%e#       e%ej                  dd e?j                  d             e%ej                  dd e?j                  d            f       ebe j                  j                  e e%ej                  ddde j                  g      f       ebe j                  j                   ee}dd      e8d e% e!e j                  di      dd      f       ebe j                  j                   ee}dd      e8d e%ej                  ddde j                  g      f e% e!e j                  di      dd      f       ebe j                  j                   ee}dd      e8d e%e#d       e%ej                  dd      f e% e!e j                  di      dd      f       ebe j                  j                   ee}ddd      e8d ee j                         e%ej                  dde j                  fd      f e% e!e j                  di      dd       e% e!e j                  di      dd      f«       ebe j                  j                   ee}ddd      e8d ee j                         e%ej                  dde j                  e j                  e j                  fë       e%ej                  ddde j                  e j                  g       e%ej                  dde j                  fd       e%e<dde j                  fdĬ      f e% e!e j                  di      dd       e% e!e j                  di      dd      f«       ebe j                  j                   ee}ddd       ee j                        e8d e%e#       e%ej                  dd       e%ej                  dde j                  fd       e%e<dde j                  fdĬ      f e% e!e j                  di      dd       e% e!e j                  di      dd       e% e!e j                  di      dd      fū       ebe j                  j                  e~ e% ej                  d      dd      f       ebe j                  j                  e e%ej                  ddde j                  g      f       ebe j                  j                  ee8 e%e#       e% ej                  d      dd      f       ebe j                  j                   ee8 e%e#       e% ej                  d      dd      f       ebe j                  j                  e e% ej                  d      dd      f       ebe j                  j                  e e% ej                  d      dd       e%e7dde j                  gd      f       ebe j                  j                   ee}dd      e8d e%e"      f e% e!e j                  di      dd      f       ebe j                  j                   ee}dd      e8d e%e"       e%ej                  ddde j                  g      f e% e!e j                  di      dd      f       ebe j                  j
                   ee}dd      e8d e%e"       e%e#       e%ej                  dd      f e% e!e j                  di      dd      f       ebe j                  j                   ee}ddd      e8d e%e"      f e% e!e j                  di      dd      f       ebe j                  j                   ee}ddd      e8d e%e"       e%ej                  ddde j                  g      f e% e!e j                  di      dd      f       ebe j                  j                   ee}ddd      e8d e%e"       e%e#       e%ej                  dd      f e% e!e j                  di      dd      f       ebe j                  j                  ec e% ej                  d      dd      f       ebe j                  j                  ed e% ee j                   e ddƫ      e j                   e ddƫ      i      dddȬ      g e% ej                  d      dd       e%ej                  ddde j                  g      fɫ       ebe j                  j                  e e% ej                  d      dd       e% ej                  d      dd      f       ebe j                  j                  e e% ej                  d      dd       e% ej                  d      dd       e%ej                  dd e?j                  d      xr e d      f       ebe j                  j                  e e% ej                  d      dd       e% ej                  d      dd       e% ej                  d      dd       e%e7d      f       ebe j                  j                  e       ebe j                  j                  eeʫ       ebe j                  j                  ee8˫       ebe j                  j                   ee e% ej                  d      dd       e% ej                  d      dd       e% ej                  d      dd       e% ej                  d      dd      f       ebe j                  j"                  ei e% ej                  d      dd       e%ej                  ddde j                  g      f       ebe j                  j$                  e e% ej                  d      dd      f       ebe j                  j&                  e e% ej                  d      dd       e%e7dd       e% ej                  d      dd      f       ebe j                  j(                  ee e% ej                  d      dd       e%e7dd       e% ej                  d      dd      f̫       ebe j                  j*                  e e% ej                  d      dd       e%ej                  ddde j                  g      f       ebe j                  j,                  e e% ej                  d      dd      f       ebe j                  j.                  efe e% ej                  d      dd       e%ej                  ddde j                  g      f̫       ebe j                  j0                  eg e% ej                  d      dd      f       ebe j                  j2                  eh e% ej                  d      dd      f       ebe j                  j4                  e e% ej                  d      dd      f       ebe j                  j6                  e e% ej                  d      dd       e%ej                  ddde j                  g      f       ebe j                  j8                  e e% ej                  d      dd       e%e7de j                  gd      f       ebe j                  j:                  e e% ej                  d      dd       e%e7de j                  gd      f       ebe j                  j<                  ee eddͫ       e%ej                  dd       e% ee j                   e ddϬƫ      i      dde j                  gdȬ       e%ej                  dde j                  gd       e%ej                  dde j                  gdĬ      fЫ       ebe j                  j>                  e e% ej                  d      dd       e%e7dd       e% ej                  d      dd       e% ej                  d      dd       e% ej                  d      dd      f       ebe j                  j@                  e e%ej                  ddde j                  g      f       ebe j                  jB                  e       ebe j                  jD                  ee eddѫ       e% ej                  d      dd       e% ee j                   e ddƫ      i      dddȬ       e% ej                  d      ddd       e% ej                  d      ddd       e% ej                  d      dddĬ       e% ej                  d      dde9d      fҫ       ebe j                  jF                  e       ebe j                  jH                  edӫ       ebe j                  jJ                  e       ebe j                  jL                   eedԫ      d e% ej                  d      dd      fի       ebe j                  jN                   eedԫ      d e% ej                  d      dd      fի       ebe j                  jP                   eedԫ      d e%edd       e%edd       e%eddǫ       e%edd׫       e%eddث       e% ej                  d      dd      fի       ebe j                  jR                  e       ebe j                  jT                  e e% ej                  d      dd      f       ebe j                  jV                  e       ebe j                  jX                  de e% ej                  d      dddȬ      g e% ej                  d      dd       e%ej                  dd٫      fګ       ebe j                  jZ                  de e% ee j                   e ddƫ      i      ddde:۫       e% ee j                   e ddܬƫ      i      ddd       e% ej                  d      dddȬ      g e% ej                  d      dd      fګ       ebe j                  j\                  e e% ej                  d      dddȬ      g e% ej                  d      dd      fɫ       ebe j                  j^                  ed e% ej                  d      dddȬ      g e% ej                  d      dd      fݫ       ebe j                  j`                  de e% ej                  d      dd      fޫ       ebe j                  jb                  eee8 e% ee j                   e ddƫ      i      ddd      g e% ej                  d      dd      f߫       ebe j                  jd                  eerdn) e%ej                  dd e?j                  d      d      f       ebe j                  jf                  e       ebe j                  jh                  e e%e#      f       ebe j                  jj                  e e%e#      f       ebe j                  jl                   eed      e֬ʫ       ebe j                  jn                  ee֬ʫ       ebe j                  jp                  ee׬ʫ       ebe j                  jr                  eerdn) e%ej                  dd e?j                  d      d      f       ebe j                  jt                  e e%ej                  ddde j                  g      f       ebe j                  jv                  e       ebe j                  jx                  e       ebe j                  jz                  e e% ej                  d      dd       e%ej                  ddde j                  g      f       ebe j                  j|                  e e% ej                  d      dd       e%ej                  ddde j                  g      f       ebe j                  j~                  e e% ej                  d      dd      f       ebe j                  j                  e e%e#      f       ebe j                  j                  e e%e#      f       ebe j                  j                  e       ebe j                  j                  eerdn) e%ej                  dd e?j                  d      d      f       ebe j                  j                  eerdn) e%ej                  dd e?j                  d      d      f       ebe j                  j                  e e%e#      f       ebe j                  j                  e e%e#      f       ebe j                  j                  d eed      eeԬ       ebe j                  j                  d eed      eeԬ       ebe j                  j                  dee e%e#      feԬ       ebe j                  j                  eì       ebe j                  j                  ee8 e% ej                  d      ddd       e% ej                  d      ddd       e% ej                  d      dddĬ      f       ebe j                  j                  ee8 e% ej                  d      ddd       e% ej                  d      ddd       e% ej                  d      dddĬ      f       ebe j                  j                  eƬ       ebe j                  j                  ee8 e% ej                  d      ddd       e% ej                  d      ddd       e% ej                  d      dddĬ      f       ebe j                  j                  ee8 e% ej                  d      ddd       e% ej                  d      ddd       e% ej                  d      dddĬ      f       ebe j                  j                  e e%e#      f       ebe j                  j                  eɬ       ebe j                  j                  e e%ej                  ddd      f       ebe j                  j                  e e%ej                  ddd      f       ebe j                  j                  ee٬ʫ       ebe j                  j                  eeڬʫ       ebe j                  j                  ee e%ej                  dd      f̫       ebe j                  j                  e̬       ebe j                  j                  e e%ej                  ddd      f       ebe j                  j                  e e%ej                  ddd      f      ZZeFeb   eGd<   yc c}w c c}} w c c}w )    N)deepcopy)Enum)wrapspartial)chainproduct)pack_padded_sequence)make_tensor)
TEST_CUDNN)floating_typesfloating_and_complex_types_andget_all_fp_dtypes)_TestParametrizer_update_param_kwargsexpectedFailureMPStoleranceOverridetolprecisionOverrideskipMetaskipMPS)DecorateInfo)cosineembeddingloss_referencecross_entropy_loss_referencectcloss_referencehingeembeddingloss_referencehuberloss_referencekldivloss_referencemarginrankingloss_referencemultimarginloss_referencemultilabelmarginloss_referencenllloss_referencenlllossNd_referencesmoothl1loss_referencesoftmarginloss_referenceget_reduction)freeze_rng_state	skipIfMPSGRADCHECK_NONDET_TOLTEST_WITH_ROCM
IS_WINDOWSskipIfTorchDynamo	skipIfXpu)
ModuleTypeMODULE_NAMESPACESMODULES_TO_SKIPMODULE_CLASSESMODULE_CLASS_NAMESztorch. z.modules.TrainEvalMode)
train_only	eval_onlytrain_and_evalc                   >    e Zd ZdZdej
                  dfdZd Zd Zy)moduleszQ PROTOTYPE: Decorator for specifying a list of modules over which to run a test. NTc                 h    t        |      | _        |t        |      nd | _        || _        || _        y N)listmodule_info_listsetallowed_dtypestrain_eval_modeskip_if_dynamo)selfmodule_info_iterabler?   r@   rA   s        f/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/testing/_internal/common_modules.py__init__zmodules.__init__L   s5     $%9 :5C5Oc.1UY.,    c                 X   g }| j                   t        j                  k(  s| j                   t        j                  k(  r|j	                  d       | j                   t        j
                  k(  s| j                   t        j                  k(  r|j	                  d       |j                  s|d d }|S )NTF   )r@   r4   r5   r7   appendr6   train_and_eval_differ)rB   module_infotraining_flagss      rD   _get_training_flagszmodules._get_training_flagsS   s      M$<$<<$$(D(DD!!$'  M$;$;;$$(D(DD!!%( 00+BQ/NrF   c           
   #   P  K   |t        d      | j                  D ]U  }t        |j                  |j                              }| j
                  |j                  | j
                        }| j                  |      }t        ||      D ]  \  }}|j                  }	t        |      dkD  r|	d|rdnd z  }	d|i}
t        |
d|       t        |
d|       	 t              fd	       }| j                  r?t        j                  j                   j"                  j$                  s t'        d
      |      }t)        |j*                  |j,                  j,                  |j                  |      }||	|
|f  X y # t.        $ r#}t1        d|	 d|j2                   d       |d }~ww xY ww)NzThe @modules decorator is only intended to be used in a device-specific context; use it with instantiate_device_type_tests() instead of instantiate_parametrized_tests()rH   _
train_mode	eval_moderK   dtypetrainingc                       | i |S r;    )argskwargstests     rD   test_wrapperz/modules._parametrize_test.<locals>.test_wrapper}   s    #T4V44rF   z/Policy: we don't run ModuleInfo tests w/ DynamozFailed to instantiate z for module !)RuntimeErrorr=   r>   supported_dtypesdevice_typer?   intersectionrM   r   formatted_namelenr   r   rA   torchtesting	_internalcommon_utilsTEST_WITH_TORCHINDUCTORr+   r   get_decorators__name__	Exceptionprintname)rB   rX   generic_cls
device_clsrK   dtypesrL   rS   rR   	test_nameparam_kwargsrY   decorator_fnexs    `            rD   _parametrize_testzmodules._parametrize_testc   s      B C C  00 "	K55j6L6LMNF"".,,T-@-@A!55kBN%,^V%D !5 (66	~&*1X\;$O!PPI !.{;$\7EB$\:xH4[5 !5 **5==3J3J3W3W3o3o'k'89j'klx'y#*;+E+E{G[G[+/==*:P:PRW$YL (L,OO1"	> ! 29+\+JZJZI[[\]^Hs+   CF&BE71F&7	F# FF##F&)	rg   
__module____qualname____doc__r4   r7   rE   rM   rr   rU   rF   rD   r9   r9   I   s"    [<@!.!=!=d- (rF   r9   c                 <    | t         v r	t         |    S | j                  S r;   )r1   rg   )
module_clss    rD   get_module_common_namerx      s"    ''!*--"""rF   c                       e Zd ZdZddgZd Zy)FunctionInputz: Contains args and kwargs to pass as input to a function. rV   rW   c                      || _         || _        y r;   )rV   rW   )rB   rV   rW   s      rD   rE   zFunctionInput.__init__   s    	rF   Nrg   rs   rt   ru   	__slots__rE   rU   rF   rD   rz   rz      s    D"IrF   rz   c                        e Zd ZdZg dZddZy)ModuleInputzA Contains args / kwargs for module instantiation + forward pass. constructor_inputforward_inputdescreference_fnNc                 x    || _         || _        || _        | _        t	              fd       }|| _        y y )Nc                 x    t        |      t        |      }} | t        | j                               g|i |S r;   )r   r<   
parameters)mrV   rW   r   s      rD   copy_reference_fnz/ModuleInput.__init__.<locals>.copy_reference_fn   s;      (~x/?f $AtALLN';MdMfMMrF   )r   r   r   r   r   )rB   r   r   r   r   r   s       ` rD   rE   zModuleInput.__init__   sP    !2*	(#< N !N !2D $rF   )Nr2   Nr|   rU   rF   rD   r   r      s    KNI2rF   r   c                       e Zd ZdZdZdZy)ModuleErrorEnumz7 Enumerates when error is raised when testing modules. r   rH   N)rg   rs   rt   ru   CONSTRUCTION_ERRORFORWARD_ERRORrU   rF   rD   r   r      s    AMrF   r   c                   :    e Zd ZdZg dZej                  eddZy)ErrorModuleInputzw
    A ModuleInput that will cause the operation to throw an error plus information
    about the resulting error.
    module_error_inputerror_on
error_typeerror_regex)r   r   c                <    || _         || _        || _        || _        y r;   r   )rB   r   r   r   r   s        rD   rE   zErrorModuleInput.__init__   s"     #5 $&rF   N)	rg   rs   rt   ru   r}   r   r   r[   rE   rU   rF   rD   r   r      s"    
 PI
 *<<(		'rF   r   c                       e Zd ZdZdd e       ej                  ej                  fej                  ej                  fddddddddZ	d	 Z
d
 Zed        Zed        Zy)
ModuleInfoz+ Module information to be used in testing. rU   NT        F)skips
decoratorsrm   dtypesIfMPSdtypesIfHpusupports_gradgradgradcheck_nondet_tolmodule_memformat_affects_outrJ   module_error_inputs_funcgradcheck_fast_modec                @   || _         || _        g |r|ng |r|ng | _        || _        || _        || _        || _        |	| _        |
| _        || _	        || _
        || _        t        |t        j                  j                  j                   j"                        | _        y r;   )rw   module_inputs_funcr   rm   r   r   r   r   r   rJ   r   r   
issubclassra   nnr9   lazyLazyModuleMixinis_lazy)rB   rw   r   r   r   rm   r   r   r   r   r   rJ   r   r   s                 rD   rE   zModuleInfo.__init__   s    & %"4Z:Z2Z5%VXZ&&!2$8!,H)%:"(@%#6 !*ehh.>.>.C.C.S.STrF   c                     g }| j                   D ]U  }t        |t              r2|j                  |||||      s)|j	                  |j                          E|j                  |       W |S r;   )r   
isinstancer   	is_activeextendrI   )rB   
test_classrn   devicerR   ro   result	decorators           rD   rf   zModuleInfo.get_decorators   s_     	)I)\2&&z9fe\ZMM)"6"67i(	) rF   c                 ^    |dk(  r| j                   S |dk(  r| j                  S | j                  S )Nmpshpu)r   r   rm   )rB   r]   s     rD   r\   zModuleInfo.supported_dtypes   s4    %###E!###;;rF   c                 ,    t        | j                        S r;   )rx   rw   rB   s    rD   rj   zModuleInfo.name  s    %doo66rF   c                 :    | j                   j                  dd      S )Nr3   rO   )rj   replacer   s    rD   r_   zModuleInfo.formatted_name  s    yy  c**rF   )rg   rs   rt   ru   r   ra   float16float32bfloat16rE   rf   r\   propertyrj   r_   rU   rF   rD   r   r      s~    5  &(#mmU]]<#nnemm=#'&).3',*.%)UB 7 7 + +rF   r   c           
      $   t        t        |||      }t        t        dd      t         |d            d       t        t        ddd	      t         |d            d
d       t        t        dd      t         |d            dd       g}|S )Nr   rR   requires_grad
         r   )inputc                     t        j                  ||d   j                               |d   j                  dd      j	                  dd      z   S )Nr   rH   r   r   )ra   mmtviewexpand)r   pr   s      rD   <lambda>z/module_inputs_torch_nn_Linear.<locals>.<lambda>  sE    UXXeQqTVVX5NQRSTQUQZQZ[\^`QaQhQhijlmQn5n rF   r   r   r   Fbiasno_biasc                 P    t        j                  ||d   j                               S )Nr   )ra   r   r   r   r   is      rD   r   z/module_inputs_torch_nn_Linear.<locals>.<lambda>  s    !QqTVVX1F rF   r         no_batch_dimc                     t        j                  |j                  dd      |d   j                               j                  d      |d   z   S )NrH   r   r   )ra   r   r   r   r   s      rD   r   z/module_inputs_torch_nn_Linear.<locals>.<lambda>  s=    !&&B-11R1W1WXZ1[^_`a^b1b rF   r   r
   r   rz   )rK   r   rR   r   rS   rW   
make_inputmodule_inputss           rD   module_inputs_torch_nn_Linearr     s    V5P]^J 	mB&:"/j6I"J!n	p 	mB&F"/
70C"D"!F	H 	mAq&9"/
1">'!b	dM rF   c                 b   t        t        |||      }ddt        t        ddd      t         |d       |d                  t        t        dddd	
      t         |d       |d            dfd      t        t        ddd      t         |d       |d            dfd      g}|S )Nr   c                     t        j                  d||d   |      }|rp|j                  d   dk(  r|j                  d      |d   z   }|S ||d   j                  dd      j	                  |j                  d   |d   j                  d         z   }|S )Nzbn,anm,bm->bar   rH   r   )ra   einsumshaper   r   )r   r   x1x2r   r   s         rD   bilinear_reference_fnz>module_inputs_torch_nn_Bilinear.<locals>.bilinear_reference_fn%  s    or1Q4<xx{aR1Q4/   !A$))Ar"2"9"9"((1+qtzzRS}"UUrF      r   r   )r   r   )r   r   r   Fr   r   c                      | |||d      S )NFr   rU   r   r   r   r   r   s       rD   r   z1module_inputs_torch_nn_Bilinear.<locals>.<lambda>5  s    6KAqRTVX_d6e rF   r   r   c           	      Z     | ||j                  dd      |j                  dd            S )NrH   r   )r   r   s       rD   r   z1module_inputs_torch_nn_Bilinear.<locals>.<lambda>9  s3    6KAqRTRYRYZ[]_R`bdbibijkmobp6q rF   )Tr   )	rK   r   rR   r   rS   rW   r   r   r   s	           @rD   module_inputs_torch_nn_Bilinearr   "  s    V5P]^J 	mAq!&<"/
60BJvDV"W!6	8 	mAq!%&H"/
60BJvDV"W"!e	g 	mAq!&<"/
1z!}"M'!q	sM rF   c           
      b   t        t        |||      }di fdddifdddifddd	ifd
d
difg}g }|D ]  \  }	}
|
fd} |d      j                         }|j                  d
d      r |d      n |d      j                         }|j	                  t        t        di |
t        ||      |	|              |d      j                         }|j                  d
d      r |d      n |d      j                         }|j	                  t        t        di |
t        ||      d|	z   |              |S )Nr   r2   reduction_sum	reductionsumreduction_batchmean	batchmeanreduction_nonenone
log_targetTc                     t        ||fi |S r;   )r   r   r   r   r   constructor_kwargss        rD   r   z6module_inputs_torch_nn_KLDivLoss.<locals>.reference_fnL      &q!B/ABBrF   r   r   Fr   rU   scalar_)r   r
   loggetrI   r   rz   )rK   r   rR   r   rS   rW   r   casesr   r   r   r   r   targetscalar_inputscalar_targets                   rD    module_inputs_torch_nn_KLDivLossr   ?  sh   V5P]^J 
R	;./	k :;	K01	d+,%E M$) 
  8J 	C 8$((*)/L%)HH%jYaNbNfNfNh-*M:L*M&3E6&B!%13	
 ""~))+ %jju=JrN:b>CUCUCW 	 	-*M:L*M&3L,&O&-%13	
%
2 rF   c                 t   |||fd}t        t        ||d      }di fdddifddd	ifd
d
difdd |d      j                         ifd |d      j                         ddfd |d      j                         ddfg}g }	|D ]4  \  }
}|fd}|	j                  t	        t        d i |t         |d      t        j                  d|      j                         j                  d      j                         j                               |
|             |fd}|	j                  t	        t        d i |t         |d      t        j                  ddd|      j                         j                  d      j                         j                               d|
 |             |	j                  t	        t        d i |t         |d      t        j                  ddddd|      j                         j                  d      j                         j                               d|
 |             |	j                  t	        t        d i |t         |d      t        j                  dd|      j                         j                  d      j                         j                               d|
 |             7 |	S )!Nc                 ^    t        | ||d      j                  d      j                  |      S )NFr   rH   dim)r
   log_softmaxrequires_grad_)r   r   rR   r   s       rD   r   z2module_inputs_torch_nn_NLLLoss.<locals>.make_inputh  s/    5u).00;0B>>R_C`	arF   Fr   r2   r   r   r   r   r   ignore_indexr   weightsweightr   weights_ignore_index)r  r  weights_ignore_index_negr   c                     t        ||fi |S r;   )r!   r   s        rD   r   z4module_inputs_torch_nn_NLLLoss.<locals>.reference_fn~  s    $Q@-?@@rF   )   r   r	  r   r   c                     t        ||fi |S r;   )r"   r   s        rD   nd_reference_fnz7module_inputs_torch_nn_NLLLoss.<locals>.nd_reference_fn  r   rF   )r   r   r   r   r   nd_)r   r   r   r   r   r   higher_dim_)r   r   r   3d_rU   )r   r
   absrI   r   rz   ra   emptyuniform_mulfloorlong)rK   r   rR   r   rS   rW   r   make_weightr   r   r   r   r   r  s                 rD   module_inputs_torch_nn_NLLLossr  g  s   !'uM a +fEQVWK 
R	;./	K01	.!,-	X{1~11345	KN,>,>,@RS!TU	#A0B0B0DVX%YZ%E M$) )
  8J 	A 	-*M:L*M&3Jw4G49KK64R4[4[4]4a4abc4d4j4j4l4q4q4s'u!%1	3	
 <N 	C 	-*M:L*M&3&|4!KK1a?HHJNNqQWWY^^`'b  #4&\%46	
 	-*M:L*M&3&'9:!KK1aAfENNPTTUVW]]_ddf'h  +4&1%46	
 	-*M:L*M&3&y1!KK1V<EEGKKANTTV[[]'_  #4&\%46	
E)
V rF   c                    t        t        |||      }t        t        ||d      }di fdddifdddifd	dd
ifdddifg}g }	|D ]  \  }
}|j                  dd      }|r |dd      j                         n |dd      j                         }|	j	                  t        t        di |t         |dd       |dd      |      |
t                      |	S )Nr   Fr2   r   r   r   reduction_meanmeanr   r   homoscedasticTrH   r   r   r   rU   )r   r
   popr  rI   r   rz   no_batch_dim_reference_fn)rK   r   rR   r   rS   rW   r   make_targetr   r   r   r   r  	var_inputs                 rD   &module_inputs_torch_nn_GaussianNLLLossr     s   V5P]^J+fEQVWK 
R	;./	K01	K01	?D12%E M$) 

  *..F.;Jq!$((*AqAQAUAUAW	-*M:L*M&3Jq!4D4?14E4='? "%>@	


 rF   c                    t        t        |||      }t        t        ||d      }di fdddifdddifd	dd
ifdddifdddifddddfg}ddg }	|D ]  \  }
}|ffd	}|j                  dd      }|r |d      n$ |d      j                         j	                  d      }|	j                  t        t        di |t        | |d      j                         j                               |
|              |	S )Nr   Fr2   r   r   r   r  r  r   r   fullTno_log_input	log_inputfull_no_log_input)r"  r$  c                    |r#| j                         |j                  |       z
  }n%| |j                  | |z   j                               z
  }|r`||j                  |j                               |z
  ddt        j                  z  |z  j                         z  z   j                  |dk  d      z  }|dk(  r|S |dk(  r!|j                         | j                         z  S |j                         S )N      ?       @rH   r   r   r  )expr  r   mathpimasked_fillr   numel)r   r   r$  r"  r   epsr   s          rD   poissonnllloss_reference_fnzJmodule_inputs_torch_nn_PoissonNLLLoss.<locals>.poissonnllloss_reference_fn  s    UUWquuQx'FC}}//FquuQUUW~)C2<!3C2H2H2J,JJWWXY]^X^`abbFM& ::<!'')++::<rF   c                      ||fi |S r;   rU   )r   r   r   r   r   r/  s        rD   r   z;module_inputs_torch_nn_PoissonNLLLoss.<locals>.reference_fn  s    .q!J7IJJrF   r   r   r   r   MbP?r   )TFr  g:0yE>rU   )
r   r
   r   r  addrI   r   rz   floor_abs_)rK   r   rR   r   rS   rW   r   r  r   r   r   r   r   r$  r   r/  s                  @rD   %module_inputs_torch_nn_PoissonNLLLossr6    s<   V5P]^J+fEQVWK 
R	;./	K01	K01	&$ 	+u-.	t%@A%E   M$) 
  8J 	K '**;=	,5
<(:l;S;W;W;Y;];]^c;d-*M:L*M&3E4?4M4T4T4V4[4[4]'_!%1	3	

 rF   c                    t        t        |||      }t        t        ||d      }di fdddifdddifd	dd
ifg}dd}	g }
|D ]  \  }}|
j                  t        t	        di |t	         |d       |d            |t        |	fi |             |
j                  t        t	        di |t	         |d       |d            | dt        |	fi |              |
S )Nr   Fr2   r   r   r   r  r  r   r   c                     |dk(  r||z
  j                  d      S |dk(  r3||z
  j                  d      j                         |j                         z  S ||z
  j                  d      j                         S )Nr   r   r  )powr   r-  )r   r   r   r   r   s        rD   mse_loss_reference_fnz=module_inputs_torch_nn_MSELoss.<locals>.mse_loss_reference_fn  se    E;;q>!& E;;q>%%'!'')33E;;q>%%''rF   r1  r   rU   _scalar)r  r   r
   rI   r   rz   )rK   r   rR   r   rS   rW   r   r  r   r:  r   r   r   s                rD   module_inputs_torch_nn_MSELossr=    s#   V5P]^J+fEQVWK 
R	;./	K01	K01	%E( M$) 
  -*M:L*M&3J|4L4?4M'O!%,-B%YFX%Y	[	
 	-*M:L*M&3JrN4?O'E $vW-%,-B%YFX%Y	[	

  rF   c                    fd} |dd      rdnd} |dd      } |dd	      }|`t        |t              st        d
t        |             j	                         D ]&  \  }}	||v s|	||   }
|	j                  |
      |<   ( |D cg c]  }|j                  |       }}t               5   | |i j                  |      }ddd       |r!t        |       }|dk(  rj                  d      S S c c}w # 1 sw Y   3xY w)a  Reference function for modules supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.

    Currently it only supports modules which return a single Tensor as output.
    You can bind the following kwargs.
    Kwargs:
        batch_first[bool] : If True, all the Tensors in `args` while be unsqueezed at dim `0` .
                        and output will be squeezed at dim `0` else dim `1` for both.
        kwargs_to_batchify[dict] : Dictionary specifying the name of the argument and dimension to unsqueeze.
                               Useful if there are few arguments whose batch dimension are different
                               from the ones selected by `batch_first`.
        is_criterion[bool] : Specify if the module is a criterion and handle the reduction for output accordingly.
    c                 V    j                  | |      }| v rj                  |        |S r;   )r   r  )keydefaultvrW   s      rD   get_and_popz.no_batch_dim_reference_fn.<locals>.get_and_pop/  s*    JJsG$&=JJsOrF   batch_firstTr   rH   kwargs_to_batchifyNis_criterionFz.Expected kwargs_to_batchify to be a dict, got r   )	r   dictAssertionErrortypeitems	unsqueezer&   squeezer%   )r   r   rV   rW   rC  	batch_dimrE  rF  krB  bdimr   single_batch_input_argsoutputr   s      `           rD   r  r    s.   $ !51I$%94@~u5L%,d3 #QRVWiRjQk!lmmLLN 	.DAq&&1=)!,KK-q		.
 HLLeuy9LL		 J+6v6>>yIJ !!$	>>!$$M MJ Js   C50C::Dc                 ~   |j                  dd      rdnd}d|v r|j                  d       d|v r|d   |d   j                  d      |d<   |D cg c]  }|j                  |       }}t               5   | |i |}|d   j	                  |      |d   j	                  d      fcddd       S c c}w # 1 sw Y   yxY w)a  Reference function for MultiheadAttention supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.
    rD  Tr   rH   key_padding_maskN)r   r  rK  r&   rL  )r   r   rV   rW   rM  r   rP  rQ  s           rD   no_batch_dim_reference_mharT  L  s     ZZt4!I

=!V#/A(B(N%+,>%?%I%I!%L!"GKLeuy9LL		 D+6v6q	!!),fQi.?.?.BCD D MD Ds   B.40B33B<c                 r   t        |      dk(  r|\  }d}n$t        |      dk(  r|\  }}|j                  d      }|d   rdnd}|j                  d       j                  |      }|f}t               5   | |i |}|d   j	                  |      |d   j	                  d      fcddd       S # 1 sw Y   yxY w)a  Reference function for RNN and GRU supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.
    rH   Nr   rD  r   r`   rK  r  r&   rL  	r   r   rV   rW   inphrM  rP  rQ  s	            rD   no_batch_dim_reference_rnn_grurZ  _  s     4yA~	TaQKKNM*I
JJ}
--	
"C"Ah		 D+6v6q	!!),fQi.?.?.BCD D Ds   30B--B6c                    t        |      dk(  r|\  }d}n;t        |      dk(  r-|\  }}|d   j                  d      |d   j                  d      f}|d   rdnd}|j                  d       j                  |      }|f}t               5   | |i |}|d   j	                  |      |d   d   j	                  d      |d   d   j	                  d      ffcddd       S # 1 sw Y   yxY w)a  Reference function for LSTM supporting no batch dimensions.

    Unbatched inputs are unsqueezed to form a
    single batch input before passing them to the module.
    The output is squeezed to compare with the
    output of unbatched input to the module.
    rH   Nr   r   rD  rV  rW  s	            rD   no_batch_dim_reference_lstmr\  w  s     4yA~	TaQqT^^A!q 12M*I
JJ}
--	
"C"Ah		 b+6v6q	!!),vay|/C/CA/Fq	RSH\H\]^H_.`ab b bs   
A
CC'c                    |\  }\  }}|j                  d      |j                  d      |j                  d      ff}t               5   | |i |}|d   j                  d      |d   j                  d      fcddd       S # 1 sw Y   yxY w)zReference function for LSTMCell supporting no batch dimensions.

    The module is passed the input and target in batched form with a single item.
    The output is squeezed to compare with the no-batch input.
    r   rH   N)rK  r&   rL  )	r   r   rV   rW   rX  rY  crP  rQ  s	            rD   no_batch_dim_reference_lstmcellr_    s     KC!Q"}}Q/!++a.!++a.1QR		 <+6v6q	!!!$fQi&7&7&:;< < <s   0B  B	c                     dD cg c]C  }t        t        |      t         | d       | d            t        t        d      d|       E c}S c c}w )	N)r   r  r   )r   r   r   TrF  no_batch_dim_r   r   r   r   )r   rz   r   r  )r   r   s     rD   $generate_regression_criterion_inputsre    s[     34  	+i@'
5(9:a>J !:N ,		
4 4 4s   AAc                 d   t        t        |||      }t        t        d      t         |d            dt              t        t        d      t         |d                  t        t        d	d	      t         |d            d
      t        t        ddd      t         |d            d      gS )Nr   r   )kernel_size)r      r   r   )r   r   rh  r   r   r   strider   r   r   rH   
stride_padr   r
   r   rz   r  rK   r   rR   r   rS   rW   r   s          rD    module_inputs_torch_nn_AvgPool1drp    s    V5P]^J 	m&B"/
60B"C'!:	< 	mA&6"/
90E"F	HmD$&?"/
90E"F!	# 	mAq!&<"/
90E"F%	'( (rF   c                 R   t        t        |||      }t        t        d      t         |d            dt              t        t        d      t         |d                  t        t        dd      t         |d            d	      t        t        ddd
      t         |d            d	      t        t        dd      t         |d            d	      t        t        ddd      t         |d            d	      t        t        ddd
d      t         |d            d	      gS )Nr   r   r   r   rh  rh  r   r   r   r   rh  rh  ri  rk  rl  rH   rH   rm  rH   divisor_overridedivisordivisor_stridedivisor_stride_padrn  ro  s          rD    module_inputs_torch_nn_AvgPool2dr{    s   V5P]^J 	mF&;"/
90E"F'!:	< 	mF&;"/
<0H"I	KmFF&C"/
<0H"I!	# 	mFFF&K"/
<0H"I%	' 	mFQ&O"/
<0H"I"	$ 	mFFUV&W"/
<0H"I)	+ 	mFFF]^&_"/
<0H"I-	/'0 0rF   c                    t        t        |||      }t        t        d      t         |d            dt              t        t        d      t         |d                  t        t        dd      t         |d	            d
      t        t        ddd      t         |d	            d      t        t        ddd      t         |d	            d      t        t        ddd      t         |d            d      t        t        ddd      t         |d            d      t        t        ddd      t         |d            d      t        t        dd      t         |d            d      t        t        ddd      t         |d	            d      t        t        dddd      t         |d	            d      t        t        dddd      t         |d	            d      t        t        dddd      t         |d            d      t        t        dddd      t         |d            d       t        t        dddd      t         |d            d!      gS )"Nr   r   r   r   r   r   r   r   r   r   r   r   r   r   r   ri  r   r   r   r   r   r   rk  rl  rH   rH   rH   rm  r   )rH   r   rH   stride_pad_gpu_fixedkw_output)r   r   r   rH   )rH   rH   r   )r   r   r   r   r   stride_pad_gpu_general_outputr   r   stride1_pad0_gpu_inputstride_pad_gpu_input_nooverlaprv  rx  ry  rz  %divisor_stride_pad_gpu_fixedkw_output%divisor_stride_pad_gpu_general_outputdivisor_stride1_pad0_gpu_input&divisor_stride_pad_gpu_input_nooverlaprn  ro  s          rD    module_inputs_torch_nn_AvgPool3dr    sQ   V5P]^J 	mI&>"/
<0H"I'!:	< 	mI&>"/
?0K"L	NmAy&A"/
?0K"L!	# 	mAq)&D"/
?0K"L%	' 	mAq)&D"/
?0K"L8	: 	mIq)&L"/
?0K"L8	: 	mAq!&<"/
?0K"L1	3 	mAq)&D"/
?0K"L9	; 	mIPQ&R"/
?0K"L"	$ 	mAyST&U"/
?0K"L)	+ 	mAq)VW&X"/
?0K"L-	/ 	mAq)VW&X"/
?0K"L@	B 	mIq)^_&`"/
?0K"L@	B 	mAq!a&P"/
?0K"L9	; 	mAq)VW&X"/
?0K"LA	CW-D -DrF   c           
         t        t        |||      }t        t        d      t         |d            d      t        t        d      t         |d            t        d      t        t        d	      t         |d            d
      gS )Nr   r   rH   r   r   singlerl  r   r   r   rd  rH   
one_outputrn  ro  s          rD   (module_inputs_torch_nn_AdaptiveAvgPool1dr    s    V5P]^J 	mA&7"/
90E"F!	# 	mA&7"/
60B"C!:'	) 	mA&7"/
90E"F%	'
( 
(rF   c                    t        t        |||      }t        t        d      t         |d            d      t        t        d      t         |d            t        d      t        t        d	      t         |d            d
      t        t        d      t         |d            d      t        t        d      t         |d            d      gS )Nr   r   rH   r   r   rh  r  rl  r   r   rh  r   rd  rH   single_1x1outputr   r   tupler   N
tuple_nonern  ro  s          rD   (module_inputs_torch_nn_AdaptiveAvgPool2dr    s    V5P]^J 	mA&7"/
<0H"I!	# 	mA&7"/
90E"F!:'	) 	mA&7"/
<0H"I+	- 	mF&;"/
<0H"I 	" 	mI&>"/
<0H"I%	'( (rF   c                    t        t        |||      }t        t        d      t         |d            d      t        t        d      t         |d            t        d      t        t        d	      t         |d
            d      t        t        d      t         |d
            d      t        t        d      t         |d            d      gS )Nr   r   )r   r   r   r      r  rl  )r   r   r   r  r   rd  r   r   r   )r   r   r   r   r  r  )Nr   r   r  )r   r   r   )rH   rH   r   r   rh  last_dimrn  ro  s          rD   (module_inputs_torch_nn_AdaptiveAvgPool3dr  -  s    V5P]^J 	mA&7"/
?0K"L!	# 	mA&7"/
<0H"I!:'	) 	mI&>"/
?0K"L 	" 	mL&A"/
?0K"L%	' 	mI&>"/
?0K"L#	%& &rF   c           	          t        t        |||      }t        t        d      t         |d            d      t        t        d      t         |d            t        d      gS )	Nr   r   r  r  rl  r  r   rd  rn  ro  s          rD   (module_inputs_torch_nn_AdaptiveMaxPool1dr  C  sb    V5P]^J 	mA&7"/
90E"F!	# 	mA&7"/
60B"C!:'	)	* *rF   c                 ^   t        t        |||      }t        t        d      t         |d            d      t        t        d      t         |d            t        d      t        t        d	      t         |d            d
      t        t        d      t         |d            d      gS )Nr   r   r  r  rl  r  r   rd  r  r  r  r  rn  ro  s          rD   (module_inputs_torch_nn_AdaptiveMaxPool2dr  P  s    V5P]^J 	mA&7"/
<0H"I!	# 	mA&7"/
90E"F!:'	) 	mF&;"/
<0H"I 	" 	mI&>"/
<0H"I%	'( (rF   c                    t        t        |||      }t        t        d      t         |d            d      t        t        d      t         |d            t        d      t        t        d	      t         |d            d
      t        t        d      t         |d            d      t        t        d      t         |d            d      t        t        d	      t         |d            d      gS )Nr   r   )r   r   r   rh  r  r  rl  )r   r   rh  r  r   rd  r  r  )r   Nr   r  )r   r      	   r   single_nonatomic)r   r   rh  r   r   tuple_nonatomicrn  ro  s          rD   (module_inputs_torch_nn_AdaptiveMaxPool3dr  c  s    V5P]^J 	mA&7"/
?0K"L!	# 	mA&7"/
<0H"I!:'	) 	mI&>"/
?0K"L 	" 	mL&A"/
?0K"L%	' 	mA&6"/
;K0L"M+	- 	mI&>"/
;K0L"M*	,#- -rF   c                    t        t        |||      }t        t        d      t         |d            d      t        t        d      t         |d            d      t        t        dd	d       t         |d            d
      t        t        dd	dd      t         |d            d      t        t        dd	ddd      t         |d            d      t        t        dd	dd      t         |d            d      t        t        dd	dd      t         |d            d      t        t        dd      t         |d            d      gS )Nr   r   r   affinerl  r   )r   r   r   3d_inputr2  affine_simple_average333333?F
not_affineTnot_tracking_stats3d_input_not_affine)r   r   r  
zero_batchr   affine_not_biasr   ro  s          rD   "module_inputs_torch_nn_BatchNorm1dr  |  sE   V5P]^J 	mB&8"/
70C"D!	# 	mA&7"/
90E"F#	% 	mBd&C"/
70C"D0	2 	mBc5&I"/
70C"D%	' 	mBc4&O"/
70C"D-	/ 	mAtS%&H"/
90E"F.	0 	mAtS%&H"/
90E"F%	' 	mBU&C"/
70C"D*	,-. .rF   c                 P   t        t        |||      }t        t        d      t         |d                  t        t        ddd       t         |d            d      t        t        ddd      t         |d            d	      t        t        dddd
      t         |d            d      t        t        ddddd
      t         |d            d      t        t        dddd
      t         |d            d      t        t        dd
      t         |d            d      gS )Nr   r   rt  ri  r2  2d_simple_averagerl  g?momentumFr  Tr  r   r  )r   r   r   r   r  r   r  r   ro  s          rD   "module_inputs_torch_nn_BatchNorm2dr    s   V5P]^J 	mA&7"/
<0H"I	KmAtT&B"/
<0H"I,	. 	mAtS&A"/
<0H"I#	% 	mAtS%&H"/
<0H"I%	' 	mAtS$&N"/
<0H"I-	/ 	mAtS%&H"/
<0H"I%	' 	mAE&B"/
<0H"I*	,%. .rF   c                 P   t        t        |||      }t        t        d      t         |d                  t        t        ddd       t         |d            d      t        t        ddd      t         |d            d	      t        t        dddd
      t         |d            d      t        t        ddddd
      t         |d            d      t        t        dddd
      t         |d            d      t        t        dd
      t         |d            d      gS )Nr   r   r  ri  r2  3d_simple_averagerl  ffffff?r  Fr  Tr  r   r  )r   r   r   r   r   r  r   r  r   ro  s          rD   "module_inputs_torch_nn_BatchNorm3dr    s   V5P]^J 	mA&7"/
?0K"L	NmAtT&B"/
?0K"L,	. 	mAtS&A"/
?0K"L#	% 	mAtS%&H"/
?0K"L%	' 	mAtS$&N"/
?0K"L-	/ 	mAtS%&H"/
?0K"L%	' 	mAE&B"/
?0K"L*	,%. .rF   c                    t        t        |||      }| j                  t        j                  j
                  k(  rd}n,| j                  t        j                  j                  k(  rd}nd}t        t        t        dd      t         ||                  t        j                  t        d	
      t        t        t        dd      t         ||                  t        j                  t        d	
      gS )Nr   )r   r   )r   r   r   r   )r   r   r   r   r   r   g      )r.  ri  zeps must be positiver   r   r   r   )r   r
   rw   ra   r   BatchNorm1dBatchNorm2dr   r   rz   r   r   
ValueError)rK   r   rR   r   rS   rW   r   input_shapes           rD   .module_error_inputs_torch_nn_BatchNorm1d_2d_3dr    s    V5P]^J!5!55			588#7#7	7#& 	"/"=+J{,CD %22!.	
 	"/"<+J{,CD %22!.	
 rF   c                    |d   }|j                  dd      }|j                  dd      }t        t        |||      }	|ri gni ddig}
d\  }}}|ft        d	 t	        |      D              z   }d
|z   }t        j                  ddg|
      D cg c]N  \  }}t        |rt        ||fi |nt        |||fi |t         |	|r|n|            |rdnd|rd nt              P c}}S c c}}w )NNr   F
transposedr   paddingsamer  c              3   &   K   | ]	  }|d z     ywr  rU   ).0r   s     rD   	<genexpr>z0module_inputs_torch_nn_ConvNd.<locals>.<genexpr>  s     *CQ1q5*Cs   rj  Tr2   r   r   )
r   r   r
   r  range	itertoolsr   r   rz   r  )rK   r   rR   r   rS   rW   r  r   r  r   conv_kwargs_listrg  C_inC_outinput_no_batch_shapeinput_batch_shape
with_batchconv_kwargss                     rD   module_inputs_torch_nn_ConvNdr    s   sA::fe$DL%0JV5P]^J)tY4G/H&Ku 7U*C%(*C%CC33 (1'8'8$HX'Y $J 	[_}UK'W;'W'4T5+']Q\']"/
-7)=Q1S #T *"*4$:S	V  s   AC$c                 J   t        t        |||      }t        t        ||d      }di fdddifdddifd	dd
ifdddifg}g }	|D ]\  \  }
}|fd}|	j                  t        t	        di |t	         |d       |d       |d      j                               |
|             ^ |	S )Nr   Fr2   r   r   r   r  r  r   r   marginr  c                     t        |||fi |S r;   )r   r   r   i1i2r   r   s         rD   r   z@module_inputs_torch_nn_CosineEmbeddingLoss.<locals>.reference_fn  s    0RQ>PQQrF   r	  r   )r	  r   rU   r   r
   rI   r   rz   signrK   r   rR   r   rS   rW   r   r  r   r   r   r   r   s                rD   *module_inputs_torch_nn_CosineEmbeddingLossr    s    V5P]^J+fEQVWK 
R	;./	K01	K01	Hc?#%E M$) 

  =O 	R 	-*M:L*M&3Jx4H*U]J^4?4F4K4K4M'O!%1	3	
	

 rF   c                 d   t        t        |||      }t        t        d      t         |d            d       t        t        d      t         |d            d	      t        t               t         |d
            dt              t        t        d      t         |d            d	      gS )Nr   r(  alphar   r   r   c                 ^    t        j                  |dk\  |d|j                         dz
  z        S )Nr   r   rH   ra   wherer)  r   s      rD   r   z,module_inputs_torch_nn_ELU.<locals>.<lambda>$  s'    Q!VQQUUWWX[HY1Z rF   r   rU   scalarrl  r   r   r   r   r   r   r   4d_inputrn  ro  s          rD   module_inputs_torch_nn_ELUr    s    V5P]^J 	m"&="/
90E"F!Z	\ 	m"&="/
2"?!	# 	mo"/
40@"A'!:	< 	m"&="/
<0H"I#	%& &rF   c           
          t        t        |||      }t        t        d      t         |d            d       t        t        d      t         |d            d d	
      t        t        d      t         |d            dt              gS )Nr   r(  r  r  c                 d    t        j                  |dk\  |dd|z  j                         dz
  z        S Nr   r(  r'  rH   r  r   s      rD   r   z-module_inputs_torch_nn_CELU.<locals>.<lambda>7  -    Q!VQrTUvlln_`N`Ha1b rF   r   rU   c                 d    t        j                  |dk\  |dd|z  j                         dz
  z        S r  r  r   s      rD   r   z-module_inputs_torch_nn_CELU.<locals>.<lambda>:  r  rF   r  rd  r  r   r   rn  ro  s          rD   module_inputs_torch_nn_CELUr  1  s    V5P]^J 	m"&="/
90E"F!b	d 	m"&="/
2"?!b!	# 	m"&="/
40@"A'!:	<= =rF   c           
         t        t        |||      }t        t               t         |d                  t        t        d      t         |d            d      t        t               t         |d            d	t        
      gS )Nr   )r   rh  ri  rH   r   rh  r  r   rl  ra  r   r   rn  ro  s          rD   module_inputs_torch_nn_GLUr  B  sy    V5P]^J 	mo"/
60B"C	EmA&6"/
90E"F	  	mo"/
40@"A'!:	<	= 	=rF   c           
         t        t        |||      }t        t        d      t         |d            d d      t        t        d      t         |d            d 	      t        t               t         |d
            dt              gS )Nr   r   rU   c                 j    |dz  dt        j                  |t        j                  d      z        z   z  S Nr'        ?r(  ra   erfr*  sqrtr   r   xrO   s       rD   r   z-module_inputs_torch_nn_GELU.<locals>.<lambda>W  0    QWeiiPQTXT]T]^aTbPbFc@c5d rF   r  rd  r  c                 j    |dz  dt        j                  |t        j                  d      z        z   z  S r  r  r  s       rD   r   z-module_inputs_torch_nn_GELU.<locals>.<lambda>[  r  rF   r   r  r   r   rn  ro  s          rD   module_inputs_torch_nn_GELUr   Q  s    V5P]^J 	mF&;"/
2"?!d!	# 	mF&;"/
90E"F!d	f 	mo"/
40@"A'!:	<= =rF   c                 V   t        t        |||      }t        t               t         |d            d      t        t               t         |d            t        d      t        t               t         |d            d	      t        t               t         |d
            d      gS Nr   rU   r  rl  r   r   rd  r1  channels_last_mem_format)r   r   r   r   r   channels_last_3d_mem_formatrn  ro  s          rD   module_inputs_torch_nn_ReLUr  b      V5P]^J 	mo"/
2"?!	# 	mo"/
1">!:'	) 	mo"/
<0H"I3	5 	mo"/
?0K"L6	89 9rF   c                 V   t        t        |||      }t        t               t         |d            d      t        t               t         |d            t        d      t        t               t         |d            d	      t        t               t         |d
            d      gS r  rn  ro  s          rD   module_inputs_torch_nn_ReLU6r  u  r  rF   c                    t        t        |||      }t        t               t         |d                  t        t               t         |d            t        d      t        t        d      t         |d            d	      t        t        d
      t         |d            d	      t        t        d      t         |d            d	      gS )Nr   r  ri  r   r   rd  r'  with_negvalrl  r   r   with_zero_negvalrU   with_negval_scalarrn  ro  s          rD    module_inputs_torch_nn_LeakyReLUr    s    V5P]^J 	mo"/
90E"F	Hmo"/
1">!:'	) 	mC&8"/
90E"F&	( 	mC&8"/
80D"E+	- 	mC&8"/
2"?-	/0 0rF   c                    t        t        |||      }t        t               t         |d            d      t        t               t         |d            t        d      t        t               t         |d            d	 d
      t        t        d      t         |d            d d      t        t               t         |d            d d      t        t        d      t         |d            d d      t        t               t         |d            d d      t        t        d      t         |d            d d      gS )Nr   rU   r  rl  r   r   rd  r   r   r   c                 r    t        j                  |d      t        j                  |d      |d   d   z  z   S Nr   min)maxra   clampr   s      rD   r   z.module_inputs_torch_nn_PReLU.<locals>.<lambda>  5    QA1FUV\]I^abcdaefgahIh1h rF   1dr   c                 r    t        j                  |d      t        j                  |d      |d   d   z  z   S r  r  r   s      rD   r   z.module_inputs_torch_nn_PReLU.<locals>.<lambda>  r  rF   1d_multiparamr1  c                 r    t        j                  |d      t        j                  |d      |d   d   z  z   S r  r  r   s      rD   r   z.module_inputs_torch_nn_PReLU.<locals>.<lambda>  r  rF   2dc                 r    t        j                  |d      t        j                  |d      |d   d   z  z   S r  r  r   s      rD   r   z.module_inputs_torch_nn_PReLU.<locals>.<lambda>  r  rF   2d_multiparam)r   r   r   r   rh  c                 r    t        j                  |d      t        j                  |d      |d   d   z  z   S r  r  r   s      rD   r   z.module_inputs_torch_nn_PReLU.<locals>.<lambda>  r  rF   3dc                 r    t        j                  |d      t        j                  |d      |d   d   z  z   S r  r  r   s      rD   r   z.module_inputs_torch_nn_PReLU.<locals>.<lambda>  r  rF   3d_multiparamrn  ro  s          rD   module_inputs_torch_nn_PReLUr#    s)   V5P]^J 	mo"/
2"?!	# 	mo"/
1">!:'	) 	mo"/
90E"F!h	 	mA&6"/
90E"F!h(	* 	mo"/
<0H"I!h	 	mA&6"/
<0H"I!h(	* 	mo"/
?0K"L!h	 	mA&6"/
?0K"L!h(	*9+ +rF   c           
         t        t        |||      }t        t               t         |d                  t        t               t         |d            t        d      t        t               t         |d            d	      gS )
Nr   r  ri  r   r   rd  rU   r  rl  rn  ro  s          rD   module_inputs_torch_nn_SELUr%    st    V5P]^J 	mo"/
90E"F	Hmo"/
1">!:'	) 	mo"/
2"?!	#	$ 	$rF   c           
         t        t        |||      }t        t               t         |d            d d      t        t               t         |d            t        d      t        t               t         |d            d	 
      gS )Nr   rU   c                 2    |t        j                  |      z  S r;   ra   sigmoidr  s       rD   r   z-module_inputs_torch_nn_SiLU.<locals>.<lambda>      Qq9I5I rF   r  rd  r   r   r  c                 2    |t        j                  |      z  S r;   r(  r  s       rD   r   z-module_inputs_torch_nn_SiLU.<locals>.<lambda>  r*  rF   r   rn  ro  s          rD   module_inputs_torch_nn_SiLUr,    s~    V5P]^J 	mo"/
2"?!I!	# 	mo"/
1">!:'	) 	mo"/
90E"F!I	KL LrF   c           
         t        t        |||      }t        t        d      t         |d            d       t        t        d      t         |d            d d	
      t        t        d      t         |d            t        d
      gS )Nr   rH   r      c                     t        j                  |      j                  t        j                  |      j                  dd      j	                  dd            S NrH   Tr   r/  )ra   r)  divr   r   r   s      rD   r   z0module_inputs_torch_nn_Softmax.<locals>.<lambda>  s?    11A1A%))A,BRBRSTVZB[BbBbcegiBj1k rF   r   r   rU   c                     t        j                  |      j                  t        j                  |      j                  dd            S )Nr   Tra   r)  r2  r   r   s      rD   r   z0module_inputs_torch_nn_Softmax.<locals>.<lambda>  s0    11A1A%))A,BRBRSTVZB[1\ rF   r  rd  r   r   r   r   rn  ro  s          rD   module_inputs_torch_nn_Softmaxr6    s    V5P]^J 	mA&6"/
80D"E!k	m 	mA&6"/
2"?!\!	# 	mB&7"/
60B"C!:'	)* *rF   c           	          t        t        |||      }t        t               t         |d            d       t        t               t         |d            t        d      gS )Nr   rH   r   r   r/  c                     t        j                  |      j                  t        j                  |      j                  dd            S NrH   Fr4  r   s      rD   r   z2module_inputs_torch_nn_Softmax2d.<locals>.<lambda>  s0    11A1A%))A,BRBRSTV[B\1] rF   r   r  r   rd  rn  ro  s          rD    module_inputs_torch_nn_Softmax2dr;    s]    V5P]^J 	mo"/
>0J"K!]	_ 	mo"/
90E"F!:'	)	* *rF   c                 h   t        t        |||      }t        t        d      t         |d            d       t        t        d      t         |d            d d	      t        t        d
      t         |d            d d	      t        t        d      t         |d            t        d	      gS )Nr   rH   r.  c                     t        j                  |      j                  t        j                  |      j                  dd      j	                  dd            j                         S r1  )ra   r)  div_r   r   log_r   s      rD   r   z3module_inputs_torch_nn_LogSoftmax.<locals>.<lambda>  sH    11B1B599Q<CSCSTUW[C\CcCcdfhjCk1l1q1q1s rF   r   r8  c                     t        j                  |      j                  t        j                  |      j                  dd            j	                         S r:  ra   r)  r>  r   r?  r   s      rD   r   z3module_inputs_torch_nn_LogSoftmax.<locals>.<lambda>	  9    11B1B599Q<CSCSTUW\C]1^1c1c1e rF   
multiparamrd  r   rU   c                     t        j                  |      j                  t        j                  |      j                  dd            j	                         S )Nr   FrA  r   s      rD   r   z3module_inputs_torch_nn_LogSoftmax.<locals>.<lambda>  rB  rF   multiparam_scalarr   r5  r   rn  ro  s          rD   !module_inputs_torch_nn_LogSoftmaxrF     s    V5P]^J 	mA&6"/
80D"E!s	u 	mA&6"/
>0J"K!e%	' 	mA&6"/
2"?!e,	. 	mB&7"/
60B"C!:'	)* *rF   c                 \   t        t        |||      }t        t        d      t         |d                  t        t        d      t         |d            d      t        t        d      t         |d	            d
      t        t        d      t         |d            t        d      gS )Nr   rH   r.  ri  )r   r   r   r   multidimrl  r   rU   r  r   )r   r   r   r   rd  rn  ro  s          rD   module_inputs_torch_nn_SoftminrI    s    V5P]^J 	mA&6"/
80D"E	GmA&6"/
=0I"J#	% 	mA&6"/
2"?!	# 	mB&7"/
:0F"G!:'	)* *rF   c                    t        t        |||      }t        t               t         |d            d       t        t        d      t         |d            d d      t        t        dd	      t         |d            d
 d      t        t        dd	      t         |d            d d      t        t               t         |d            t        d      gS )Nr   r.  c                 R    t        j                  t        j                  |            S r;   ra   log1pr)  r   s      rD   r   z1module_inputs_torch_nn_Softplus.<locals>.<lambda>-  s    UYYq\1J rF   r   r   c                 ^    dt        j                  t        j                  d|z              z  S )Nr'  r   rL  r   s      rD   r   z1module_inputs_torch_nn_Softplus.<locals>.<lambda>0  s$    5;;uyyQRUVQVGW;X1X rF   betard  c                     |dz  dkD  j                  |      |z  |dz  dk  j                  |      dz  dz  t        j                  t        j                  d|z              z  z   S Nr   rP  r  r(  type_asra   rM  r)  r   s      rD   r   z1module_inputs_torch_nn_Softplus.<locals>.<lambda>5  b    !a%4(@(@(Ca(GEd?33A6;b@5;;uyyYZ]^Y^O_C``)a rF   beta_thresholdrU   c                     |dz  dkD  j                  |      |z  |dz  dk  j                  |      dz  dz  t        j                  t        j                  d|z              z  z   S rR  rS  r   s      rD   r   z1module_inputs_torch_nn_Softplus.<locals>.<lambda>;  rU  rF   beta_threshold_scalarr   r   rn  ro  s          rD   module_inputs_torch_nn_SoftplusrY  '  s    V5P]^J 	mo"/
80D"E!J	L 	mA&6"/
80D"E!X	! 	mAt&<"/
80D"Ea)	+ 	mAt&<"/
2"?a0	2 	mo"/
1">!:'	))* *rF   c                 X   t        t        |||      }t        t               t         |d                  t        t        d      t         |d            d      t        t        d      t         |d            d      t        t               t         |d	            t        d
      gS )Nr   r  ri  rH   lambdarl  rU   lambda_scalarr   r   rd  rn  ro  s          rD   !module_inputs_torch_nn_Softshrinkr]  D  s    V5P]^J 	mo"/
90E"F	HmA&7"/
90E"F!	# 	mA&7"/
2"?(	* 	mo"/
1">!:'	)* *rF   c           
         t        t        |||      }t        t               t         |d            d       t        t               t         |d            d d      t        t               t         |d	            t        d
      gS )Nr   r  c                 P    |j                  dt        j                  |      z         S NrH   r2  ra   r  r   s      rD   r   z1module_inputs_torch_nn_Softsign.<locals>.<lambda>\      q599Q<7G1H rF   r   rU   c                 P    |j                  dt        j                  |      z         S r`  ra  r   s      rD   r   z1module_inputs_torch_nn_Softsign.<locals>.<lambda>_  rb  rF   r  rd  r   r   rn  ro  s          rD   module_inputs_torch_nn_Softsignrd  V  s|    V5P]^J 	mo"/
90E"F!H	J 	mo"/
2"?!H!	# 	mo"/
1">!:'	)* *rF   c           
         t        t        |||      }t        t               t         |d                  t        t               t         |d            d      t        t               t         |d            t        d	      gS 
Nr   r1  ri  rU   r  rl  r   r   rd  rn  ro  s          rD   module_inputs_torch_nn_Tanhrg  g  t    V5P]^J 	mo"/
<0H"I	Kmo"/
2"?!	# 	mo"/
1">!:'	)	* 	*rF   c           
         t        t        |||      }t        t               t         |d                  t        t               t         |d            d      t        t               t         |d            t        d	      gS rf  rn  ro  s          rD   !module_inputs_torch_nn_Tanhshrinkrj  w  rh  rF   c                 f   t        t        |||      }t        t        dd      t         |d            d      t        t        dd      t         |d            d      t        t        dd      t         |d	            d
      t        t        dd      t         |d            t        d      gS )Nr   r(  r  r1  threshold_valuerl  g      $@large_valuerU   threshold_value_scalarr   r   rd  rn  ro  s          rD    module_inputs_torch_nn_Thresholdro    s    V5P]^J 	mB&;"/
<0H"I*	, 	mB&<"/
<0H"I&	( 	mB&;"/
2"?1	3 	mB&;"/
1">!:'	)* *rF   c           
         t        t        |||      }t        t               t         |d            d       t        t               t         |d            d d      t        t               t         |d	            t        d
      gS )Nr   r  c                 X    |t        j                  t        j                  |            z  S r;   ra   tanhFsoftplusr   s      rD   r   z-module_inputs_torch_nn_Mish.<locals>.<lambda>      UZZ

15N1N rF   r   rU   c                 X    |t        j                  t        j                  |            z  S r;   rr  r   s      rD   r   z-module_inputs_torch_nn_Mish.<locals>.<lambda>  rv  rF   r  rd  r   r   rn  ro  s          rD   module_inputs_torch_nn_Mishrx    s|    V5P]^J 	mo"/
90E"F!N	P 	mo"/
2"?!N!	# 	mo"/
1">!:'	)* *rF   c           
          t        t        |||      }t        t               t         |d       |d            d       t        t               t         |d       |d            d d      gt	        |      z   S )	Nr   r  c           	      f    d|j                         z  t        d t        ||d      D              z  S )Nr  c              3   d   K   | ](  \  }}||z
  j                         j                          * y wr;   )r  r   )r  abs      rD   r  zBmodule_inputs_torch_nn_L1Loss.<locals>.<lambda>.<locals>.<genexpr>  s6      ImMQQPQ KLa%IZIZI\ Ims   .0T)strict)r-  r   zipr   r   r   r   s       rD   r   z/module_inputs_torch_nn_L1Loss.<locals>.<lambda>  s<    BNS ImUXYZ\]fjUkIm Fm 5m rF   r   rU   c                 l    d|j                         z  ||z
  j                         j                         z  S )Nr  )r-  r  r   r  s       rD   r   z/module_inputs_torch_nn_L1Loss.<locals>.<lambda>  s(    BNa!e[[]EVEVEX4X rF   r  rd  )r   r
   r   rz   re  ro  s          rD   module_inputs_torch_nn_L1Lossr    s    V5P]^J 	mo"/
90E0:90E#G"m	n
 	mo"/
2
2"O!X!	#	$ 'K:&V	W 	WrF   c                 p   t        t        |||      }di fdddifdddifddd	ifg}g }|D ]  \  }	}
|
fd
}|j                  t        t	        di |
t	         |d       |d            |	|             |j                  t        t	        di |
t	         |d       |d            d|	 |              |S )Nr   r2   r   r   r   r  r  r   r   c                     t        ||fi |S r;   )r#   r   s        rD   r   z9module_inputs_torch_nn_SmoothL1Loss.<locals>.reference_fn  s    )!QE2DEErF   r   r   r   rU   r   r<  rK   r   rR   r   rS   rW   r   r   r   r   r   r   s               rD   #module_inputs_torch_nn_SmoothL1Lossr    s    V5P]^J 
R	;./	K01	K01	%E M$) 
  8J 	F 	-*M:L*M&3Jw4G4>w4G'I!%1	3	
 	-*M:L*M&3JrN4>rN'D&tf-%1	3	

& rF   c                    t        t        |||      }t        t        ||d      }t        t        ||d      }di fdddifdddifd	dd
ifdd |d      ifg}	dd}
g }|	D ]m  \  }}|j                  t        t	        di |t	         |ddd       |d      j                  d      j                  |            |t        |
fi |             o  |d      }|j                  t        t	        |      t	         |ddd       |d      j                  d      j                  |            dt        |
|                   |S )Nr   Fr2   r   r   r   r  r  r   r   r  r  r   c                     ||j                         z  d|z
  d|z
  j                         z  z    }|||z  }|dk(  r|S |dk(  r!|j                         |j                         z  S |j                         S )NrH   r   r  )r   r   r-  r   r   r   r   r   r  r   s          rD   bce_loss_reference_fnz=module_inputs_torch_nn_BCELoss.<locals>.bce_loss_reference_fn  ss    quuw;!a%AE;;=!889f_FM& ::<!'')++::<rF   r  {Gz?Gz?lowhighr   r   rU   r  scalar_weightr  Nr   r
   rI   r   rz   gtto)rK   r   rR   r   rS   rW   r   r  r  r   r  r   r   r   r  s                  rD   module_inputs_torch_nn_BCELossr    sw   V5P]^J+fEQVWK+fEQVWK 
R	;./	K01	K01	X{5123%E  M$) 
  -*M:L*M&3JxTX`4a4?4I4L4LQ4O4R4RSX4Y'[!%,-B%YFX%Y	[	

  OMm=&I"/
24h0W0;B0B0B10E0H0H0O#Q(!()>}!U		W rF   c                    t        t        |||      }t        t        ||d      }t        t        ||d      }di fdddifdddifd	dd
ifdd |d      ifdd |d      ifg}	dd}
g }|	D ]m  \  }}|j                  t        t	        di |t	         |ddd       |d      j                  d      j                  |            |t        |
fi |             o |S )Nr   Fr2   r   r   r   r  r  r   r   r  r  r  scalar_weightsrU   c                    | j                  d      }d|z
  j                  |      j                  |      j                  | j                         j                  | |z
  j                               j	                               }|||z  }|dk(  r|S |dk(  r!|j                         |j                         z  S |j                         S )Nr   r  rH   r   r  )r  mul_add_exp_r?  r   r-  )r   r   r   r   r   r  max_valr   s           rD   bce_withlogitsloss_reference_fnzQmodule_inputs_torch_nn_BCEWithLogitsLoss.<locals>.bce_withlogitsloss_reference_fn  s    2***#a%a%%g.33gXOO4E4J4JQBQXLK^K^K`4a4f4f4hif_FM& ::<!'')++::<rF   r  r  r  r  r   r   r  r  )rK   r   rR   r   rS   rW   r   r  r  r   r  r   r   r   s                 rD   (module_inputs_torch_nn_BCEWithLogitsLossr    s    V5P]^J+fEQVWK+fEQVWK 
R	;./	K01	K01	X{5123	Hk"o67%E  M$) 
  -*M:L*M&3JxTX`4a4?4I4L4LQ4O4R4RSX4Y'[!%,-L%cPb%c	e	

 rF   c                    t        t        |||      }t        t        |t        j                  d      }t        t        ||d      }g d}	di fdd |d      ifddd	ifd
d
difdd	ddfg}
g }t	        |	|
      D ]  \  }\  }}||fd}|j                  t        t        d)d|i|t         |d       |ddd            d| d| |             |j                  t        t        d)d|i|t         |d       |ddd            d| d| |             |j                  t        t        d)d|i|t         |d       |ddd            d| d| |             |j                  t        t        d)d|i|t         |d       |ddd            d | d| |             |j                  dd       D|j                  t        t        d)d|i|t         |d!       |d!      j                  d	"            d#| d| |             |j                  t        t        d)d|i|t         |d$       |d$      j                  d	"            d%| d| |             |j                  t        t        d)d|i|t         |d&       |d&      j                  d	"            d'| d| |             |j                  t        t        d)d|i|t         |d       |d      j                  d	"            d(| d| |             |j                  t        t        d)d|i|t         |d       |d)dd            d*| d| t        t        d+,                    |S )-Nr   F)r  r   r   r2   r  r  r  r  rH   label_smoothingg333333?ignore_index_label_smoothing)r  r  c                      t        ||fd|i|S )Nr   )r   )r   r   r   r   r   r   s         rD   r   z=module_inputs_torch_nn_CrossEntropyLoss.<locals>.reference_fnF  s    /1`	`M_``rF   r   )r   r   r   r   )r   r   r   r   r   r  4d_rO   r   r   r   r   )r   r   r  r   r   r   2d_)r   r   r   r   r   r   )r   r   r   r   r   r  )r   r   r   r   r   4d_prob_target_)r   r   r   3d_prob_target_)r   r   2d_prob_target_higher_dim_prob_target_rU   rc  Trb  )r   r
   ra   r  r   rI   r   rz   r   softmaxr  )rK   r   rR   r   rS   rW   r   r  r  
reductionsr   r   r   r   r   r   s                   rD   'module_inputs_torch_nn_CrossEntropyLossr  6  s(   V5P]^J+fEJJV[\K+fEQVWK3J	R	X{4012	.!,-	.56	'!PT)UV%E M18U1K D-	-D,/8M_ 	a 	-*b)*bOa*b&3J|4L4?	qWX4Y'["4&)5%1	3	
 	-*b)*bOa*b&3Jy4I4?ATU4V'X"4&)5%1	3	
 	-*b)*bOa*b&3Jv4F4?QR4S'U"4&)5%1	3	
 	-*b)*bOa*b&3J?Q4R4?UV]^4_'a*4&)=%1	3	
 !!.$7?  m.fi.fSe.f*7
<8P8B<8P8X8X]^8X8_+a#24&)!E)5	7   m.fi.fSe.f*7
98M8B98M8U8UZ[8U8\+^#24&)!E)5	7   m.fi.fSe.f*7
68J8B68J8R8RWX8R8Y+[#24&)!E)5	7   m.fi.fSe.f*7
CU8V8BCU8V8^8^cd8^8e+g#:4&)!M)5	7   m.fi.fSe.f*7
48H8CBATU8V+X#0a	{!C)01JY])^	`}DL rF   c                    t        t        |||      }t        t        ||d      }t        t        t	               t	         |d      t        j                  d|t
        j                                    t        j                  t        d      t        t        t	               t	         |d       |d                  t        j                  t        d	      t        t        t	               t	         |d      t        j                  g d
|t
        j                                    t        j                  t        d      t        t        t	         |d            t	         |d      t        j                  ddg|t
        j                                    t        j                  t        d      t        t        t	        d      t	         |d       |d      j                  d      j                                     t        j                  t        d      t        t        t	        d      t	         |d      t        j                  ddg|t
        j                                    t        j                  t        d      t        t        t	               t	         |d      t        j                  d|t
        j                                    t        j                  t        d      gS )Nr   Fr  r   rR   ri  z@Expected floating point type for target with class probabilitiesr  rj  z(expected target dtype to be Long or Byter   rH   r   z7Expected input batch_size .* to match target batch_sizer  r   rH   zGweight tensor should be defined either for all .* classes or no classes)r  r   z7ignore_index is not supported for floating point target      ?)r  z+label_smoothing must be between 0.0 and 1.0r  )r   r   zExpected target size)r   r
   r   r   rz   ra   zerosr  r   r   r[   tensorr  r  detach)rK   r   rR   r   rS   rW   r   r  s           rD   -module_error_inputs_torch_nn_CrossEntropyLossr    sZ   V5P]^J+fEQVWK 	"//+v&KKvUZZH %22#Z	
 	"//+v&t$ %22#B	
 	"//+v&LL6L %22!Q	
 	"/{47H"I+v&LL!QejjI %22#a	
 	"/Q"?+v&v&..1.5<<> %22#Q	
 	"/"D+v&LL!QejjI %22#E	
 	"//+y)KKvUZZH %22#.	
a\ \rF   c                    t        t        |||      }t        t        |d      }di fdddifddd	ifd
ddifdddifg}t        j                  t        j                  g}	g }
t        |	|      D ]  \  }\  }}|fd}|j                  dd      }|dk(  rdnd}|dk(  rdnd}|
j                  t        t        di |t         |d      j                  d       |d|||      dd      | d|             |
j                  t        t        di |t         |d      j                  d       |d|||      t        j                  d|      t        j                  d|            | d|             |
j                  t        t        di |t         |d      j                  d       |d|||      dd      | d|             |
j                  t        t        di |t         |d      j                  d       |d|||      t        j                  d|      t        j                  d|            | d|              |
S ) Nr   F)r   r   r2   r   r   r   r  r  r   r   blank   c                      t        ||||fi |S r;   )r   )r   r   r   r   iltlr   s          rD   r   z4module_inputs_torch_nn_CTCLoss.<locals>.reference_fn   s    $Q2rH5GHHrF   r   rH   r	  )2   r   r	  r   )r      )rR   r  r  )r  r  r  )r     r/  _lengths_intlistsr   r
  _lengths_tensors)K   _1d_target_lengths_intlists_1d_target_lengths_tensorsrU   )r   r
   ra   intr  r   r   rI   r   rz   r  r  )rK   r   rR   r   rS   rW   r   r  r   target_dtypesr   target_dtyper   r   r   r  r  r  s                     rD   module_inputs_torch_nn_CTCLossr    s   V5P]^J+fEJK 
R	;./	K01	K01	7B- %E YY

+MM4;M54Q -
00t/@R 	I #&&w2B;aAb[rb"/"E2D"E+J{,C,O,OPQ,R,7|Y\cg,h,8,H v./)+	
 	"/"E2D"E+J{,C,O,OPQ,R,7|Y\cg,h,1LLf,U,1LLf,UW v-.)+		
 	"/"E2D"E+J{,C,O,OPQ,R,7|adko,p,8,H v89)+	
 	"/"E2D"E+J{,C,O,OPQ,R,7|adko,p,1LLf,U,1LLf,UW v78)+		
I-
^ rF   c                 H   t        t        |||      }t        t        ddd      t         |d            d      t        t        dddd	      t         |d            d
      t        t        ddd      t         |d            d      t        t        ddd      t         |d            d      t        t        dddd      t         |d            d      t        t        dddd      t         |d            d      t        t        ddd      t         |d            d      t        t        dddd	      t         |d            d      t        t        dddd      t         |d            d      t        t        dddd      t         |d            d      g
S )Nr   r   rh  r2  )r   rh  r   	1d_affinerl  Fr   1d_affine_not_biasr  )r   r  1d_affine_GNrH   )   rh  1d_affine_large_batchr   r   r   r   1d_no_affine_INr   r   1d_no_affine_LN)r   rh  r   r   	2d_affiner  )r   r  r   r   2d_affine_not_bias)r   r   r   r   2d_no_affine_IN2d_no_affine_LNr   ro  s          rD    module_inputs_torch_nn_GroupNormr  1  s   V5P]^J 	+Aq$7'
9(=>	 	+Aq$UC'
9(=>%	' 	+Ar48'
7(;<	! 	+Aq$7'
8(<=(	* 	+Aq$>'
9(=>"	$ 	+Ar4?'
7(;<"	$ 	+Aq$7'
<(@A	 	+Aq$UC'
<(@A%	' 	+Aq$>'
<(@A"	$ 	+Aq$>'
<(@A"	$K(& (&rF   c           	          t        t        t        dd      t                     t        j                  t
        d      t        t        t        dd      t                     t        j                  t
        d      gS )	zT
    Error inputs for GroupNorm that test error messages include actual values.
    r   r   ri  z9num_channels \(10\) must be divisible by num_groups \(3\)r  r      z9num_channels \(13\) must be divisible by num_groups \(5\))r   r   rz   r   r   r  )rK   r   rR   r   rS   rW   s         rD   &module_error_inputs_torch_nn_GroupNormr  _  sp    
 	"/2"6+o %77!T	
 	"/2"6+o %77!T	
 rF   c           
         t        t        |||      }t        t        d      t         |d                  t        t        d      t         |d            d      t        t               t         |d            t        d	
      gS )Nr   r(  )r   r   r   r   ri  rU   r  rl  r   r   rd  rn  ro  s          rD   !module_inputs_torch_nn_Hardshrinkr  y  s~    V5P]^J 	+B/'
<(@A	
 	+B/'
27	

 	+o'
162		
 rF   c           	          t        t        |||      }t        t               t         |d            t        d      t        t               t         |d            d      gS )Nr   r   r   rd  r  r  rl  rn  ro  s          rD    module_inputs_torch_nn_Hardswishr    s]    V5P]^J 	+o'
162		
 	+o'
<(@A	 rF   c           
         t        t        |||      }t        t               t         |d            d       t        t               t         |d            d d      t        t               t         |d	            t        d
      gS )Nr   r  c                 &    |j                  dd      S Nr   rH   r  r   s      rD   r   z1module_inputs_torch_nn_Hardtanh.<locals>.<lambda>      Q rF   r   rU   c                 &    |j                  dd      S r  r  r   s      rD   r   z1module_inputs_torch_nn_Hardtanh.<locals>.<lambda>  r  rF   r  rd  r   r   rn  ro  s          rD   module_inputs_torch_nn_Hardtanhr    s~    V5P]^J 	+o'
9(=>7	

 	+o'
277		
 	+o'
162		
 rF   c                    t        t        |||      }t        t        ||d      }di fdddifdddifd	dd
ifdddifg}g }	|D ]   \  }
}|fd}|	j                  t        t	        di |t	         |d       |d      j                  d      j                  |      j                  d      j                  d            |
|             |	j                  t        t	        di |t	         |d       |d      j                  d      j                  |      j                  d      j                  d            d|
 |              |	S )Nr   Fr2   r   r   r   r  r  r   r   r  r'  c                     t        ||fi |S r;   )r   r   s        rD   r   z?module_inputs_torch_nn_HingeEmbeddingLoss.<locals>.reference_fn  s    /1K8JKKrF   r  r   r   rH   r   rU   r   )	r   r
   rI   r   rz   r  r  r  sub_r  s                rD   )module_inputs_torch_nn_HingeEmbeddingLossr    sx   V5P]^J+fEQVWK 
R	;./	K01	K01	Hc?#%E M$) 
  8J 	L 	-*M:L*M&3Ju4E4?4F4I4I!4L4O4OPU4V4[4[\]4^4c4cde4f'h!%1	3	
 	-*M:L*M&3JrN4?O4F4Fq4I4L4LU4S4X4XYZ4[4`4`ab4c'e&tf-%1	3	

& rF   c                     t        t        |||      }di fdddifdddifddd	ifg}g }|D ]G  \  }	}
|
fd
}|j                  t        t	        di |
t	         |d       |d            |	|             I |S )Nr   r2   r   r   r   r  r  r   r   c                     t        ||fi |S r;   )r   r   s        rD   r   z6module_inputs_torch_nn_HuberLoss.<locals>.reference_fn  r   rF   r  r   rU   r<  r  s               rD    module_inputs_torch_nn_HuberLossr    s    V5P]^J 
R	;./	K01	K01	%E M$) 

  8J 	C 	-*M:L*M&3Jw4G4>w4G'I!%1	3	
	

 rF   c                 <   t        t        |||      }|j                  dd      }|d   }d\  }	}
}}}dddd	}||   }d
|z   }t        |rt	        |
|      nt	        |	|
|      t	         ||                  t        |rt	        |
|||      nt	        |	|
|||      t	         ||            d      t        |rt	        |
|      nt	        |	|
|      t	         ||            t
        d      t        |rt	        |
|||      nt	        |	|
|||      t	         ||            t
        d      t        |rt	        |
|d      nt	        |	|
|d      t	         ||            d      t        |rt	        |
|dd      nt	        |	|
|dd      t	         ||            d      gS )Nr   r   Fr  )r   r2  r  FT)r   r	  rs  r~  rH   r   r   ra  ri  tracking_statsrl  tracking_stats_no_batch_dimrd  r   T)r  r  )r  r   r  )r   r
   r   r   rz   r  )rK   r   rR   r   rS   rW   r   r   r  num_featuresr.  r  r  track_running_statsinput_no_batch_shape_dictr  r  s                    rD   %module_inputs_torch_nn_InstanceNormNdr    s   V5P]^J::fe$DsA?X<L#x)<$+	l K4Q733 	04c8,-VY[c:d'
3D(EF		H
 	MQc8V5HIlC6CVW'
3D(EF!	# 	04c8,-VY[c:d'
3G(HI2.	0 	MQc8V5HIlC6CVW'
3G(HI2	! 	=Ac8D9lC$G'
3D(EF	 	IMc8DuElC$US'
3D(EF"	$G)& )&rF   c                 ^   t        t        |||      }t        t        dgd      t         |d            d      t        t        dgd      t         |d            d      t        t        dgdd	      t         |d            d
      t        t        g dd      t         |d            d      t        t        g ddd	      t         |d            d      t        t        dgd      t         |d            d      t        t        g dddd	      t         |d            d      gS )Nr   r   r2  r  1d_elementwise_affinerl     r   r   !1d_elementwise_affine_large_batchF1d_no_elementwise_affiner   r   r   r   r   r   r   3d_elementwise_affine3d_no_elementwise_affiner   r   1d_empty_elementwise_affineT)elementwise_affiner   3d_elementwise_affine_no_biasr   ro  s          rD    module_inputs_torch_nn_LayerNormr  )  s+   V5P]^J 	+QC6'
9(=>(	* 	+QC6'
;(?@4	6 	+QCu='
9(=>+	- 	+It<'
<(@A(	* 	+ItUC'
<(@A+	- 	+QC6'
6(:;.	0 	+ItPT[`a'
<(@A0	23 rF   c                    t        t        |||      }d }t        t        dgd      t         |d            d|      t        t        dgd      t         |d            d	|      t        t        dgdd
      t         |d            d|      t        t        g dd      t         |d            d|      t        t        g ddd
      t         |d            d|      t        t        dgd      t         |d            d|      gS )Nr   c                    | j                   }|)t        j                  |j                        j                   }|j                  }| j
                  }| j                  }t        t        |            D cg c]
  }||z
  dz
   }}j                         }|t        j                  |j                  d      j                  |d      | j                   z         z  }	||	|z  }	|	j                  |      S c c}w )NrH   r   T)r   keepdim)r.  ra   finforR   ndimnormalized_shaper  r  r`   floatrsqrtr9  r  rT  )
r   r   r   r.  r	  r
  r  dims
upcasted_ir   s
             rD   rms_norm_reference_fnz=module_inputs_torch_nn_RMSNorm.<locals>.rms_norm_reference_fnN  s    ee;++agg&**Cvv--&+C0@,A&BCq1CCWWY
ekk*..*;*@*@TSW*@*X[\[`[`*`aafF~~a   Ds   2C0r   r2  r  r  r   r  r  Fr  r  r  r  r  r   r  r   )rK   r   rR   r   rS   rW   r   r  s           rD   module_inputs_torch_nn_RMSNormr  K  s   V5P]^J! 	+QC6'
9(=>(.		0
 	+QC6'
;(?@4.		0
 	+QCu='
9(=>+.		0
 	+It<'
<(@A(.		0
 	+ItUC'
<(@A+.		0
 	+QC6'
6(:;..		05 rF   c           
         t        t        |||      }t        t        d      t         |d            d      t        t        d      t         |d            d      t        t        d	d
dd      t         |d            d      gS )Nr   r   )rH   r   r  r  rl  r   )rH   r   r  r  2d_uneven_padrH   r  r'  r(  )rH   r   r  r  r  3d_custom_paramsr   ro  s          rD   (module_inputs_torch_nn_LocalResponseNormr  ~  s    V5P]^J 	+A/'
9(=>	 	+A/'
<(@A 	" 	+Ar3;'
?(CD#	% rF   c           
         t        t        |||      }t        t        dd      t         |d            d      t        t        ddd      t         |d                  t        t        ddd      t         |d	            t        d
      gS )Nr   r  r   )rH   r   r  normrl  r   ri  )r   r  r   rd  rn  ro  s          rD   module_inputs_torch_nn_LPPool1dr    s    V5P]^J 	+C3'
9(=>	 	+Aq!4'
9(=>	@ 	+Aq!4'
6(:;2		! rF   c           
         t        t        |||      }t        t        ddd      t         |d                  t        t        ddd      t         |d            t        d      t        t        dd      t         |d            d	
      gS )Nr   r   rH   r   r  r  ri  r   r  r  r   rd  r  r  rl  rn  ro  s          rD   module_inputs_torch_nn_LPPool2dr    s    V5P]^J 	+Aq!4'
<(@A	C 	+Aq!4'
9(=>2		!
 	+C3'
<(@A	 rF   c           
         t        t        |||      }t        t        ddd      t         |d                  t        t        ddd      t         |d            t        d      t        t        dd      t         |d            d	
      gS )Nr   r   )rH   r   r  r  r  ri  )r   r  r  r  r   rd  r  r  rl  rn  ro  s          rD   module_inputs_torch_nn_LPPool3dr    s    V5P]^J 	+Aq!4'
?(CD	F 	+Aq!4'
<(@A2		!
 	+C3'
?(CD	 rF   c           
         t        t        |||      }t        t        d      t         |d            d      t        t        dd      t         |d            d      t        t        dd      t         |d            d	      gS )
Nr   r   )r   r   r   r  rl  rk  Treturn_indicesr   r   ro  s          rD    module_inputs_torch_nn_MaxPool1dr!    s    V5P]^J 	+A.'
:(>?	 	+Aq1'
:(>?	 	+AdC'
:(>?!	# rF   c           
         t        t        |||      }t        t        ddd      t         |d            d      t        t        ddd      t         |d            d	      t        t        dddd
      t         |d            d      gS )Nr   r   r   rr  ru  r  r  rl  r  r  Tr  r   r   ro  s          rD    module_inputs_torch_nn_MaxPool2dr$    s    V5P]^J 	+FFFC'
9(=>	 	+FFFC'
<(@A	 	+FFFSWX'
<(@A!	# rF   c                    t        t        |||      }t        t        t	        d      t	         |d                  t
        j                  t        d      t        t        t	        d      t	         |d                  t
        j                  t        d      t        t        t	        dd	
      t	         |d                  t
        j                  t        d      gS )zQ
    Error inputs for MaxPool2d that test error messages for invalid inputs.
    r   r   r  ri  z;non-empty 3D or 4D \(batch mode\) tensor expected for inputr  rH   r   r   r   r   r   r   r  )rH   rH   r   r   z3pad should be at most half of effective kernel size)r   r
   r   r   rz   r   r   r[   ro  s          rD   &module_error_inputs_torch_nn_MaxPool2dr(    s     V5P]^J 	"/"2+Jv,>? %22#V	
 	"/"2+J,GH %22#V	
 	"/1"=+J~,FG %22#N	
- rF   c                 `   t        t        |||      }t        t        d      t         |d                  t        t        dd      t         |d            d      t        t        ddd      t         |d            d	      t        t        dddd
      t         |d            d      gS )Nr   r}  r  ri  r   rk  rl  r  stride_paddingTr  r   r   ro  s          rD    module_inputs_torch_nn_MaxPool3dr+  	  s    V5P]^J 	+I6'
?(CD	F 	+Ay9'
?(CD	 	+Aq)<'
?(CD!	# 	+Aq)DQ'
?(CD!	# rF   c                    t        t        ||      }fd}t        t        dd |             t         |d            d      t        t        d	d
 |             t         |d            d      t        t        dd |       d      t         |d            d      t        t        dd |             t         |d            t        d      t        t        d	d
 |             t         |d            t        d      gS )Nr   c                  l    t        j                  dt         j                         j                         S )N)rH   r   r   rR   r   ra   r  doubler  r
  s   rD   make_random_sampleszGmodule_inputs_torch_nn_FractionalMaxPool2d.<locals>.make_random_samples2	  #    {{9ELLHQQSSrF   r   r'  output_ratio_random_samples)rH   r   r   r  ratiorl  r  )r   r   output_sizer5  )rH   r   r  rh  sizeTr4  r5  r   ratio_return_indices)r   r   r  ratio_no_batch_dimrd  )r   r  rh  size_no_batch_dimrn  rK   r   rR   r   rS   rW   r   r1  s    `      rD   *module_inputs_torch_nn_FractionalMaxPool2dr?  /	  s   V5P]^JT 	+ACQdQfg'
<(@A	 	+FXkXmn'
<(@A	 	+5H5J[_ (
<(@A'	) 	+ACQdQfg'
9(=>2%		'
 	+FXkXmn'
9(=>2$		&) rF   c                 j   t        t        ||      }fd}t        t        dd |             t         |d            d      t        t        d	d
 |             t         |d            d      t        t        dd |             t         |d            d      t        t        dd |       d      t         |d            d      t        t        dd |             t         |d            t        d      t        t        d	d
 |             t         |d            t        d      gS )Nr   c                  l    t        j                  dt         j                         j                         S )Nr   r   r   r.  r/  r
  s   rD   r1  zGmodule_inputs_torch_nn_FractionalMaxPool3d.<locals>.make_random_samplesT	  r2  rF   r   r'  r3  )r   r   r   r   r   r6  rl  r}  )r   r   r   r7  )r   r   r  r  r  r9  )r   r   r   )r   r   r   )r   r      r  r   asymsizeTr:  r;  )r   r   r   r   r<  rd  )r   r  r  r  r=  rn  r>  s    `      rD   *module_inputs_torch_nn_FractionalMaxPool3drE  Q	  s/   V5P]^JT 	+ACQdQfg'
?(CD	 	+I9^q^st'
?(CD	 	+I:_r_tu'
3C(DE	 	+5H5J[_ (
?(CD'	) 	+ACQdQfg'
<(@A2%		'
 	+I9^q^st'
<(@A2$		&1 rF   c                 V   t        t        |||      }t        t               t         |d            d      t        t               t         |d            t        d      t        t               t         |d            d	      t        t               t         |d
            d      gS r  rn  ro  s          rD   module_inputs_torch_nn_SigmoidrG  w	  s    V5P]^J 	+o'
27	

 	+o'
162		
 	+o'
<(@A+	

 	+o'
?(CD.	
# rF   c           
         t        t        |||      }t        t               t         |d            d d      t        t               t         |d            d       t        t               t         |d	            t        d
      gS )Nr   rU   c                 >    |j                         j                         S r;   r)  r   r   s      rD   r   z3module_inputs_torch_nn_LogSigmoid.<locals>.<lambda>	      ): rF   r  rd  r  c                 >    |j                         j                         S r;   rJ  r   s      rD   r   z3module_inputs_torch_nn_LogSigmoid.<locals>.<lambda>	  rK  rF   r   r   r   rn  ro  s          rD   !module_inputs_torch_nn_LogSigmoidrM  	  s~    V5P]^J 	+o'
27:		
 	+o'
9(=>:	

 	+o'
162		
 rF   c                 f   t        t        |||      }t        t        |t        j                  d      }di fdddifdddifd	dd
ifdddifg}g }	|D ]\  \  }
}|fd}|	j	                  t        t        di |t         |d       |d       |d      j                               |
|             ^ |	S )Nr   Fr2   r   r   r   r  r  r   r   r  r'  c                     t        |||fi |S r;   )r   r  s         rD   r   z>module_inputs_torch_nn_MarginRankingLoss.<locals>.reference_fn	  s    .r2qO<NOOrF   )r  r   rU   )r   r
   ra   r  rI   r   rz   r  r  s                rD   (module_inputs_torch_nn_MarginRankingLossrP  	  s    V5P]^J+fEJJV[\K 
R	;./	K01	K01	Hc?#%E M$) 

  =O 	P 	-*M:L*M&3Ju4EzRWGX4?4F4K4K4M'O!%1	3	
	

 rF   c                    t        t        |||      }t        t        |t        j                  d      }di fdddifdddifd	dd
ifg}g }	|D ]  \  }
}|fd}|	j	                  t        t        di |t         |d       |ddd            d|
 |             |	j	                  t        t        di |t         |d       |ddd            |
|              |	S )Nr   Fr2   r   r   r   r  r  r   r   c                     t        ||fi |S r;   )r    r   s        rD   r   zAmodule_inputs_torch_nn_MultiLabelMarginLoss.<locals>.reference_fn	  s    1!QM:LMMrF   r  r   r   r  1d_r   r  rU   r   r
   ra   r  rI   r   rz   r  s                rD   +module_inputs_torch_nn_MultiLabelMarginLossrU  	  s   V5P]^J+fEJJV[\K 
R	;./	K01	K01	%E M$) 
  8J 	N 	-*M:L*M&3Ju4E4?!RT4U'W"4&\%1	3	
 	-*M:L*M&3Jw4G4?QUW4X'Z!%1	3	

( rF   c                    t        t        |||      }t        t        |t        j                  d      }t        t        ||d      }di fdddifdddifd	dd
ifdddifdddifdd |d      ifg}	g }
|	D ]J  \  }}|fd}|
j	                  t        t        di |t         |d       |ddd            ||             L |
S )Nr   Fr2   r   r   r   r  r  r   r   r   r   r  r'  r  r  r   c                     t        ||fi |S r;   )r   r   s        rD   r   z<module_inputs_torch_nn_MultiMarginLoss.<locals>.reference_fn	  s    ,QH5GHHrF   r  r   r   r  r   rU   rT  )rK   r   rR   r   rS   rW   r   r  r  r   r   r   r   r   s                 rD   &module_inputs_torch_nn_MultiMarginLossrX  	  s   V5P]^J+fEJJV[\K+fEQVWK 
R	;./	K01	K01	sAh	Hc?#	X{2/0%E M$) 

  8J 	I 	-*M:L*M&3Jw4G4?QS4T'V!%1	3	
	

 rF   c                     t        t        |||      }t        t        t	               t	         |d      t        j                  g d|t
        j                                    t        j                  t        d      gS )z
    Error inputs for MultiMarginLoss that test the improved error message
    for inconsistent target size.
    Regression test for issue #106251.
    r   r  r  r  ri  zYtarget tensor should be 1-D with size equal to.*Expected target size \[5\].*but got \[3\]r  )r   r
   r   r   rz   ra   r  r  r   r   r[   ro  s          rD   ,module_error_inputs_torch_nn_MultiMarginLossrZ  	
  sk     V5P]^J 	"//+w'LL6L %22#t	
 rF   c                    t        t        |||      }t        t        |t        j                  d      }t        t        ||d      }di fdddifdddifd	dd
ifdd |d      ifg}	dd}
g }|	D ]O  \  }}|j	                  t        t        di |t         |d       |ddd            |t        |
fi |             Q |S )Nr   Fr2   r   r   r   r  r  r   r   r  r   c                 `   ||j                         j                         z  d|z
  | j                         j                         z  z   }|||z  }| j                  |j                         dz
        |j	                  d      z  }|dk(  r|S |dk(  r|j                         S |j                         S )NrH   r   r   r  )r)  r   r   r   r9  r  r  s          rD   &multilabelsoftmargin_loss_reference_fnz_module_inputs_torch_nn_MultiLabelSoftMarginLoss.<locals>.multilabelsoftmargin_loss_reference_fn/
  s    QYY[__&&!a%QB<<>3E3E3G)GGfF'quuw{+affRj8M& ;;= ::<rF   r  r   r   r  r   r  rU   rT  )rK   r   rR   r   rS   rW   r   r  r  r   r]  r   r   r   s                 rD   /module_inputs_torch_nn_MultiLabelSoftMarginLossr^  "
  s    V5P]^J+fEJJV[\K+fEQVWK 
R	;./	K01	K01	Hk"o./%E  M$) 
  -*M:L*M&3Jw4G4?QUV4W'Y!%,-S%jWi%j	l	

 rF   c                 2   t        t        |||      }t        t        ||d      }di fdddifdddifd	dd
ifg}g }	|D ]U  \  }
}|fd}|	j                  t        t	        di |t	         |d       |d      j                               |
|             W |	S )Nr   Fr2   r   r   r   r  r  r   r   c                     t        ||fi |S r;   )r$   r   s        rD   r   z;module_inputs_torch_nn_SoftMarginLoss.<locals>.reference_fnV
  s    +AqG4FGGrF   )r   r   r   rU   r  r  s                rD   %module_inputs_torch_nn_SoftMarginLossra  I
  s    V5P]^J+fEQVWK 
R	;./	K01	K01	%E M$) 

  8J 	H 	-*M:L*M&3Jv4F4?4G4L4L4N'P!%1	3	
	

 rF   c           	         g }t        d ||||      D ]  }|j                  j                  |j                  j                  }	}||	d<   ||	d<   t	        j
                  j                  |i |	}
d}|j                  }d|j                  v r)|j                  d   |j                  d<   |j                  d= |j                  t        t        |
|      ||j                                |S )Nr   rR   r   src_maskmaskrl  ).module_inputs_torch_nn_TransformerEncoderLayerr   rV   rW   ra   r   TransformerEncoderLayerr   rI   r   rz   r   )rK   r   rR   r   rS   rW   sampleslayer_module_inputl_argsl_kwargsencoder_layer
num_layersr   s                rD   )module_inputs_torch_nn_TransformerEncoderrm  d
  s    GL&%:  /@@EE.@@GG #!88&MHM
 +88---+8+?+?
+KM  ($$Z0{+M:F'#((
 	( NrF   c                    t        t        |||      }t        t        dddd      t         |d            d      t        t        ddd	dt        j
                        t         |d            d
      t        t        ddd	dd      t         |d            d      g}d t        j                  g d|t        j                        f}d t        j                  g d|t        j                        j                  d      f}	t        j                  |	|ddd      D ]]  \  }
}}}}|j                  t        t        ddd	d|||      t         |d      |
|      t        t        |ddi      d|              _ d }|ret        j                  dd      D ]L  \  }}|j                  t        t        ddd	dd||      t         |d            |r|nd d| d|              N |S )Nr   r   r   rC  r   r  relu_activationrl  r   gelu_activationFr   r   FFTr  r#  TFd_modelnheaddim_feedforwarddropoutrD  
norm_firstr   r  )rc  src_key_padding_maskry  r   rD  rE  no_batch_dim_batch_first_rd  c                     | j                   st        d      | j                  d       t        j                         5   | |i |}d d d        | j                  d       S # 1 sw Y   xY w)Nz#Expected module.training to be TrueFT)rS   rH  trainra   no_grad)moduler   rV   rW   rQ  s        rD   fast_path_reference_fnzNmodule_inputs_torch_nn_TransformerEncoderLayer.<locals>.fast_path_reference_fn
  s_     !FGGU]]_ 	-T,V,F	-T	- 	-s   	A!!A*T)rw  rD  rx  r   	fastpath__norm_first_)r   r
   r   rz   rt  gelura   r  boolr   r  r   rI   r  )rK   r   rR   r   rS   rW   r   rg  key_padding_masks
attn_masksrc  ry  rx  rD  r   r  s                   rD   re  re  }
  s   V5P]^J 	+Aq"c:'9% #	
 	+Aq!S!&&A'9% #	
 	+Aq!SuE'9% 	
G0 u||,@W\WaWabc%9&PUPZPZ[bbcijkJj*;]M[hiE&
K"/TU8;;ED#R ,v&Pd %%>1<RhjkQln0>
	$  ) 1 1- O 	JNN&31a$:\`' #0"9-# <@!7T$TF,zlC
	 NrF   c                 p   t        t        |||      }t        t        dddd      t         |d       |d            d      t        t        ddd	dt        j
                        t         |d       |d            d
      t        t        ddd	dd      t         |d       |d            d      g}d t        j                  g d|t        j                        f}d t        j                  g d|t        j                        j                  d      f}	t        j                  |	|ddd      D ]  \  }
}}}}|
}|}|j                  t        t        ddd	d|||      t         |d       |d      |
|||      t        t        |ddd      d|               |d       |d      }}|s$|j                  dd      |j                  dd      }}||j                  dd      fdz  \  }}|j                  t        t        ddd	d|||      t        |||
|||      d| d| d|               |S )Nr   r   r   rC  r   r  ro  rl  r   rp  Fr   r   rq  r  r#  rr  rs  r  )tgt_maskmemory_masktgt_key_padding_maskmemory_key_padding_maskr   )r  r  rz  r{  rd  rH   r   norm_first__batch_first__bias_)r   r
   r   rz   rt  r  ra   r  r  r   r  r   rI   r  	transpose)rK   r   rR   r   rS   rW   r   rg  r  r  r  r  rx  r   rD  r  r  srctgts                      rD   .module_inputs_torch_nn_TransformerDecoderLayerr  
  s   V5P]^J 	+Aq"c:'9%z)'< #	
 	+Aq!S!&&A'9%z)'< #	
 	+Aq!SuE'9%z)'< 	
G. u||,@W\WaWabc%9&PUPZPZ[bbcijkJj*;]M[hi"E&
D+ "6"/TU8;;ED#R ,v&
6(:X[f)=Wn %%>1<QRop8qs 1>	 i(*Y*?S}}Q*CMM!Q,?C+=Q=X=XYZ\]=^<`cd<d9#%9"/TU8;;ED#R ,x[)=Wn #:,mK=tfU	
	1"H NrF   c                 D   t        t        |||      }g }d t        j                  g d|t        j                        f}d t        j                  g d|t        j                        j                  d      f}	t        j                  |	|ddd      D ]
  \  }
}}}}|
fdz  \  }}|fdz  \  }}|j                  t        t        dddd	d	d
|||	      t         |d       |d      ||||      t        t        |ddd      d|               |d       |d      }}|s$|j                  dd	      }|j                  dd	      }||j                  dd      fdz  \  }}|j                  t        t        dddd	d	d
|||	      t        ||||||                    |S )Nr   rq  r  r#  rr  r   r   r   rH   r   )	rt  ru  rv  num_encoder_layersnum_decoder_layersrw  rD  rx  r   r  )r  rc  r  ry  r   )r  ry  rz  r{  rd  r  r   ri  )r   r
   ra   r  r  r   r  r   rI   r   rz   r  r  )rK   r   rR   r   rS   rW   r   rg  r  r  rd  rS  rx  r   rD  rc  r  ry  r  r  r  s                        rD   "module_inputs_torch_nn_Transformerr    s   V5P]^JGu||,@W\WaWabc%9&PUPZPZ[bbcijkJj*;]M[hi$=
D+  $gk86F5H15L22"/TUCDYZ8;akrv#x ,v&
6(:XX`)=Th %%>1<QRlm8np 1>	 i(*Y*?S--1%C--1%C':J:Q:QRSUV:W9Y\]9]6 "6"/TUCDYZ8;akrv#x ,x()=Th			7$J NrF   c           
      V   t        t        j                  |t        j                  d      }t	        t        dd      t         |dd      j                  d                  t	        t        dd      t         |dd	      j                  d      j                  d
d	            d      gS )NFr   r   r   num_embeddingsembedding_dimr   ri  rH   i   r  discontiguousrl  )r   ra   r  r  r   rz   random_r   )rK   r   rR   r   rS   rW   
make_emptys          rD    module_inputs_torch_nn_Embeddingr  9  s    V5::UZ[J+1AN'
1a(8(@(@(CD	
 	+1AN'
1c(:(B(B1(E(L(LQPS(TU 	

 
rF   c                    g }t        j                  |      j                  dk(  ru|j                  t	        t        t        dd      t        t        j                  g d|t         j                                    t        j                  t        d	             |j                  t	        t        t        dd      t        t        j                  d
dg|t         j                                    t        j                  t        d	             |j                  t	        t        t        dd      t                     t        j                  t        d	             |S )zQ
    Error inputs for Embedding that test error messages for invalid inputs.
    cpur   r   r  )r   r   r	  r  ri  zindex out of range in selfr  r  g      @zFExpected tensor for argument.*indices.*to have.*scalar type.*Long.*Intr   z/Trying to create tensor with negative dimension)ra   r   rI  rI   r   r   rz   r  r  r   r   
IndexErrorr   r[   r   )rK   r   rR   r   rS   rW   rg  s          rD   &module_error_inputs_torch_nn_Embeddingr  H  s    G ||F  E)&32UV&W"/ZPV^c^h^h0i"j )66%9
	
 NN"/rQR"S+ELL#sFZ_ZgZg,hi %22#a	

 NN"/rQR"S+o %77#J	

 NrF   c                    t        t        |||      }g }d}d t        j                  g d|t        j                        f}	d t        j                  g d|t        j                        j                  d      f}
t        j                  ||||	|
      }|D ]  \  }}}}}|j                  t        t        ddd|||      t         |d	       |d	       |d	      ||
      t                     |j                  t        t        ddd|||      t         |d	       |d	       |d	      ||
      t        t        d                    |S )Nr   rr  rq  r  )r   r   r   r   T)	embed_dim	num_headsrD  r   add_bias_kvadd_zero_attnr#  )rS  	attn_maskr   FrD  )r   r
   ra   r  r  r   r  r   rI   r   rz   rT  )rK   r   rR   r   rS   rW   r   rg  	bool_valsr  r  productsr   r  r  rS  r  s                    rD   )module_inputs_torch_nn_MultiheadAttentionr  {  sX   V5P]^JGIu||,@W\WaWabc%9&PUPZPZ[bbclmnJ  IyBSU_`HIQ 
Ek=*:I"/!qVZ59{bo#q+Jv,>
6@RT^_eTf=MYbd7	
 	"/!qV[59{bo#q+Jv,>
6@RT^_eTf=MYbd$%?US	

( NrF   c                    t        t        |||      }t        t        dd      t         |d       |d            t              t        t        ddd      t         |d       |d            t              g}|j                  dd      }|rD|j                  t        t        dddd	
      t         |d       |d            t                     |S )Nr   r   r   r   Tr   is_rnnFrelu)r   nonlinearity)r   r
   r   rz   r  r   rI   )	rK   r   rR   r   rS   rW   r   rg  r  s	            rD   #module_inputs_torch_nn_RNN_GRU_Cellr    s    V5P]^J+Ar2'
1z"~F2	

 	+Ar='
1z"~F2	
G ZZ%(F 	"/2Dv"V+JqM:b>J6	
 NrF   c                    t        t        |||      }t        t        dd      t         |d       |d       |d      f      t              t        t        ddd      t         |d       |d       |d      f      t              f}|S )Nr   r   r   r   Tr   )r   r
   r   rz   r_  rK   r   rR   r   rS   rW   r   rg  s           rD   module_inputs_torch_nn_LSTMCellr    s    V5P]^J+Ar2'
1
2
SU7WX8	

 	+Ar='
1
2
SU7WX8	
G NrF   c                     | j                   }| j                  d       t        | |      }|j                  j                  |       |S )NF)r   r  r	   data)rX  batch_sizesrequired_gradseqs       rD   make_packed_sequencer    s>    %%Mu
sK
0CHHM*JrF   Fc                    t        t        |||      }|d   }d}	d}
d}d}g }|rt        |	|
||      }nt        |
||      }|D ]  }|r|\  }}}}n|\  }}}ddd|||d}ddd|||d}|r
|d<   ||d<   |j                  t	        t        di |t         |d	            t        t        |
                   |j                  t	        t        di |t         |d	       ||rdnddf            t        t        |
                   |s|j                  t	        t        di |t        t         |d      t        j                  ddg                  t        t        |
                   |j                  t	        t        di |t        t         |d      t        j                  g d                  t        t        |
                    |S )Nr   r  )r  rs  FTr   )
input_sizehidden_sizerl  rD  r   bidirectionalr   r  r   r   r  r   r   )r   r   r   r   )r   r   r   )r   r   r   r   r   rU   )
r   r
   r   rI   r   rz   rZ  r  ra   r  )rK   r   rR   r   rS   with_packed_sequencerW   r   r  r  r   rD  r  rg  prod_genrV   nlr}  b_fbidir	cons_argscons_args_hiddens                         rD   module_inputs_torch_nn_RNN_GRUr    s   V5P]^JHF#LDK!MG<{MJ4m< * $B3 MAsE#$Qa$'UL	*+AQ+.ES (*In%/1^,"/"<)"<+Jv,>?$%CQTU	
 	"/"C2B"C+Jv,>
QVA\]_`Ka@bc$%CQTU	
  NN&3&@i&@"/0DZPYEZ\a\h\hjkmnio\p0q"r!()GUX!Y NN&3&@i&@"/0DZPYEZ\a\h\hix\y0z"{!()GUX!YI*X NrF   c                     t        t        |||      }d}d}d}	d}
g }t        |||	|
      }|D ]  }|\  }}}}d}d|d||||d}d|d||||d}|j                  t	        t        di |t         |d            t        t        |      	             |d
kD  r|n|} ||rdnd|f       ||rdnd|f      f}|j                  t	        t        di |t         |d      |      t        t        |      	              |S )Nr   r  )r   r   r   r   )r  r  rl  	proj_sizerD  r   r  rr  r  r   r   r   r  rU   )r   r
   r   rI   r   rz   r\  )rK   r   rR   r   rS   rW   r   r   rD  r  
proj_sizesrg  r  rV   r}  r  r  r  r  r  r  h_outhxs                          rD   module_inputs_torch_nn_LSTMr    sD   V5P]^JDK!MJGt[-DH 
#' 3y#$[PQ`i$'UL	*+KWXgp+.ES 	"/"<)"<+Jv,>?$%@cR	
 ']	u!!U34ju!RSU`Aa6bc"/"C2B"C+Jv,>C$%@cR	
%
6 NrF   c           	          t        t        |||      }t        t        d      t         |d            t              t        t        d      t         |d                  gS )Nr   rH   r  r   rH   r   r  ri  rn  ro  s          rD   &module_inputs_torch_nn_ReflectionPad1dr  ;  ^    V5P]^J 	+A.'
6(:;2	

 	+F3'
9(=>	

 
rF   c           	          t        t        |||      }t        t        d      t         |d            t              t        t        d      t         |d                  gS Nr   rH   r  r   rH   r   r   r   r   r   r   rh  ri  rn  ro  s          rD   &module_inputs_torch_nn_ReflectionPad2dr  J  ^    V5P]^J 	+A.'
9(=>2	

 	+L9'
<(@A	

 
rF   c           	          t        t        |||      }t        t        d      t         |d            t              t        t        d      t         |d                  gS )Nr   rH   r1  r   rH   r   rH   r   rH   r   )r   r   r   r   r   ri  rn  ro  s          rD   &module_inputs_torch_nn_ReflectionPad3dr  Y  _    V5P]^J 	+A.'
<(@A2	

 	+,>?'
?(CD	

 
rF   c           	          t        t        |||      }t        t        d      t         |d            t              t        t        d      t         |d                  gS Nr   rH   r  r   r  r  ri  rn  ro  s          rD   'module_inputs_torch_nn_ReplicationPad1dr  h  r  rF   c           	          t        t        |||      }t        t        d      t         |d            t              t        t        d      t         |d                  gS r  rn  ro  s          rD   'module_inputs_torch_nn_ReplicationPad2dr  w  r  rF   c           	          t        t        |||      }t        t        d      t         |d            t              t        t        d      t         |d                  gS )Nr   rH   r  r   rH   r   r   r   r   rh  )r   r   r   rh  r  ri  rn  ro  s          rD   'module_inputs_torch_nn_ReplicationPad3dr    r  rF   c           	          t        t        |||      }t        t        d      t         |d            t              t        t        d      t         |d                  gS r  rn  ro  s          rD    module_inputs_torch_nn_ZeroPad1dr    r  rF   c           	          t        t        |||      }t        t        d      t         |d            t              t        t        d      t         |d                  gS )Nr   rH   r  r   r  ri  rn  ro  s          rD    module_inputs_torch_nn_ZeroPad2dr    r  rF   c           	          t        t        |||      }t        t        d      t         |d            t              t        t        d      t         |d                  gS )Nr   rH   r  r   r  r&  ri  rn  ro  s          rD    module_inputs_torch_nn_ZeroPad3dr    r  rF   c           	          t        t        |||      }t        t        dd      t         |d            t              t        t        dd      t         |d            	      gS )
Nr   rH   r   r  r   r  r   r  ri  rn  ro  s          rD   $module_inputs_torch_nn_ConstantPad1dr    sb    V5P]^J 	+Aq1'
6(:;2	

 	+FA6'
9(=>	

 
rF   c           	          t        t        |||      }t        t        dd      t         |d            t              t        t        dd      t         |d                  gS )	Nr   rH   r   r  r   r  r   ri  rn  ro  s          rD   $module_inputs_torch_nn_ConstantPad2dr    sb    V5P]^J 	+Aq1'
9(=>2	

 	+L!<'
<(@A	

 
rF   c           	          t        t        |||      }t        t        dd      t         |d            t              t        t        dd      t         |d            	      gS )
Nr   rH   r   r  r   r  r  )rH   r   rH   r   rH   ri  rn  ro  s          rD   $module_inputs_torch_nn_ConstantPad3dr    sc    V5P]^J 	+Aq1'
<(@A2	

 	+,>B'
?(CD	

 
rF   c                 v   t        t        |||      }d t        t        d      t         |d            t              t        t        d      t         |d            fd      t        t        d	      t         |d            fd
      t        t        d      t         |d            fd      gS )Nr   c           
      t    t        j                  | dddd|d    df   | | ddddd|d   f   gd      S )z input:
                [[[0., 1., 2.],
                  [3., 4., 5.]]]
                pad: (1, 2)
                output:
                    [[[2., 0., 1., 2., 0., 1.],
                      [5., 3., 4., 5., 3., 4.]]]
            Nr   rH   r   r   ra   catrX  pads     rD   padding1d_circular_refzDmodule_inputs_torch_nn_CircularPad1d.<locals>.padding1d_circular_ref  sB     yy#aSVGHn-sC1gs1vg4FGQOOrF   rH   r  r   r  r  c                 *     || j                         S r;   r'  r   r   r   r  s      rD   r   z6module_inputs_torch_nn_CircularPad1d.<locals>.<lambda>      )?199)M rF   )r   rH   c                 *     || j                         S r;   r'  r  s      rD   r   z6module_inputs_torch_nn_CircularPad1d.<locals>.<lambda>  r  rF   r#  c                 *     || j                         S r;   r'  r  s      rD   r   z6module_inputs_torch_nn_CircularPad1d.<locals>.<lambda>  r  rF   rn  )rK   r   rR   r   rS   rW   r   r  s          @rD   $module_inputs_torch_nn_CircularPad1dr    s    V5P]^J	P 	+A.'
6(:;2	

 	+F3'
9(=>M	

 	+F3'
9(=>M	

 	+F3'
9(=>M	
! rF   c                 v   t        t        |||      }d t        t        d      t         |d            t              t        t        d      t         |d            fd      t        t        d	      t         |d            fd
      t        t        d      t         |d            fd      gS )Nr   c                     t        j                  | dddd|d    df   | | ddddd|d   f   gd      } t        j                  | dddddd|d    df   | | ddddddd|d   f   gd      S )aS  input:
                [[[[0., 1., 2],
                   [3., 4., 5.]]]]
                pad: (1, 2, 2, 1)
        output:
            [[[[2., 0., 1., 2., 0., 1.],
               [5., 3., 4., 5., 3., 4.],
               [2., 0., 1., 2., 0., 1.],
               [5., 3., 4., 5., 3., 4.],
               [2., 0., 1., 2., 0., 1.]]]]
        Nr   r   r   r   rH   r  r  s     rD   padding2d_circular_refzDmodule_inputs_torch_nn_CircularPad2d.<locals>.padding2d_circular_ref  s     iiQCF78^,c3q!Wc!fW}3EFANyy#aAAwx/0#s1aGSVG;K7LMSTUUrF   rH   r  r   )rH   r   r   rH   )rH   rH   r   r   c                 *     || j                         S r;   r'  r   r   r   r   s      rD   r   z6module_inputs_torch_nn_CircularPad2d.<locals>.<lambda>/  r  rF   )r   r   r   r   c                 *     || j                         S r;   r'  r  s      rD   r   z6module_inputs_torch_nn_CircularPad2d.<locals>.<lambda>4  r  rF   )r   r   r   rH   )rH   rH   r   r   c                 *     || j                         S r;   r'  r  s      rD   r   z6module_inputs_torch_nn_CircularPad2d.<locals>.<lambda>9  r  rF   rn  )rK   r   rR   r   rS   rW   r   r   s          @rD   $module_inputs_torch_nn_CircularPad2dr    s    V5P]^JV  	+A.'
9(=>2	

 	+L9'
<(@AM	

 	+L9'
<(@AM	

 	+L9'
<(@AM	
! rF   c                 v   t        t        |||      }d t        t        d      t         |d            t              t        t        d      t         |d            fd      t        t        d	      t         |d            fd
      t        t        d      t         |d            fd      gS )Nr   c                 |   t        j                  | dddd|d    df   | | ddddd|d   f   gd      } t        j                  | dddddd|d    df   | | ddddddd|d   f   gd      } t        j                  | dddddddd|d    df   | | ddddddddd|d   f   gd      S )	aN  input:
                [[[[[ 0.,  1.,  2.],
                    [ 3.,  4.,  5.]],
                   [[ 6.,  7.,  8.],
                    [ 9., 10., 11.]]]]]
            pad: (1, 2, 2, 1, 1, 2)
            output: [[[[[ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.]],

                       [[ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.]],

                       [[ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.]],

                       [[ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.],
                        [ 5.,  3.,  4.,  5.,  3.,  4.],
                        [ 2.,  0.,  1.,  2.,  0.,  1.]],

                       [[ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.],
                        [11.,  9., 10., 11.,  9., 10.],
                        [ 8.,  6.,  7.,  8.,  6.,  7.]]]]]
        Nr   r   r   r   r   r   rH   r  r  s     rD   padding3d_circular_refzDmodule_inputs_torch_nn_CircularPad3d.<locals>.padding3d_circular_refA  s    J iiQCF78^,c3q!Wc!fW}3EFANiiQ1s1vgh./c!Q7CF7:J6KLRSTyy#aAq3q6'(23S#aAq'3q6'>Q:RSYZ[[rF   rH   r  r   r  )rH   rH   r   r   r   c                 *     || j                         S r;   r'  r   r   r   r  s      rD   r   z6module_inputs_torch_nn_CircularPad3d.<locals>.<lambda>s  r  rF   )r   r   r   rH   rH   r   c                 *     || j                         S r;   r'  r
  s      rD   r   z6module_inputs_torch_nn_CircularPad3d.<locals>.<lambda>x  r  rF   )r   r   r   rH   r   r   c                 *     || j                         S r;   r'  r
  s      rD   r   z6module_inputs_torch_nn_CircularPad3d.<locals>.<lambda>}  r  rF   rn  )rK   r   rR   r   rS   rW   r   r  s          @rD   $module_inputs_torch_nn_CircularPad3dr  =  s    V5P]^J'\T 	+A.'
<(@A2	

 	+,>?'
?(CDM	

 	+,>?'
?(CDM	

 	+,>?'
?(CDM	
! rF   
TestModule	test_gradcuda)	active_ifr]   test_gradgradtest_non_contiguous_tensors)r  rm   r]   c           	         t        j                  g ||      j                  d      }|t         j                  k(  rt         j                  nt         j                  }t        j                  g ||      }t        j                  g |t         j
                        }	t        t        t        |      t        ||	            t        j                  t        d      gS )z
    Error inputs for NLLLoss that test weight dtype must match input dtype.
    Regression test for device parity between CPU and CUDA with empty tensors.
    r  )r   r   r  ri  z&expected scalar type \w+ but found \w+r  )ra   r  reshaper   r   r  r   r   rz   r   r   r[   )
rK   r   rR   r   rS   rW   input_tweight_dtypeweight_ttarget_ts
             rD   $module_error_inputs_torch_nn_NLLLossr    s    
 ll2fE:BB6JG$)U]]$:5==L||Bv\BH||BvUZZ@H 	"/x"@+GX> %22#A	

 
rF   c                    t        t        |||      }t        t        t	        dd      t	         |dd       |dd                  t
        j                  t        d      t        t        t	        dd      t	         |dd       |dd	                  t
        j                  t        d
      t        t        t	        dd      t	         |dd       |dd                  t
        j                  t        d      t        t        t	        dd      t	         |dd       |dddd                  t
        j                  t        d      t        t        t	        ddd      t	         |dd       |dd	                  t
        j                  t        d
      t        t        t	        ddd      t	         |dd       |dd	                  t
        j                  t        d
      g}|S )Nr   r   r/  r      ri  5input has inconsistent input_size: got 11 expected 10r     9hidden0 has inconsistent hidden_size: got 21, expected 20r   5Input batch size 3 doesn't match hidden0 batch size 5rH   z.Expected hidden to be 1D or 2D, got 4D insteadr  rs  	r   r
   r   r   rz   r   r   r[   r  r  s           rD   )module_error_inputs_torch_nn_RNN_GRU_Cellr"    s   V5P]^J"/B"7+Jq",=z!R?PQ %22#O	
 	"/B"7+Jq",=z!R?PQ %22#S	
 	"/B"7+Jq",=z!R?PQ %22#O	
 	"/B"7+Jq",=z!QPQSU?VW %22!H	
 	"/B"?+Jq",=z!R?PQ %22#S	
 	"/B"?+Jq",=z!R?PQ %22#S	
]7Gp NrF   c                    t        t        |||      }t        t        t	        dd      t	         |dd       |dd       |dd      f            t
        j                  t        d      t        t        t	        dd      t	         |dd       |dd	       |dd	      f            t
        j                  t        d
      t        t        t	        dd      t	         |dd       |dd       |dd      f            t
        j                  t        d      t        t        t	        dd      t	         |dd       |dddd       |dddd      f            t
        j                  t        d      g}|S )Nr   r   r/  r   r  ri  r  r  r  r  r   r   rH   z/Expected hx\[0\] to be 1D or 2D, got 4D insteadr!  r  s           rD   %module_error_inputs_torch_nn_LSTMCellr$    s|   V5P]^J"/B"7+Jq",=
1b@QS]^_acSd?ef %22#O	
 	"/B"7+Jq",=
1b@QS]^_acSd?ef %22#S	
 	"/B"7+Jq",=
1b@QS]^_acSd?ef %22#O	
 	"/B"7+Jq",=
1aQRTV@WYcdeghjkmoYp?qr %22!K	
9%GL NrF   c                    t        |t              r|j                  d      d   n|j                  }|t        j
                  k(  r&|dk(  rt        j                  nt        j                  }nt        j
                  }t        t        |||      }t        t        t        ddd            t        j                  t        d	      t        t        t        ddd            t        j                  t        d
	      t        t        t        dd||      t         |d                  t        j                   t        d	      t        t        t        ddd            t        j                  t"        d	      t        t        t        ddd            t        j                  t"        d	      t        t        t        ddd            t        j                  t"        d	      t        t        t        ddd            t        j                  t"        d	      t        t        t        ddd            t        j                  t"        d	      t        t        t        ddd            t        j                  t"        d	      t        t        t        dd            t        j                  t"        d	      t        t        t        dd            t        j                  t"        d	      t        t        t        dd            t        j                  t        d	      t        t        t        dd            t        j                  t        d	      t        t        t        dd            t        j                  t"        d 	      g}	|	S )!N:r   r   r   r   rH   )r   z%hidden_size must be greater than zeror  z$num_layers must be greater than zeror   r   r.  rB  ri  zsRNN input dtype .* does not match weight dtype .* Convert input: input\.to\(.*\), or convert model: model\.to\(.*\)r   z%bias should be of type bool, got: intTruez%bias should be of type bool, got: strr  z,batch_first should be of type bool, got: intFalsez,batch_first should be of type bool, got: strg      @z,input_size should be of type int, got: float10z*input_size should be of type int, got: strz$input_size must be greater than zeror   g      @z-hidden_size should be of type int, got: float)r   strsplitrI  ra   r   r   float64r   r
   r   r   rz   r   r   r  r   	TypeError)
rK   r   rR   r   rS   rW   r]   mismatched_dtyper   rg  s
             rD   $module_error_inputs_torch_nn_RNN_GRUr/    s    +5VS*A&,,s#A&v{{K,75,@5==emm ==V;K[hiJ 	-Aq*AB$77!?		
 	-B*BC$77!>		
 	"/1E&"Q+Jy,AB %22!^		
 	-11*EF$77 ?		
 	-11*EF$77 ?		
 	-16*JK$77 ?		
 	-1!*LM$77 F		
 	-1!*LM$77 F		
 	-1'*RS$77 F		
 	-Q*?@$77 F		
 	-a*@A$77 D		
 	-1*=>$77!>		
 	-A*>?$77!>		
 	-3*?@$77 G		
s_G@ NrF   c           
          t        t        |||      }|j                  dd      }t        t	        |rt        dd      n
t        d      t         |d                  t        j                  t        d	      gS )
Nr   is_constantFrH   r   r1  ri  z(expected 2D or 3D input \(got 4D input\)r  	r   r
   r   r   r   rz   r   r   r  rK   r   rR   r   rS   rW   r   r1  s           rD   "module_error_inputs_torch_nn_Pad1dr4    sq    V5P]^J**]E2K 	9D-1"5-XYJZ+J|,DE %22!C		
 rF   c           
          t        t        |||      }|j                  dd      }t        t	        |rt        dd      n
t        d      t         |d                  t        j                  t        d	      gS )
Nr   r1  FrH   r   r  ri  z(expected 3D or 4D input \(got 2D input\)r  r2  r3  s           rD   "module_error_inputs_torch_nn_Pad2dr6    q    V5P]^J**]E2K 	9D-1"5-XYJZ+Jv,>? %22!C		
 rF   c           
          t        t        |||      }|j                  dd      }t        t	        |rt        dd      n
t        d      t         |d                  t        j                  t        d	      gS )
Nr   r1  FrH   r   r  ri  z(expected 4D or 5D input \(got 2D input\)r  r2  r3  s           rD   "module_error_inputs_torch_nn_Pad3dr9    r7  rF   r	  )r   r   test_memory_formatrS   )r  )r   r   r   zSkipped!)r   )r   r   )r   r   r   TTestEagerFusionModuleInfo,test_aot_autograd_symbolic_module_exhaustive#test_aot_autograd_module_exhaustive)rJ   r   r   r   test_check_inplacer   )r]   rm   rH   )r  r   g-C6?)r   r   r   r   r   )r   r   r   r   r   r   )r]   )r  r   r  test_cpu_gpu_parity)rm   r]   g{Gzt?)r   r   r   rm   r   r   )rm   xpu)r   rm   r   r   r   r   )atolrtoltest_forwardr  )r   r   r   )r   r   )r   r   )r   r   r   )include_halfinclude_bfloat16gQ?r2  )r   r   rm   r   )rE  rD  )r   r   rm   r   )r   r   )r  )r   rJ   r   TestModuleMPStest_non_contiguoustest_save_loadtest_factory_kwargs)rJ   r   r   r   )r]   r  gMb`?)r   r   r   r   )rJ   r   r   )r   r   r   r   r   )r  )rJ   r   r   r   )rJ   r   r   r   r   	module_db)F([  ra   unittestcopyr   enumr   	functoolsr   r   r  r   r   r*  torch.nn.functionalr   
functionalrt  torch.nn.utils.rnnr	   torch.testingr
   #torch.testing._internal.common_cudar   $torch.testing._internal.common_dtyper   r   r   *torch.testing._internal.common_device_typer   r   r   r   r   r   r   r   2torch.testing._internal.common_methods_invocationsr   !torch.testing._internal.common_nnr   r   r   r   r   r   r   r   r    r!   r"   r#   r$   r%   $torch.testing._internal.common_utilsr&   r'   r(   r)   r*   r+   r,   typesr-   operatorr9   aoqatquantizable	quantizedr.   r<   __annotations__Module	Container	NLLLoss2d	MaxPool2dr/   r>   rI  from_iterable__all__getattrr0   r1   rG  r*  	namespacemodule_namerw   rg   r   namespace_namer4   rx   rz   r   r   r   r   r   r   r   r  r   r6  r=  r  rT  rZ  r\  r_  re  rp  r{  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r   r  r  r  r#  r%  r,  r6  r;  rF  rI  rY  r]  rd  rg  rj  ro  rx  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r!  r$  r(  r+  r?  rE  rG  rM  rP  rU  rX  rZ  r^  ra  rm  re  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  r  expectedFailurer  #rnn_gru_lstm_module_info_decoratorsr  r"  r$  r/  r4  r6  r9  backendsr   is_availableis_macos_or_newer_macos15_or_newerAdaptiveAvgPool1dAdaptiveAvgPool2d
itemgetterAdaptiveAvgPool3dskipAdaptiveMaxPool1dAdaptiveMaxPool2dAdaptiveMaxPool3d	AvgPool1d	AvgPool2d	AvgPool3dr  r  BatchNorm3dCELUr   Conv1dr   Conv2dr,  Conv3dConvTranspose1dchalfConvTranspose2d	complex32	complex64
complex128ConvTranspose3dCosineEmbeddingLossELUFractionalMaxPool2dFractionalMaxPool3dL1LossSmoothL1Loss
LazyConv1d
LazyConv2d
LazyConv3dLazyConvTranspose1dLazyConvTranspose2dLazyConvTranspose3dLinearBilinearLPPool1dLPPool2dLPPool3d	MaxPool1d	MaxPool3d	KLDivLossMSELossMarginRankingLossMultiLabelMarginLossMultiMarginLossSoftMarginLossMultiLabelSoftMarginLossNLLLossGaussianNLLLossPoissonNLLLossHingeEmbeddingLoss	HuberLossBCELossBCEWithLogitsLossCrossEntropyLossCTCLossGELUGLU	GroupNorm
Hardshrink	HardswishHardtanhInstanceNorm1dInstanceNorm2dInstanceNorm3dLocalResponseNorm	LayerNormRMSNormTransformerEncoderrf  TransformerDecoderLayerTransformerMultiheadAttention	EmbeddingReLU	LeakyReLUReLU6PReLURNNCellGRUCellLSTMCellSigmoid
LogSigmoidSiLUSoftmax	Softmax2d
LogSoftmaxSoftminSoftplus
SoftshrinkSoftsignTanh
Tanhshrink	ThresholdMishRNNGRULSTMReflectionPad1dReflectionPad2dReflectionPad3dReplicationPad1dReplicationPad2dReplicationPad3dSELU	ZeroPad1d	ZeroPad2d	ZeroPad3dCircularPad1dCircularPad2dCircularPad3dConstantPad1dConstantPad2dConstantPad3drJ  )rg  rh  clss   000rD   <module>r     s+       $ $     3 % :G G* * * Lm m m m
" " "   
HH	HHKKOO	HHKK##	HHKK!!	HHKK!!' 4
#  
HHOO	HH	HH	HHKK##	HHKK##T *2u22&4( 9B8I8IJWY$J4(  ) *T
 * "0N#3o3M#N ') DsO (" OI (( OY4
"++33HbAII*VXY //0>/?q-Nz*OO _&STB BJ# 2 2,d 
' '(<+ <+@(:%P@F8-`$N,^D&D0b0
<4($080Dh( (*&,
*(&-2.<.4.4@(8&&="=="9&9&0*"+J$L"*"
****$*:*$*"* **&*"WF-`'TUp`F=@+&\4,"0 F42&h D0f&(&&&&%P,D#L808 F>2$N62HV@F+\/f>>$;|&T#J'RBR   ,4n"46   ,4n"46
   ,0M4n"46
   ,0M0.5;;.V\'' #6,:x(VkZ$$& NN&&335e%..:L:L:^:^_acd:e Vuxx))"J  (+V uxx))$8"J  (//#+!4!4!4Z!@			V* uxx))$8"Jj 9<I]^(++V: uxx))"J;V@ uxx))$8"JAVH uxx))$8"Jj 9<I]^(+IVX uxx!!"BYV^ uxx!!"B
  //#+!4!4!4Z!@#),	
_Vx uxx!!"B$8  j 9<I]^7TU	XyVJ uxx##%)"D(V  //1LE!4!4!4Z!@  //1L<!4!4!4Z!@	KVp uxx##%)"D(V   2LBVbubjbubu  wA  cB  C  //1LE!4!4!4Z!@  //1L<!4!4!4Z!@	qVZ uxx##%)"D(V  (  //1LE!4!4!4Z!@  //1L<!4!4!4Z!@	[VD uxx}}"=   8 8,H\,15==/KMEVR uxx")*G1SX"Y$8,0 15==%2H I<Ymn	SV` uxx")*G1SX"Y$8,0   8 8,H\,2EMM?L   15==%2H I<YmnaVz uxx")*G1SX"Y$8,0  U;   8 8,H\]   15==%2H I<Ymn{VV uxx''")*G1SXei"j$8,04U[[A   8 8,H](-~6K   15==%2H I<Ymn 15;;2F GWklWVr uxx''")*G1SXei"j$8,04U[[A   8 8,H\(-%JZJZ'[]   8 8,H\jp(-u7G7G'HJ   8 8,H](-~6K	<9N(-~5J   15==%2H I<Ymn 15;;2F GWkl'sV` uxx''")*G1SXei"j4U[[A$8,0  (   8 8,H\] 8 8,H](-~6K	<9N(-~5J   15==%2H I<Ymn 15??E2J K\[pq 15;;2F GWkl#aVL uxx++"L  j 9<I]^aMVX uxx||"<   8 8,H\,15==/KMYVf uxx++"L$8  (j 9<I]^agVv uxx++"L$8  (j 9<I]^awVF uxx"?  j 9<I]^aGVR uxx$$"E  j 9<I]^  	<9V(-EKM	SVf uxx""")*G1SW"X$8,0  )   15==%2H I<YmngV~ uxx""")*G1SW"X$8,0  )   8 8,H\,2EMM?L   15==%2H I<YmnV^	 uxx""")*G1SW"X$8,0  )(   8 8,H\]	   15==%2H I<Ymn_	V@
 uxx++")*G1SWdh"i$8,0  )   15==%2H I<YmnA
VX
 uxx++")*G1SWdh"i$8,0  )   8 8,H\jp(-8   15==%2H I<YmnY
Vx
 uxx++")*G1SWdh"i$8,0  )(   8 8,H\]	   15==%2H I<Ymny
VZ uxx"?  j 9<I]^a[Vf uxx  "A( ==#4d*C ==#4d*C*E F $^	H  j 9<I]^ 8 8,,15==/K	MgVF uxx  "Aj 9<Uj 9<Y\GVR uxx  "Aj 9<Uj 9<Y  //#+!4!4!4Z!@!ZIZEZ#(SVp uxx  "Aj 9<Uj 9<Yj 9<I]^	u=	@qV@ uxx!!"BAVF uxx!!"B(NGVN uxx!!"B$8OVV uxx!!"B  j 9<I]^j 9<I^_j 9<Uj 9<Y\	WVj uxx"@  j 9<I]^ 8 8,,15==/K	MkV| uxx))"J  j 9<I]^a}VH uxx,,"M  j 9<I]^	<UKj 9<Y\	IV\ uxx''"H(T  j 9<I]^	<UKj 9<Y\
]Vr uxx&&"G  j 9<I]^ 8 8,,15==/K	MsVD uxx00"Q  j 9<I]^aEVP uxx"@(L  j 9<I]^ 8 8,,15==/K	M	QVd uxx''"H  j 9<I]^abeVn uxx&&"G  j 9<I]^aboVx uxx**"K  j 9<I]^ayVD uxx!!"B  j 9<I]^ 8 8,,15==/K	MEVV uxx"@  j 9<I]^	<]bc	fWVf uxx))"J  j 9<I]^	<]bc	fgVv uxx(("I(U'TER   8 8,H\] 15==#4VZB[2\ ]_k .TY[ 8 8,H]glgtgtfu,24 8 8,H]glgtgtfu,135	wVR uxx"@  j 9<I]^	<ULj 9<Uj 9<Yj 9<Ifg	jSVl uxx}}"=   8 8,,15==/KMmVz uxx||"<{V@ uxx!!"B(N'DQ  j 9<I^_ 15==#4VZB[2\ ] ,.BPUW  j 9<I]kqrj 9<I]kpqj 9<I]kpqj 9<*8fN
P	AV` uxx"""CaVf uxx!!"B!&(gVl uxx  "AmVr uxx&&")*OST"U%)  j 9<I]^asV@ uxx&&")*OST"U%)  j 9<I]^aAVN uxx&&")*OST"U%)   2OEYZ 2OEbc 2O^T 2OEZ[ 2OEUVj 9<I]^aOVh uxx))"JiVn uxx!!"B  j 9<I]^aoVz uxx"@{VB uxx**%)"K  j 9<,13  j 9<I]^   8 8,H]^aCV` uxx//%)"P 15==#4VZB[2\ ] ,.K,1ZI   15==#4VZB[2\ ] ,n,13  j 9<,13
  j 9<I]^aaVD uxx//"P  j 9<,13  j 9<I]^a
EVZ uxx##"D $(  j 9<,13  j 9<I]^a[Vv uxx**%)"K  j 9<I]^awVD uxx!!"B(N$8 15==#4VZB[2\ ] ,.K,134
  j 9<I]^a
EVZ uxx}}"=.T  //#+!4!4!4Z!@#(	5[Vt uxx!!"BuVz uxx~~">  (+{VH uxx~~">  (+IVV uxx")*MVZ"[(QWV^ uxx"E(Q_Vf uxx  "A(MgVn uxx"@.T  //#+!4!4!4Z!@#(	5oVH uxx"""C   8 8,disx  tA  tA  sB  CEIVT uxx}}"=UVZ uxx"@[V` uxx!!"B  j 9<I]^ 8 8,disx  tA  tA  sB  C	EaVp uxx"""C  j 9<I]^ 8 8,disx  tA  tA  sB  C	EqV@ uxx"@  j 9<I]^aAVL uxx  "A  (+MVZ uxx"""C  (+[Vf uxx  "AgVl uxx}}"=.T  //#+!4!4!4Z!@#(	5mVF uxx"""C.T  //#+!4!4!4Z!@#(	5GV` uxx!!"B  (+aVn uxx}}"=  (+oVz uxx||%)")*HQU"V(L=	{VF uxx||%)")*HQV"W(L=	?GVP uxx}}%)"=(L  (+ >?QV` uxx''"HaVf uxx''"H$8j 9<I],24j 9<I],13j 9<I],135
gV| uxx''"H$8j 9<I],24j 9<I],13j 9<I],135
}VR uxx(("ISVX uxx(("I$8j 9<I],24j 9<I],13j 9<I],135
YVn uxx(("I$8j 9<I],24j 9<I],13j 9<I],135
oVD uxx}}"=  (+EVR uxx!!"BSVX uxx!!"B   8 8,H\jopsYVd uxx!!"B   8 8,H\jopseVp uxx%%"F(JqVx uxx%%"F(JyV@ uxx%%"F(J   8 8,H\]`AVN uxx%%"FOVT uxx%%"F   8 8,H\jopsUV` uxx%%"F   8 8,H\jopsaV	4
 V_t K4( Os+   BZ4,BZ/<BZ4	BZ: BZ:Z/BZ4