
    9j                     3   U d dl Z d dlZd dlZd dlZd dlZd dlmZ d dlmZm	Z	m
Z
 d dlmZ d dlZd dlmZ d dlmZ d dlZd dlmZmZmZ d dlmZmZmZmZmZ d dl m!Z! d dl"m#Z# d d	l$m%Z% d d
l&m'Z' d dl(m)Z) d dlm*Z*m+Z+m,Z,m-Z- d dl.m/Z0 d dl1m2Z2m3Z3 ddl4m5Z5m6Z6 ddlm7Z7m8Z8m9Z9  e
d      Z: ed      Z;ejx                  jz                  ejx                  j|                  z  Z?e	e@d<    ej                  eB      ZCej                  j                  ZEej                  j                  ZFej                  j                  ZGej                  j                  ZHej                  j                  ZI eg eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj^                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  e'eEj                  eEj                  eEj                  eEj                  j                  eEj                  j                  eEj                  eGj                  eHj                        Zui  e       euZveEj                  eEj                  eEj                  eEj                  j                  eEj                  eEj                  eEj                  eEj8                  eEj^                  eEj                  eEj                   eEj                  eEj                  eEj                  eEj                  j
                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  eEj                  j                  eEj                  eEj                  eEj                   eEj"                  j                  eEj$                  eEj&                  eEj(                  gZeejx                  j.                  ejx                  j|                  z     e@d<    eeve       de?ee?   z  deee;e:f   gee;e:f   f   fdZ eeEj&                  j                  g      dej
                  dej
                  dedej
                  fd       Z eeEj&                  j
                  g      dej
                  dej
                  dej
                  dej
                  fd       Z eeEj8                  g      dej
                  dej
                  d eod!eod"edej
                  fd#       Z eeEj^                        d$ej
                  d%ej<                  dej
                  dz  d&ej
                  dz  d'edeej
                  ej
                  ej
                  f   fd(       Z eeEjB                  j                  g      ddd)d*ejD                  d+ejF                  jH                  dz  d,ejF                  jH                  dz  ddfd-       Z eeEjJ                  g      e	 	 dd.ej
                  d+ejF                  jH                  dz  d,ejF                  jH                  dz  dej
                  fd/              Z eeEj                  g      ed.ej
                  dej
                  fd0              Z eeEjL                  g      d1eeoejD                  z     d2ejF                  jH                  d3edej
                  fd4       Z eeEj                  g      dd5d.ej
                  d6eod7ej
                  d8ej
                  d9ejF                  jH                  dej
                  fd:       Z eeEjN                  j                  g      d1eeoejD                  z     d;eeo   d3edej
                  fd<       Z eeEjP                  g      dej
                  d$ej
                  dej
                  d=eeo   d>eoeeo   z  d?eoeeo   z  d@eoeeo   z  dAedBeeo   dCeodDee   deej
                  ej
                  ej
                  f   fdE       Z eeEjR                  jT                  g      dd.ej
                  dFeodej
                  fdG       Z eeEjX                  g      e	 ddHej
                  dIej
                  dJejZ                  dz  dej
                  fdK              Z eeEj\                  g      e	 	 	 ddHej
                  dLej
                  dMej
                  dJejZ                  dz  dNejF                  jH                  d9ejF                  jH                  dej
                  fdO              Z eeEj^                  g      e	 ddHej
                  dPej
                  dJejZ                  dz  dej
                  fdQ              Z eeEj`                  j                  g      	 ddReej
                     d6eodej
                  fdS       Z eeEjb                  g      d.ej
                  dej
                  fdT       Z eeEjd                  g      dd5d.ej
                  dUej
                  d9ejF                  jH                  dz  dej
                  fdV       Z eeEjf                  g      dHej
                  dej
                  fdW       Z eeEjh                  eEjj                  g      dHej
                  dej
                  fdX       Z eeEjl                  eFjl                  g      dHej
                  dYej
                  dej
                  fdZ       Z eeEjn                  eFjn                  g      dHej
                  dYej
                  dej
                  fd[       Z eeEjp                        	 	 ddHej
                  d6eodz  d]edej
                  fd^       Z eeEjr                        	 	 ddHej
                  d6eodz  d]edej
                  fd_       Z eeEjt                  g      dHej
                  d6eodeod`eodej
                  f
da       Z eeEjv                  j                  g      dHej
                  d1eeoejD                  z     dej
                  fdb       Z eeEjv                  jZ                  g      dHej
                  dcejZ                  dej
                  fdd       ZdHej
                  deej<                  ej|                  f   fdeZ eeEj                        dddd\d\ej                  dfdHej
                  d2eoez  dcejZ                  dz  dgej                  dz  dhej                  dz  diedjedkej                  dej
                  fdl       Zddej                  dmdnedoej
                  f   dHej
                  dcejZ                  dz  dhej                  dz  dkej                  d3edej
                  fdpZ ej0                  eEj                  ge!      dHej
                  d3edej
                  fdq       Z ej0                  eEj                  ge!      dHej
                  d3edej
                  fdr       Z ej0                  eEj                  j                  ge!      dHej
                  dseod3edej
                  fdt       Z ej0                  eEj                  j                  ge!      dHej
                  dueodseod3edej
                  f
dv       Z eeEj                  j                        dseod1eeoejD                  z     d3edej
                  fdw       Z eeGj                  j                        	 dd$ej
                  dej
                  d&ej
                  dz  dej
                  fdx       Zs eeHj                  j                        d$ej
                  dyej
                  dzej
                  dej
                  d{ej
                  d|ej
                  d&ej
                  d}ej
                  d~ej
                  deodej
                  fd       Zt eej                  j                  j                        dej
                  dej
                  fd       Z eeEj                  g      e	 	 	 ddej
                  dej
                  deodeodedej
                  fd              Z eeEj                  j                        deej
                     deej
                     dejF                  jH                  deej
                     fd       Z eeEj                  j                        deej
                     deej
                     deejF                  jH                     deej
                     fd       ZeEj                  j                  j                  ej                  j                  j                         eeEj                        d$ej
                  dej
                  d&ej
                  dz  dej
                  dz  dej
                  dz  dedededeej
                  ej
                  ej
                  f   fd              Ze j                  deeedoef   f   fd       Zdeeedoef   f   fdZ eeEj                        dHej
                  dej
                  dej
                  dej
                  fd       Z eeIj                  j                        d$ej
                  deodeod'edcejZ                  deej
                  ej
                  f   fd       Z eeEj                        	 ddHej
                  d7ej
                  dej
                  dedej
                  f
d       Z eeEj                        	 ddHej
                  d7ej
                  dej
                  dedej
                  f
d       Z eeEj                  j                        edej
                  dej
                  d6eodejZ                  dej
                  f
d              Z| eeEj                        dddHej
                  d6eod7ej
                  dej
                  dededej
                  fd       Zd.ej
                  deeo   d>eoeeo   z  dz  d?eoeeo   z  d@eoeeo   z  ded6eodeej
                  ej
                  f   fdZ eeEj                        	 	 	 	 dd.ej
                  deeo   d>eoeeo   z  dz  d?eoeeo   z  d@eoeeo   z  dedeej
                  ej
                  f   fd       Z eeEj                        	 	 	 	 dd.ej
                  deeo   d>eoeeo   z  dz  d?eoeeo   z  d@eoeeo   z  dedeej
                  ej
                  f   fd       Z eeEj                        d.ej
                  deeo   deej
                  ej
                  f   fd       Z eeEj                  j                        d\d\ddddej
                  dHejF                  jH                  dedededz  dej
                  dz  dej
                  fd       Z eeEj                  j                        d\d\ddHejF                  jH                  dej
                  dededej
                  f
d       Z eeEj                        	 	 	 	 ddHej
                  dej
                  dedededej                  dz  deej
                  ej
                  f   fd       Z eeEj                  j
                        	 ddej
                  deodz  dej
                  fd       Z	 	 	 	 	 dd$ej
                  dej
                  d&ej
                  dz  d>eeo   d?eeo   d@eeo   dCeodej
                  fdZy)    N)Callable)Any	TypeAliasTypeVar)	ParamSpec)core_aten_decompositionsget_decompositionsremove_decompositions)_grid_sampler_2d
_index_addembedding_dense_backwardpw_cast_for_opmath"pw_cast_for_opmath_non_tensor_args)extra_random_decomps)counters)	is_fbcode)	out_dtype)pad_listlike)elementwise_dtypesELEMENTWISE_TYPE_PROMOTION_KINDsuggest_memory_formattype_to_dtype)native_layer_norm)guard_or_falsestatically_known_true   )configinductor_prims)is_gpu,needs_fallback_due_to_atomic_add_limitationsuse_scatter_fallback_T_P_GenericOperatordecomps_to_excludeopsreturnc                     t        | t              r| n| gD ]!  }|t        v st        j	                  d|        # t        j                  | t              S )Nzduplicate decomp: %s)
isinstancelistdecompositionslogwarningdecompregister_decomposition)r&   ops     ]/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/_inductor/decomposition.pyr/   r/      sL      T*c 5KK.45 ((n==    startendweightc                     t        |       }|t        j                  k7  r|j                  |      }|| z
  }|dk\  s|dk  rt        j                  ||d|z
         S t        j                  | ||      S )Nmemory_format      ?g            ?alpha)r   torchcontiguous_format
contiguousadd)r3   r4   r5   fmtdiffs        r1   _lerp_scalarrC      sn      
&C
e%%%nn3n/;D}$yydC&L/::99UD//r2   c                 ,   t        |       }|t        j                  k7  r|j                  |      }|| z
  }|j	                         dk\  }d|z
   }t        j
                  |||      }t        j
                  |||       }t        j                  |||d      S )Nr7   r9   r:   r   )value)r   r=   r>   r?   abswhereaddcmul)	r3   r4   r5   rA   rB   maskneg_omwwbases	            r1   _lerp_tensorrM      s      
&C
e%%%nn3n/;D::<3DfoGD'6*A;;tS%(D==q$a00r2   grad_outputindicesnum_weightspadding_idxscale_grad_by_freqc                 D    | j                   rt        S t        | ||||      S N)is_xpuNotImplementeddecomp_embedding_dense_backward)rN   rO   rP   rQ   rR   s        r1   _embedding_dense_backwardrX      s,     *Wk;8J r2   inputnormalized_shapebiasepsc                 D    | j                   rt        S t        | ||||      S rT   )is_mtiarV   decomp_native_layer_norm)rY   rZ   r5   r[   r\   s        r1   _native_layer_normr`      s%     }}#E+;VT3OOr2   )minmaxsymbolra   rb   c                     y rT    )rc   ra   rb   s      r1   sym_constrain_range_for_sizerf      s     r2   xc                 R    || j                  |      } || j                  |      } | S rT   )	clamp_min	clamp_max)rg   ra   rb   s      r1   clamprk      s/     KK
KKHr2   c                 J    | d| j                         j                         z   z  S )Nr   )negexp)rg   s    r1   siluro      s      AEEGKKM!""r2   size
fill_valuekwargsc                     |j                  d      }|.t        t        |            |d<   t        j                  | |fi |S t
        S )Ndtype)getr   typer=   fullrV   )rp   rq   rr   rt   s       r1   rw   rw      sF     JJwE}'Z(89wzz$
5f55r2   r;   dimindextensorr<   c                ~    t               s#| j                  t        j                  k(  rt        S t        | |||d|      S )NF)inplacer<   )r   rt   r=   bfloat16rV   r   )rg   rx   ry   rz   r<   s        r1   	index_addr~     s3     ;177enn4!S%eLLr2   physical_layoutc                 H   t        |      t        t        t        |                  k(  }|rt        j                  | fi |S dgt        |       z  }t        |      D ]
  \  }}|||<    t        j                  |D cg c]  }| |   	 c}fi |j                  |      S c c}w Nr   )r*   rangelenr=   empty	enumeratepermute)rp   r   rr   is_identitypermpls          r1   empty_permutedr   !  s     '4c/6J0K+LLK{{4*6**sSYo. 	DAqDG	{{_=DG=HHPPQUVV=s   <B
bias_sizesstridepaddingdilation
transposedoutput_paddinggroupsoutput_maskc                 .   |
d   rt        | j                  j                        st        S t        j                  | dgt        t        d| j                                     z         }t        j                  | |||||||||	|
d   |
d   dg      \  }}}|||fS )N   r   r   F)
r   devicerv   rV   atensumr*   r   rx   convolution_backward)rN   rY   r5   r   r   r   r   r   r   r   r   	grad_biasgrad_inpgrad_weight_s                  r1   r   r   2  s     q>(:(:(?(?!@qcDq+//:K1L,M&MNI#88	QQ/ Hk1 k9--r2   decimalsc                 H    d|z  }t         j                  | |z        d|z  z  S )Ng      $@r:   )r   round)rg   r   ten_pow_decimalss      r1   	round_decr   S  s,    X~::a**+s5E/EFFr2   selfbatch2r   c                 $   t        | j                  d   dk(        st        |j                  d   dk(        r| |z  j                         S t        j                  r| j
                  j                  dvrkt        | j                  d   dk(        st        |j                  d   dk(        r5| j                  d      |j                  d      z  j                  d      }|S | j
                  j                  dk(  rt        | j                  d      dk(        ryt        |j                  d      dk(        r\t        d   dxx   dz  cc<   t        j                  | j                  d      |j                  d      z  dd	
      j                  d      S t        S )Nr   r   cpumpsrx   r   inductordecompose_bmmTrx   keepdim)r   shaper?   r   coordinate_descent_tuningr   rv   	unsqueezer   rp   r   r=   squeezerV   )r   r   r   outs       r1   bmmr   Y  sW    TZZ]a/04IQ15 v))++ ''DKK,<,<N,R A!!348MLLOq 9
 >>"%(8(8(;;@@Q@GCJ{{5  1!238MKKOq 9
 Z 1Q6199Q&.."44!Til r2   mat1mat2betac                    |j                   j                  dvr{t        |j                  d      dk(        r^t        |j                  d      dk7        rAt        |j                  d      dk7        r$t        d   dxx   dz  cc<   ||z  }||z  || z  z   S | j                   j                  dk(  r7t        |j                  d      dk(        rt        |j                  d      dk(        rgt        d   dxx   dz  cc<   t        j                  |j                  d      |j                  d      z  dd	      j                  d      }||z  || z  z   S t        |j                  d      dk(        ryt        |j                  d      d
k        r\t        |j                  d      d
k        r?t        d   dxx   dz  cc<   |j                  |z  j                  dd	      }||z  || z  z   S t        S )Nr   r   r   r   r   decompose_addmmr   Tr      )r   rv   r   rp   r   r=   r   r   r   r   TrV   )r   r   r   r   r   r<   r   s          r1   addmmr   z  s    {{~-!$))B-1"45%diila&78%diila&78Z !23q83+C3;,,{{5  1!238MIIbMQ9
 Z !23q83))Q$,,r"224il  3;,,!$))A,!"34tyy|r12tyy|r12Z !23q8366D=%%!T%:C3;,,r2   input2c                    t         j                  r| j                  j                  dvrit	        | j
                  d   dk(        st	        |j
                  d   dk(        r3| j                  d      |j                  d      z  j                  d      S t	        | j                  d      dk(        rt	        | j                  d      dk7        rt	        |j                  d      dk7        r| j                  j                  dvs| j                  j                  dk(  rt	        | j                  d      dkD        rt	        |j                  d      dk(        ri| j                  |j                  k(  rPt        t        j                  |       t        j                  |      z   dk        rt        d	   d
xx   dz  cc<   | |z  S | j                  j                  dk(  rt	        | j                  d      dk(        ryt	        |j                  d      dk(        r\t        d	   d
xx   dz  cc<   t        j                  | j                  d      |j                  d      z  dd      j                  d      S t        S )Nr   r   r   r   r   r   r       r   decompose_mmTr   )r   r   r   rv   r   r   r   r   rp   rt   r   r=   numelr   r   rV   )r   r   r   s      r1   mmr     s    ''DKK,<,<N,R A!!348MLLOq 9
 NN1%(8(8(;;@@Q@GG 	diimq01!$))A,!"34!&++a.A"56;;>1KK%%diilQ&67%fkk!n&9:v||+D 1EKK4G GBNOZ 0A50&= {{5  1!238MKKOq 9
 Z 0A5099Q&.."44!Til r2   tensorsc                    dt         j                  dt        ffd}t        t	        ||             t              dk(  rRt        | dt        j                  id   }d   }||j                  k(  r|j                         S |j                  |      S dt              cxk  rt        |       k  r#n n t        j                  j                        S t              dkD  rt        fdD              rd   }t        |j                         }dk  rt        |j                         z   n|j#                  t                      |j%                        j&                  | j)                  dz         j                         S t*        S )	Nrg   r'   c                     t        | j                        dk(  rt        | j                  d   dk(        ryt        | j                        k  rt        | j                     dk(        ryy)Nr   r   FT)r   r   r   )rg   rx   s    r1   non_empty_tensorzcat.<locals>.non_empty_tensor  sU    " qww<1
a!@QWW.1B"Cr2   r   type_promotion_kindr   rt   c              3   ,   K   | ]  }|d    u   yw)r   Nre   ).0tfiltered_tensorss     r1   	<genexpr>zcat.<locals>.<genexpr>  s!      )%&a  )s   )r=   Tensorboolr*   filterr   r   r   DEFAULTrt   clonetor   catdefaultallr   insertr   expandflattenrV   )r   rx   r   promoted_dtype
filtered_tinpr   r   s    `     @r1   r   r     s   
ELL T 2 F#3W=>
!+
 ? G G
  &a(
 !1!11 	
 ^4	

 
S!"	1S\	1xx 0#66 q S )*:) & q!SYY&)AgcC		N"3S#./0(s}}S!((%088cAgFLLNN r2   c                 "   | j                         ret        j                  t        j                  | j                        t        d      t        j                  | j                  | j                              S t        | t        j                        \  }}t        j                  t        j                  || j                        }t        j                  | dk  |d      }t        j                  t        j                  |       t        d      |      S )Nnan)r   rt   r   r   g        )
is_complexr=   rG   isnanrealfloatatan2imagr   r   INT_TO_FLOATscalar_tensormathpir   )rg   r   rt   r   rets        r1   angler     s    ||~{{KKuu{{166166/J
 	
 "	;HHHAu 
		TWWE!((	CB
++a!eR
%C;;u{{1~uU|S99r2   yc                   t        j                  |       xr | j                         }t        j                  |      xr |j                         }|r|st        S dt         j                  dt
        fd}d}| j                  dk(  r|j                  dk(  rd}| j                  dk(  r| j                  d      } |j                  dk(  r|j                  d      }|}|||z  }t        j                  | j                  |j                        } ||       s ||      rt        S dt         j                  dt         j                  fd}	| dz   } |dz   } |	| j                  | j                  j                              }
 |	|j                  |j                  j                              }t        j                  |
|z   d	
      j                  |      }|r|d   S |S )Nrz   r'   c                 N    | j                   dk(  ry| j                         d   dk7  S )Nr   Fr   r   )ndimr   )rz   s    r1   _requires_fallbackzadd.<locals>._requires_fallback0  s'    ;;!}}r"a''r2   Fr   Tr   c                     | j                   ^ }}|dz  dk7  rt        d      g ||dz  d}| j                  |      }|S )zNReshape tensor from [*initial_dims, last_dim] to *initial_dims, last_dim/2, 2]r   r   zQThe size of the last dimension must be even to reshape it to [..., last_dim/2, 2])r   AssertionErrorview)rz   initial_dimslast_dim	new_shapereshaped_tensors        r1   reshape_tensor_complexz#add.<locals>.reshape_tensor_complexI  s^     #),,x a<1 c 
 6l5HM515	 ++i0r2   )	start_dim)r=   	is_tensorr   rV   r   r   r   reshapepromote_typesrt   r   r   r   )rg   r   r<   x_is_complex_tensory_is_complex_tensorr   output_size_zerozcomplex_typer   
x_reshaped
z_reshapedresults                r1   r@   r@   #  s     //!,?//!,?&9(5<< (D ( vv{qvv{vv{IIaLvv{IIaL	AAI&&qww8L! 21 5u||  $ 	
AA	AA'qvv||(<=J'qvv||(<=J]]:
2bAFF|TFayMr2   c                 2    | j                         rt        S | S rT   )r   rV   r   s    r1   conj_physicalr  g  s    Kr2   c                     | S rT   re   r  s    r1   liftr
  n  s    Kr2   otherc                 b    t        j                  t        j                  |      || kD  z  | |      S rT   r=   rG   r   r   r  s     r1   fminr  s  '    ;;u{{5)UT\:D%HHr2   c                 b    t        j                  t        j                  |      || k  z  | |      S rT   r  r  s     r1   fmaxr  x  r  r2   Fr   c                 x    | j                   t        j                  k(  rt        j                  | ||      S t        S Nr   )rt   r=   r   anyrV   r   rx   r   s      r1   amaxr  }  -     zzUZZyy388r2   c                 x    | j                   t        j                  k(  rt        j                  | ||      S t        S r  )rt   r=   r   r   rV   r  s      r1   aminr    r  r2   lengthc                 n    t        j                  | |||      j                  t         j                        S )Nr7   )r=   narrowr   r>   )r   rx   r3   r  s       r1   narrow_copyr    s4     <<c5&177-- 8  r2   c                 J    t         j                  | |      j                         S rT   )r   r   r   )r   rp   s     r1   view_copy_defaultr     s    
 99T4 &&((r2   rt   c                 @    | j                         j                  |      S rT   )r   r   )r   rt   s     r1   view_copy_dtyper"    s    
 ::<U##r2   c                     t        j                  |       \  }}|D cg c]  }| j                  |    }}dgt        |      z  }t	        |      D ]
  \  }}|||<    ||fS c c}w r   )utils3compute_elementwise_output_logical_to_physical_permr   r   r   )r   r   r   r   r   permutationr   s          r1   _get_shape_permutation_liker'    sx     RRSWXOQ$34qTZZ]4E4#E
"K/* 1A ; 5s   A )rt   layoutr   
pin_memoryrequires_gradr8   r(  r   r)  r*  r8   c          	         || j                   n|}|| j                  n|}|| j                  n|}|t        j                  k7  r8t        j
                  | j                  ||||||      }|j                  |      S |t        j                  k(  sJ t        |       \  }	}
t        j
                  |	||||||      }|
t        t        t        |
                  k(  r|S |j                  |
      j                         S )N)rt   r(  r   r)  r*  r7   )rt   r(  r   r=   preserve_formatrw   r   r   stridedr'  r*   r   r   r   r   )r   rq   rt   r(  r   r)  r*  r8   r  r   r&  s              r1   	full_liker.    s      -DJJUE"NT[[F"NT[[F---JJ!'
 yy}y55 &&&8>{!'
 $uS%5677M~~k*0022r2   )rt   r   r8   rand_fn.c                j   ||j                   n|}||j                  n|}|t        j                  k7  r& | |j                  f||d|j                  |      S t        |      \  }} | |f||d|}|t        t        t        |                  k(  r|S |j                  |      j                         S )Nr   r7   )rt   r   r=   r,  r   r   r'  r*   r   r   r   r   )	r/  r   rt   r   r8   rr   r   r&  r  s	            r1   
_rand_liker1    s      -DJJUE"NT[[F---JJ

 	

 "="
)	* 5T:E; 	F d5[!1233>>+&,,..r2   c                 8    t        t        j                  | fi |S rT   )r1  r=   randr   rr   s     r1   	rand_liker5    s    ejj$1&11r2   c                 8    t        t        j                  | fi |S rT   )r1  r=   randnr4  s     r1   
randn_liker8    s    ekk42622r2   highc                 v    t        t        j                  t        j                  j
                  d|      | fi |S r   r1  	functoolspartialr   randintlow)r   r9  rr   s      r1   randint_liker@    s-    i''(8(8!TBDSFSSr2   r?  c                 v    t        t        j                  t        j                  j
                  ||      | fi |S rT   r;  )r   r?  r9  rr   s       r1   randint_like_lowrB    s/     i''(8(8#tDdUfUUr2   c                 F    t        j                  j                  d| |fi |S r   )r   r>  r?  )r9  rp   rr   s      r1   r>  r>  !  s"     <<AtT4V44r2   c                     t         j                  j                  j                  |      }t         j                  j                  j	                  | |||j                         d         S r   )r=   r&   
_quantized$wrapped_fbgemm_pack_gemm_matrix_fp16!wrapped_fbgemm_linear_fp16_weightrp   )rY   r5   r[   packed_weights       r1   #linear_dynamic_fp16_unpacked_weightrI  *  sO     II((MMfUM99AA}dFKKM!$4 r2   input_scaleinput_zero_pointweight_scaleweight_zero_point	out_scaleout_zero_pointout_channelc
           	          t         j                  j                  j                  ||||      }
t         j                  j                  j	                  | |||
|||	      S rT   )r=   r&   rE  _wrapped_linear_prepack#_wrapped_quantized_linear_prepacked)rY   rJ  rK  r5   rL  rM  r[   rN  rO  rP  rH  s              r1   wrapped_quantized_linearrT  6  s\     II((@@/M 99CC r2   packedc                     dt         j                  dt         j                  fd} || dddf         } || ddd f         }| dd df   j                  t         j                        |z  |z   S )Nu8r'   c                      fddD        \  }}}}t         j                  dk(  r4||dz  z   |dz  z   |dz  z   j                  t        j                        d   S |dz  |dz  z   |dz  z   |z   j                  t        j                        d   S )Nc              3   f   K   | ](  }d |f   j                  t        j                         * yw.N)r   r=   int32)r   nrW  s     r1   r   zPq_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32.<locals>.<genexpr>T  s&     GQbajmmEKK0Gs   .1)r   r   r      little   r      rZ  )sys	byteorderr   r=   float32)rW  rg   r   r  rK   s   `    r1   bitcast_u8_to_f32z=q_embedding_bag_byte_unpack_decomp.<locals>.bitcast_u8_to_f32S  s    G,G
1a==H$aLAG,R8>>u}}MiXX"Wb)Q!V4q8>>u}}MiXXr2   .i)r=   r   r   rc  )rU  rd  scalesoffsetss       r1   "q_embedding_bag_byte_unpack_decomprh  Q  sz    Yell Yu|| Y vc2b5j12FsBCx 01G#ss(u}}-6@@r2   agridinterpolation_modepadding_modealign_cornersc                     | j                   t        j                   d      k(  xr' |dk(  xr  | j                  t        j                         }t	        | |||||      }|S )Nr   r   r7   )rj  rk  rl  rm  _expand_grid)r   r=   is_contiguousr>   decomp_grid_sampler_2d)ri  rj  rk  rl  rm  ro  outputs          r1   grid_sampler_2drs  _  sm     	
ELL'' 	C!#	COO%*A*AOBL $	-!#!F Mr2   start_tensorsend_tensorsc           	          t         j                  j                  | t         j                  j	                  t         j
                  j                  ||       |            S rT   )r   _foreach_addList_foreach_mulScalar_foreach_sub)rt  ru  r5   s      r1   _foreach_lerp_scalarr|    sL     !!  "";>	
 r2   scalarsc           	          t         j                  j                  | t         j                  j	                  t         j
                  j                  ||       |            S rT   )r   rw  rx  ry  
ScalarListr{  )rt  ru  r}  s      r1   _foreach_lerp_scalarlistr    sL     !!$$"";>	
 r2   running_meanrunning_vartrainingexponential_average_factorepsilonc           
          t         j                  | |||||||      \  }}	}
|r||	|
fS ||j                  d      |j                  d      fS )Nr   )r   native_batch_norm	new_zeros)rY   r5   r[   r  r  r  r  r  ri  bcs              r1   miopen_batch_normr    sl     $$"	GAq! 1ay	 r2   c                      i t         t        S rT   )r+   r   re   r2   r1   fast_random_decompsr    s    5n5 455r2   c                      t         j                  rt        S t         j                  rHt        j	                  t
        j                  j                  j                  j                  d       t        S t               } | S )z"decomps can change based on configN)r   fallback_randomr+   "fallback_embedding_bag_byte_unpackpopr=   r&   	quantizedembedding_bag_byte_unpackr   r  )r  s    r1   select_decomp_tabler    sT    00599..HHPPRVW "FMr2   rI   sourcec                    ddl m}m}  || j                  |j                        rt
        j                  | |g      \  } }|j                  d      j                  d      dz
  }d | ||fD        \  }}}t
        j                  |||gd      }	t        j                  ||	|      j                  | j                        S t        S )Nr   )BackendFeaturehas_backend_featurer   r   c              3   <   K   | ]  }|j                           y wrT   )r   )r   rg   s     r1   r   z!masked_scatter.<locals>.<genexpr>  s     ,WQQYY[,Ws   )codegen.commonr  r  r   MASKED_SCATTER_WITH_INDEXr   broadcast_tensorsr   cumsum_unsafe_masked_indexr=   rG   r   r   rV   )
r   rI   r  r  r  
source_idx	self_flat	mask_flatsource_flatr  s
             r1   masked_scatterr    s     D4;;(P(PQ ++T4L9
d\\"%,,Q/!3
,W4vBV,W)	9k**;	J<QRS{{9fi8==djjIIr2   	quant_min	quant_maxc                    t        j                  |       \  }}||z
  t        ||z
        z  }t        j                  |t        j                  |g            }|t        j
                  ||z        j                  t         j                        z
  }t        j                  |||      }|j                  t         j                        |j                  t         j                        fS rT   )r=   aminmaxr   rb   r   r   r   intrk   float64int64)	rY   r  r  r\   rt   min_valmax_valscale
zero_points	            r1   choose_qparams_tensorr    s     }}U+GWw%	I(=">>EIIeU\\3%01EU[[59<<UYYGGJZI>J88EMM"JMM%++$>>>r2   
accumulatec                     | j                         }t        j                  ||g|j                  |j                        |      }|j                  | j                        S rT   )r   r=   	index_putr   r   )r   ry   r  r  	flatteneds        r1   putr    sL     IE7FNN5;;7I TZZ((r2   c                 V    t         j                  | |||      }| j                  |      S )N)r  )r   r  copy_)r   ry   r  r  r   s        r1   put_r    s(     ((4:(
>C::c?r2   rr  input_dtypec                     | |z  }t        j                  ||d      }t        j                  | ||      }| j                  |k7  r|j                  |      }|j                         S )NTr   )r=   r   r   fmart   r   r?   )rN   rr  rx   r  new_grad_outputsum_new_grad
grad_inputs          r1   _softmax_backward_datar    sc     "F*O99_#tDL##VG\?KJ K']];/
  ""r2   Tinclude_selfsrcreduction_typer  c                   |dk(  rt        | j                        s| j                  j                  xs | j                  j                  }t	        j
                  |      }|r)| }t	        j
                  |       j                  |||      }	nN| j                  ||d      }t	        j                  |       j                  |||      }	|	j                  |	dk  d      }	|j                  |||      }|r||	z  S ||	z  S t        t        j                  j                  || j                  |j                  |j                  j                  d      rt         S | j"                  |dz   d  j%                         | j"                  d | j%                         z  }
|j%                         g| j"                  |dz   d  | j"                  d | }g t'        | j(                  |z
  | j(                        dt'        d| j(                  |z
        }|j+                  t        j,                        j/                  |
      j1                  |      j3                  |      }| j5                  |||||      S )Nmeanr   r   Tr  )r    rt   is_floating_pointr   r=   	ones_liker~   
index_fill
zeros_likemasked_fillr!   r   scatter_reduce_twor   rv   rV   r   r   r   r   r   r  repeat_interleaver   r   scatter_reduce)r   rx   ry   r  r  r  true_divisiononesr   countsrepeatsindex_shaper   scatter_indexs                 r1   index_reducer  "  s    (T

) 

44M

8M8Ms#C__T*44S%FF//#ua0C%%d+55c5$GF''
A6FmmC,,sV|?#-?  

		

  jjq#))+djj#.>.D.D.FFG;;=L4::cAgi#8L4::ds;KLKNU499s?DII.NNE!TYY_4MND		7	#				  !   r2   kernel_size	ceil_modec                 2   |dk(  rdg|z  }|dk(  rdg|z  }|s|}t        ||      }t        ||      }t        ||      }t        ||      }t        j                  t        j                  |      }t
        j                  j                  j                  ||      s0|t        j                  t
        j                        j                  kD  rt        S t        j                  | |||||      \  }}	t        j                  |	|| j                   | d  |||      }
||
fS )Nr   r   )n_dim)r   r<  reduceoperatormulr=   	_inductorlowering%should_fallback_max_pool_with_indicesiinfoint8rb   rV   prims!_low_memory_max_pool_with_offsets'_low_memory_max_pool_offsets_to_indicesr   )rg   r  r   r   r   r  rx   window_sizevalsrg  rO   s              r1   _max_pool_with_indicesr  X  s"    1}39!|#) {C0KHc*H7C(G&#&F""8<<=K 	  FFs 	G 	
 UZZ0444;;	MD' ;;	G =r2   c           	      &    t        | |||||d      S )Nr   r   r  rg   r  r   r   r   r  s         r1   max_pool2d_with_indicesr          "	;9! r2   c           	      &    t        | |||||d      S )Nr]  r   r  r  s         r1   max_pool3d_with_indicesr    r  r2   output_sizec                     | j                   ^ }}}|\  }}|dk(  s|dk(  r:g |||}| j                  |      | j                  |t        j                        fS ||z  dk(  r(||z  dk(  r ||z  ||z  g}t        j                  | |      S t        S )Nr   r   )r   	new_emptyr=   r  r   r  rV   )	rg   r  batchh_inw_inh_outw_outo_sizer  s	            r1   adaptive_max_pool2dr    s     UD$LE5zUaZ'5'%''{{6"AKKekkK$JJJe|qTE\Q.u}dem4++A{;;r2   	out_int32rightsidesortersorted_sequencer  r  r  r  c                ~    t         j                  | t        j                  |g| j                        ||||      d   S )Nr   r  r   )r   searchsortedr=   rz   r   )r  r   r  r  r  r  s         r1   searchsorted_scalarr    sK     dVO$:$:;   	 	r2   r  r  
boundariesc                    t         j                  t        j                  | g|j                        |||      j                  d      S )Nr   r  r   )r   	bucketizer=   rz   r   r   )r   r  r  r  s       r1   bucketize_scalarr    sD     >>dVJ$5$56	  
 gajr2   noiselowerupper	generatorc                    |rS| dk  }t         j                  | |||      }t        j                  || |z  |       }t        j                  ||d      }	||	fS ||z   dz  }
t         j	                  | |
      t        j
                         fS )Nr   )r  r   r   )r   uniformr=   rG   
leaky_relur   )r   r  r	  r
  r  r  not_positiverrr  	noise_outnegative_slopes              r1   rrelu_with_noise_functionalr    s     qyLLueyLA\4!8T:KKa3	y  %-1,t^4ellnDDr2   repeatc                 N   t         j                  j                  rt        S |t	        |      t
        urt        S | j                  j                  dk(  rt        S | j                  t        j                  t        j                  fv sJ | j                  dk(  sJ | j                  d      }t        j                  || j                        }t        j                  ||| j                  t        j                  k(  d      }t        j                  || j!                  d      dz
        S )Nr   r   r   r   Tr  )rb   )r   tritonautotune_at_compile_timerV   rv   r  r   rt   r=   r[  r  r   r  aranger  rk   rp   )r  r  r  posrO   s        r1   repeat_interleave_Tensorr    s    
 }}-- d;/s:}}U"<<EKK5555;;!]]1F
,,{6==
9C   ;DG ;;wFKKNQ$677r2   r  c           	      8   | j                         dk(  r|j                         dk(  sJ d       |d   }|d   }|d   }| j                  d      }|j                  d      }t        j                  j	                  ||||df|df|df|      }	|	j                  d      S )Nr]  z,Expect (N,C_in,L) and (C_out,C_in//groups,K)r   r   r   )r   r   r   r   )rx   r   r   conv2dr   r   )
rY   r5   r[   r   r   r   r   input_2d	weight_2dout_2ds
             r1   conv1d_to_conv2dr     s     99;!

 1 61
 AYFajG{H r"H  $I [[  {!A ! F >>"r2   )NNrT   )Nr   r   )NF)r   r   F)F)Nr   r   F)g      ?gUUUUUU?FN)Nr   r  r!  r   )r<  loggingr   r  ra  collections.abcr   typingr   r   r   typing_extensionsr   r=   torch._decomp_decompr.   torch._prims_common_prims_commonr$  $torch.ao.quantization.fx._decomposedr   r	   r
   torch._decomp.decompositionsr   rq  r   r   rW   r   r   $torch._decomp.decompositions_for_rngr   torch._dynamo.utilsr   torch._environmentr   !torch._higher_order_ops.out_dtyper   torch._inductor.utilsr   r   r   r   r   torch._refsr   r_   %torch.fx.experimental.symbolic_shapesr   r    r   r   r   r    r!   r"   r#   _opsOperatorBaseOpOverloadPacketr$   __annotations__	getLogger__name__r,   r&   r   r  r  rE  quantized_decomposed_adaptive_avg_pool2d_backwardindex_selectaddmvr  bitwise_and_bitwise_or_
clamp_min_distelu
empty_likeflipgeluhardtanhlcmr  linalg_vector_norm_log_softmax max_pool2d_with_indices_backward_native_batch_norm_legit#_native_batch_norm_legit_functional$_native_batch_norm_legit_no_training_batch_norm_with_update"_batch_norm_with_update_functional_batch_norm_no_updatebatch_norm_backwardr  native_group_normnll_loss2d_backwardpermute_copyrrelu_with_noise_backward_softmaxsin_sqrt__to_copytril_indicestriu_indicesunbind_copyr  upsample_bilinear2dvechann_windowrI  rT  inductor_decompositionsr+   _unsafe_indexr  #_unsafe_masked_index_put_accumulate+_scaled_dot_product_flash_attention_for_cpur   r  rj   ri   r~   gluselect_scatterslice_scatterro   splitr   r   r   unbindbaddbmmrH   addcmul__foreach_addcmulrz  _foreach_addcmul_addcdivaddcdiv__foreach_addcdiv_foreach_addcdiv_lerplerp_r%   r*   
OpOverloadr/   r   rC   rM   r   rX   	ShapeTypetupler`   rf   SymInttypesNumberrk   rw   r   r   r   r   r   r   rt   r   r   r   r   r@   r  r
  detach_r  r  r  r  r  	view_copyr   r"  
StrideTyper'  r.  r,  r(  r   r8   r1  r5  r8  r@  	low_dtyperB  r>  r  rh  rs  _foreach_lerpr|  r  r  r  py_impl_CDispatchKeyAutogradcachedictr  r  r  choose_qparamsrz   r  r  r  r  strr  r  r  r  r  r  r  r  r  	Generatorr  r  r   re   r2   r1   <module>r     s       
 $ * * '   # + 
  F ( ( 7 .  F W $  T]t_#jj55

8S8SS ) Sg!yy~~		II	YY!!
yy55 ,,**,, 	

, 		,
 	, 	, 	, 			, 	, 	, 			, 			, 	, 	, 	,  	!," 	#,$ 	--%,& 	%%',( 	00),* 	11+,, 	$$-,. 	///,0 	""1,2 	  3,4 	5,6 	7,8 	9,: 	  ;,< 	=,> 	&&?,@ 	A,B 			C,D 	

E,F 	G,H 	I,J 	K,L 	M,N 	O,P 	  $$Q,R 	S,T 	55U,V 	++W,. ^ K,.J2IJ
 	,,44<<NNNN!!NNHHIIJJLLHHKKLLLLMM  LLMM  IIJJ=Q D..1L1LLM B n&8 9>	D!12	2>xB (2r6"223> ))*+0 05<< 0 05<< 0 ,0 ))*+1<<1#ll149LL1
\\1 ,1  6678\\  	
  \\ 9" ../
P<<
Poo
P LL4
P ,,
	
P
 

P 5<<u||34
P 0
P ::BBCD &*%)	LL 
		d	" 
		d	"	
 
 E %# &*%)	||				d	"	 
		d	"	 \\		 $ &	 $#ELL #U\\ #  %# $	
sU\\!
"	""	 	 \\		 %	 () !"M||M	M <<M LL	M ;;M \\M *M* ,,4456W
sU\\!
"W#YW W \\	W 7W  2234..<<. LL. S		.
 $s)O. 49_. DIo. . I. . d. 5<<u||34. 5.@ ,,-.G G GU\\ G /G
 
# %)
,,LL {{T! \\	  $> %
 %)  !#
,,#
,,# ,,# {{T!	#
 ++

# ;;# \\#  &#L 	" %)&
,,&LL& {{T!& \\	&  #&Z ))*+ :%,,:	: \\: ,:z %:U\\ :ell : &:& 
#
 (,	@||@||@ ;;$	@
 \\@ $@F ++,-   . DLL12u||   3 EJJ/0Iu|| IELL IU\\ I 1I EJJ/0Iu|| IELL IU\\ I 1I 		" 
,,	t  \\	 # 		" 
,,	t  \\	 # ))*+

,,
	
 
 	

 \\
 ,
 //01)
,,)
sU\\!
") \\) 2) --./$
,,$;;$ \\$ 0$
 
,,
 
5??E,,,-
  '
 !%"&"&).)>)>)3
,,)3e)3 ;;	)3
 LL4)3 LL4)3 )3 )3 &&)3 \\)3 ()3` !%"&).)>)>/c5<<'(/
,,/ ;;	/
 LL4/ &&/ / \\/@ /1EF2ELL 2C 2ELL 2 G2 02FG3U\\ 3S 3U\\ 3 H3  1 1 9 9:<PQTu|| T3 T# T%,, T RT  1 1 ; ;<>RSV
,,V V(+V7:V
\\V TV ,,-5
5
sU\\!
"5 5 \\	5 .5 	EEMMN !%<<LL ,,
 \\	 O 
;;CCD<< ll LL	
 ,, || ,, || LL  \\ E4 		++EEF
Au|| 
A 
A G
A --./  ||
,,  	
  \\  0@ **112
%
ell#
 KK
 
%,,	
 3
 **556
%
ell#
 %++$$%
 
%,,	
 7
 ''(<(<(E(EF../<<LL ,,
 ,,%	
 $  !&  5<<u||34 0 G< 6T#xS'9"9: 6 6	T#xS'9"9: 	 ++,
,,
,, LL \\	 -$ ,;;BBC?<<?? ? 
	?
 ;;? 5<<%&? D? !
 	
)
,,
)<<
) LL
) 	
)
 \\
) "
) 		"
 	
,,<< LL 	
 \\ # 33;;<##LL# 
# 	#
 \\#  =#( ))* 2
,,2	2 <<2 
	2
 2 2 \\2 +2j5||5c5 $s)Od"5 49_	5
 DIo5 5 
5 5<<%&5p 445 &*  !
||
c
 $s)Od"
 49_	

 DIo
 
 5<<%&
 6
 445 &*  !
||
c
 $s)Od"
 49_	

 DIo
 
 5<<%&
 6
 001||"&s)
5<<%& 2" ))001
 "&	\\	
++

	 		
 	 *	 LL4	 \\	 2	& --.
 
++

 	
  \\ / 889 %(,E
,,E<<E E 	E
 E %E 5<<%&E :E& ..556 #8LL8t8 \\8 784 !%)<<)LL) ,,
) #J	)
 3Z) Cj) ) \\)r2   