
    9j_]                   V
   U d Z ddlmZ ddlZddlZddlZddlZddlZddlZddl	Z	ddl
Z
ddlZddlZddlZddlZddlmZmZ ddlmZ ddlmZmZmZmZmZmZ ddlmZ ddlmZmZmZm Z  dd	l!m"Z"m#Z# ddl$Z$ddl%Z$ddl&Z$ddl'm(c m)Z* dd
l+m,Z, ddl-m.Z. ddl/m0Z0 ddl1m2Z2 ddl3m4Z4 ddl5m6Z6m7Z7 ddl8m9Z9 ddl:m;Z;m<Z< ddl=m>Z> ddl?m@Z@ ddlAmBZB ddlCmDZE ddlFmGZGmHZH ddlImJZJ ddlKmLZLmMZM ddlNmOZO ddlPmDZD ddlQmRZR ddlSmTZT  e	j                  eV      ZWe$j                  j                  ZYe$j                  j                  ZZeZ[e[e$j                  j                  z  Z]ej                  j                  dd      Z`daad ebd!<   dacd"ebd#<   dd$Zd G d% d&e      Ze G d' d(e      Zf G d) d*e      Zg e d+      Zhe$j                  j                  j                  ekz  Zl G d, d-      Zm em       Zn	 d	 	 	 	 	 	 	 dd.Zo G d/ d0      Zp G d1 d2eq      Zreperz  Zsdd3Zt G d4 d5      Zu G d6 d7e      Zv G d8 d9ev      Zw G d: d;ev      Zx G d< d=ev      Zy G d> d?ev      Zz G d@ dAev      Z{e|edBf   Z} G dC dDe{      Z~ G dE dFe~      Z G dG dHe~      Z G dI dJe~      Z G dK dLe{      Z G dM dNe      Z G dO dPe      Z G dQ dRe      Z G dS dTev      Z G dU dVev      Z G dW dXev      Z G dY dZ      Z G d[ d\e      Zej                   G d] d^             Zej                   G d_ d`e             Zej                   G da dbe             Zej                   G dc dde             ZddeZddfZ	 d	 	 	 	 	 	 	 	 	 ddhZeddiddgdeRf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddjZ eB       Zdkebdl<   	 	 	 	 	 	 	 	 	 	 	 	 ddmZ ee      j.                  dnz  doz  Zg Zdpebdq<   eddidgeRf	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 ddrZ eEj6                  dgs      ddieRf	 	 	 	 	 	 	 	 	 	 	 	 	 ddt       Z	 	 d	 	 	 	 	 	 	 	 	 	 	 dduZefdgdv	 	 	 	 	 	 	 	 	 ddwZefdgdv	 	 	 	 	 	 	 	 	 ddxZddyZ ejB                  dz      Zdd{Zdd|Zdd}Zdd~ZddZddZddZ G d d      ZddZ	 	 	 	 d	 	 	 	 	 	 	 	 	 	 	 ddZ e$jZ                         deRd	 	 	 	 	 	 	 	 	 dd       Z e$j^                         eRd	 	 	 	 	 	 	 dd       ZddZddZddZddZddZ eB       Zdkebd<   	 d	 	 	 	 	 	 	 ddZddZddZy)a4  
# Inductor Pattern Matcher

The pattern matcher enables search/replace within an FX graph.

The main entrypoint to the pattern matcher is register_replacement(). Given a
search function and a replacement function this will register a replacement with
a pass (such as torch._inductor.fx_passes.joint_graph.patterns).

Internally the pattern matcher represents patterns as a graph (a DAG). Creating
new patterns manually as a graph is cumbersome and error-prone so the standard
way to create patterns (using register_replacement()) is to provide a search
function and a replacement function which is traced and converted into a graph.

Because the search functions are built somewhat generic (they tend to ignore
tensor sizes, for example) register_replacement() allows you to specify an
`extra_check` function which performs additional checks to verify that the
matched pattern fully matches before returning it.

## Precompiled Patterns

New patterns are added using register_replacement(). Patterns added in this way
can have a compile-time overhead because they need to be traced before
use. Patterns can be precompiled and added using gen_register_replacement()
instead. To do this you call gen_register_replacement() instead of
register_replacement(). The arguments are the same except for an additional
unique name which is used as a lookup key.

## Internals

The match DAG is represented by a graph of `PatternExpr` nodes. Each PatternExpr
implements a `_match` method which returns either a `Match` object for a
successful match or a `FailedMatch` object for a failure to match.
    )annotationsN)ABCabstractmethod)defaultdict)Callable
Collection	GeneratorIterableMappingSequence)Path)AnyNoReturnProtocolTypeVar)SelfTypeIs)enable_python_dispatcher)counters)is_integer_dtype)unset_fake_temporarily)make_fx)guard_or_falsestatically_known_true)	_get_attr)immutable_dictimmutable_list)GraphTransformObserver)preserve_node_meta)
OrderedSet   config)aot_functionmake_boxed_func)default_partition)
FakeTensorFakeTensorMode)Transformer   )select_decomp_table)%fallback_node_due_to_unsupported_type#TORCHINDUCTOR_PATTERN_MATCH_BACKENDinductorzbool | OrderedSet[str] | None_debug_nodes_cache
str | None_debug_nodes_env_value_cachec                D    dd} |       }t        |t              r|S | |v S )Nc                     dd} t         j                  j                  d      }|t        k7  st        
 | |      a|at        S )Nc                H    | sy| dk(  ryt        | j                  d            S )NFallT,)r    split)	env_values    _/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/_inductor/pattern_matcher.pyparse_debug_envzE_should_debug_node.<locals>._get_debug_nodes.<locals>.parse_debug_enve   s'    E!iooc233    !TORCHINDUCTOR_PATTERN_MATCH_DEBUG)r8   r0   returnbool | OrderedSet[str])osenvirongetr1   r/   )r:   current_envs     r9   _get_debug_nodesz,_should_debug_node.<locals>._get_debug_nodesb   sD    	4 jjnn%HI 66:L:T!0!=+6(!!r;   )r=   r>   )
isinstancebool)	node_namerC   debug_nodess      r9   _should_debug_noderH   a   s,    "& #$K+t$##r;   c                       e Zd ZU ded<   ddZy)SearchFnstr__name__c                     y N selfargskwargss      r9   __call__zSearchFn.__call__~       r;   NrR   r   rS   r   r=   r   )rL   
__module____qualname____annotations__rT   rO   r;   r9   rJ   rJ   {   s
    M=r;   rJ   c                      e Zd ZddZy)	ReplaceFnc                     y rN   rO   rP   s      r9   rT   zReplaceFn.__call__   rU   r;   NrV   rL   rW   rX   rT   rO   r;   r9   r[   r[      s    =r;   r[   c                  $    e Zd Z	 	 	 	 	 	 	 	 ddZy)TraceFnc                     y rN   rO   )rQ   fnrR   rS   s       r9   rT   zTraceFn.__call__   s    "r;   N)ra   zSearchFn | ReplaceFnrR   r   rS   r   r=   torch.fx.GraphModuler]   rO   r;   r9   r_   r_      s'    #&#/2#>A#	#r;   r_   Tc                      e Zd ZddZy)Multiplec                2    dt               vs| t        u sJ y y )NMULTIPLE)globalsrg   rQ   s    r9   __init__zMultiple.__init__   s    *dh.>>>.>*r;   Nr=   None)rL   rW   rX   rj   rO   r;   r9   re   re      s    ?r;   re   c                t   ddl m}m} t        j                  j
                  dk(  ry| j                  dg       j                         }|j                   ||||j                               | j                  d |j                  j                         D               || d<   n0| j                  d |j                  j                         D               d|j                  v r|j                  d   | d<   d| vr d|j                  v r|j                  d   | d<   d	| vr"d	|j                  v r|j                  d	   | d	<   y y y )
Nr   )
NodeSourceNodeSourceActionr*   	from_nodec              3  v   K   | ]1  \  }}|t         j                  j                  j                  v r||f 3 y wrN   torchfxproxy_COPY_META_FIELDS.0kvs      r9   	<genexpr>z!_transfer_meta.<locals>.<genexpr>   6      
1EHHNN444 F
   79c              3  v   K   | ]1  \  }}|t         j                  j                  j                  v r||f 3 y wrN   rr   rw   s      r9   r{   z!_transfer_meta.<locals>.<genexpr>   r|   r}   stack_tracevaltensor_meta)torch.fx.tracebackrn   ro   r#   traceprovenance_tracking_levelrA   copyappendREPLACEupdatemetaitems)new_metaold_node	pass_namern   ro   new_from_nodes         r9   _transfer_metar      s"    @ ||--2 ["5::<Z)=M=U=UVW 
 ++-
 	

 !. 
 ++-
 	

 %"*--"> H(--!7"--.H$(--)G"*--"> *H$r;   c                       e Zd ZU dZded<   ded<   ded<   ded	<   d
ed<   ded<   ded<   	 	 d	 	 	 	 	 	 	 	 	 d fdZedd       ZddZddZ	ddZ
d dZd!dZd"dZ	 	 	 	 	 	 d#dZ	 	 d$	 	 	 	 	 	 	 	 	 d%dZ xZS )&Matcha  
    Represents a successfully matched pattern.

    The `Match` object is returned to represent a successfully matched
    pattern. Included in the Match are the pattern that was matched, the graph
    nodes matched, and any args that were used during the matching.

    The args and kwargs are specific to the type of pattern that was matched and
    provide hints about what was matched.
    PatternExprpattern	list[Any]rR   dict[str, Any]rS   list[torch.fx.Node]nodesz'dict[_TargetExpr, torch.fx.node.Target]targetsMatchContextctxztorch.fx.GraphModule | Nonereplacement_graphc                    t         |           || _        t        |xs g       | _        |xs i | _        g | _        i | _        || _        d | _	        y rN   )
superrj   r   listrR   rS   r   r   r   r   )rQ   r   r   rR   rS   	__class__s        r9   rj   zMatch.__init__   sO     	$	l
!%r;   c                .    | j                   j                  S rN   )r   graphri   s    r9   r   zMatch.graph   s    xx~~r;   c                2   | j                   rwt        | j                   j                               t        |j                   j                               z  D ]-  }| j                   |   |j                   |   k7  s#t        d|       | j                  j                  |j                         | j                  j                  |j                         | j                   j                  |j                          | j                  j                  |j                         y )Nzkwarg mismatch: {})	rS   r    keysFailedMatchrR   extendr   r   r   )rQ   otherkeys      r9   r   zMatch.extend   s    ;;!$++"2"2"45
5<<CTCTCV8WW A;;s#u||C'88%&:C@@A 			$

%++&5<<(EMM*r;   c                f    | j                   rt        | j                         g| _         | S g | _         | S rN   )rR   tupleri   s    r9   bundlezMatch.bundle   s.    *.))U499%&	 :<	r;   c                <    d| j                    d| j                   dS )NzMatch(..., , )rR   rS   ri   s    r9   __repr__zMatch.__repr__   s    TYYKr$++a88r;   c                    | j                   }t        | j                        D ]-  }|j                  r|j                  r|j                  |       / y rN   )r   reversedr   _erasedusers
erase_node)rQ   r   ns      r9   erase_nodeszMatch.erase_nodes   s>    

$**% 	$A99QWW  #	$r;   c                    | j                   j                  D cg c]  }|| j                   j                  |   nd ! c}S c c}w rN   )r   outputspattern_to_node)rQ   ps     r9   output_nodeszMatch.output_nodes   sF     XX%%
 -.MTXX%%a(tC
 	
 
s   $A c                B    t        d | j                         D              S )Nc              3  &   K   | ]	  }|s|  y wrN   rO   )rx   r   s     r9   r{   z$Match.output_node.<locals>.<genexpr>  s     8!aA8s   )nextr   ri   s    r9   output_nodezMatch.output_node  s    8t002888r;   c                \    t         j                  | | j                  j                  ||       y rN   )ReplacementPatternEntryreplace_with_graphr   r   )rQ   r   rR   s      r9   r   zMatch.replace_with_graph  s$     	 22$((.."3T	
r;   c                6   ddl m}m} t        |j                  |      r|j                  |j                  nt        j                         }dd}|5  |t        j                  t        |      } || j                        rzi | j                  d   j                  d   \  }	}
i |
}
t        | j                        | j                  }}dfd}t        j                   j"                  j%                  |||f|	|
f       t        j&                  j)                  |fd      } |||      }t        j&                  j)                  |d	       } |||      }t+        |j,                  j                        t+        |j,                  j                        k(  sJ t/        |j,                  j                  |j,                  j                        D ]0  \  }}d|j                  v s|j                  d   |j                  d<   2 nt        j&                  j)                  |d
       }t        j0                  j                   j3                  |      /fd}t        j&                  j4                  j7                  ||      } |||      }t+        | j                        dk(  r@|j,                  j                  D ]'  }t9        |j                  | j                  d   d       ) t:        j=                  | | j>                  j,                  ||       ddd       y# 1 sw Y   yxY w)a	  Replace with a graph generated by tracing the replacement_fn.

        Args:
            run_functional_passes (bool). If we should run passes that
                assume functional IR (like DCE, remove_noop_ops), on the
                replacement graph.

        r   )NullHandlerVNc                H   t        |       dk7  ry| d   }d|j                  vry|j                  t        t        j
                  j                  j                  t        j
                  j                  j                  t        j
                  j                  j                  g      v S )Nr*   Fr   eager_input_vals)
lenr   targetr    rs   opshigher_order triton_kernel_wrapper_functionalauto_functionalizedauto_functionalized_v2)r   nodes     r9   !should_propagate_eager_input_valszCMatch.replace_by_example.<locals>.should_propagate_eager_input_vals&  s~    5zQ8D!2;;*II**KKII**>>II**AA#  r;   )run_functional_passesr   c                Z    t        | t        j                  j                        r|| <   y y rN   )rD   rs   rt   Node)r   r   node_to_vals     r9   recordz(Match.replace_by_example.<locals>.recordI  s$    !$6,/D) 7r;   c                    |    S rN   rO   )argr   s    r9   <lambda>z*Match.replace_by_example.<locals>.<lambda>Q  s    +cBR r;   c                     | j                   d   S Nr   r   r   s    r9   r   z*Match.replace_by_example.<locals>.<lambda>W  s    #((5/ r;   c                Z    d| j                   v r| j                   d   S | j                   d   S )Nr   example_valuer   r   s    r9   r   z*Match.replace_by_example.<locals>.<lambda>m  s.    ( !$ /2 r;   c                N    t        | t              rj                  |       S | S rN   )rD   r'   from_tensor)it	fake_modes    r9   _convert_to_fake_modez7Match.replace_by_example.<locals>._convert_to_fake_modet  s(    %b*5):O#,#8#8#<<!	r;   r*   replace_by_exampler   r   r   )r   r   r=   rE   )r   torch.fx.Noder   r   r=   rl   ) torch._inductor.virtualizedr   r   rD   r   
contextlibnullcontext	functoolspartialfwd_onlyr   r   r   rR   rS   rs   utils_pytreetree_maprt   map_argr   r   zip_dynamodetect_fake_moder   map_aggregater   r   r   r   )rQ   replacement_fnrR   trace_fnr   r   r   contextr   	fake_argsfake_kwargs
match_argsmatch_kwargsr   example_valsgraph_with_eager_valsreplacementr   new_noder   r   r   r   s                        @@r9   r   zMatch.replace_by_example  s    	? q{{K8Q[[=P KK'') 		  U	$,,4I 1< !)-A););<N)O&	;-o+0+;T[[L
0 ##,,Z6K8P  %xx//6RS )1(N%  %xx//6QR&'<lK 066<<=%%++B    +.)//55{7H7H7N7N+ &Hh *X]]:<DMM.=&89	  %xx//3  "MM//@@N	("
 $)88==#>#>$&;$L '~|D4::!#$**00 A"!"!%A"6 $66	aU	 U	 U	s   FLD'LLNN)
r   r   r   r   rR   zSequence[Any] | NonerS   zdict[str, Any] | Noner=   rl   )r=   torch.fx.Graph)r   r   r=   rl   )r=   r   r=   rK   rk   )r=   zlist[torch.fx.Node | None])r=   r   )r   r  rR   Sequence[Any]r=   rl   NT)
r   r[   rR   r  r   zTraceFn | Noner   rE   r=   rl   )rL   rW   rX   __doc__rY   rj   propertyr   r   r   r   r   r   r   r   r   __classcell__r   s   @r9   r   r      s   	 
O44	22 &*(,&& & #	&
 && 
&&  +
9$
9
!/
7D
	
 $(&*{!{ { !	{
  ${ 
{r;   r   c                  4    e Zd ZU dZded<   ddZd	dZd
dZy)r   z
    Represents a unsuccessful match.

    The `FailedMatch` object is returned to represent a failure to match a
    pattern.
    rK   format_stringc                f    || _         t        |      dkD  rt        d|       || _        || _        y )N   zUFormat string too long - use lazy construction of strings instead. Format string is
 )r
  r   RuntimeErrorrR   rS   )rQ   r
  rR   rS   s       r9   rj   zFailedMatch.__init__  sB    * }#hivhwx  	r;   c                b     | j                   j                  | j                  i | j                  S rN   )r
  formatrR   rS   ri   s    r9   __str__zFailedMatch.__str__  s(    (t!!(($))Ct{{CCr;   c                     yNFrO   ri   s    r9   __bool__zFailedMatch.__bool__      r;   N)r
  rK   rR   r   rS   r   r=   rl   r  r=   rE   )rL   rW   rX   r  rY   rj   r  r  rO   r;   r9   r   r     s     	Dr;   r   c                    t        |       S )z|
    TypeIs cannot act on `self`. Thus this function exists to let mypy
    recognize FailedMatch.__bool__ as a TypeIs.
    )rE   )ms    r9   is_matchr    s    
 7Nr;   c                  d    e Zd ZU dZded<   ded<   ded<   ded	<   	 d	 	 	 	 	 	 	 ddZddZddZy
)r   zC
    Internal state needed while running PatternExpr._match().
    list[PatternExpr | None]r   z'dict[PatternExpr, torch.fx.Node | None]r   r  r   zlist[NodeOrConstant]exclusive_node_setNc               V    || _         |i n
t        |      | _        || _        g | _        y rN   )r   dictr   r   r  )rQ   r   r   r   s       r9   rj   zMatchContext.__init__  s.     %4%<r$BW
"$r;   c                    || j                   v r)| j                   |   |k(  rt        | |      S t        d      S |j                  ||       }|| j                   vsJ |r|nd| j                   |<   |S )z)wrapper to check reused nodes in patternszrepeated pattern differsN)r   r   r   _match)rQ   r   r   r  s       r9   matchzMatchContext.match  sy    d***##G,4T7++"#=>>NN4&d2222201tW%r;   c                    | j                   j                         D ci c]  \  }}|j                         r||| c}}S c c}}w rN   )r   r   has_multiple_users)rQ   r   r   s      r9   filter_multi_user_patternsz'MatchContext.filter_multi_user_patterns  sL     "&!5!5!;!;!=
))+0@ TM
 	
 
s   ArN   )r   r  r   z'dict[PatternExpr, torch.fx.Node] | Noner   r  r=   rl   )r   r   r   NodeOrConstantr=   MatchResult)r=   z dict[PatternExpr, torch.fx.Node])rL   rW   rX   r  rY   rj   r   r#  rO   r;   r9   r   r     s^     &%<<,,
 DH
%)
% A
%
 
% 

%

r;   r   c                  V    e Zd ZdZed	d       Zd
dZddZddZ	 	 	 	 	 	 ddZ	ddZ
y)r   z+
    Base class for types of patterns.
    c                     y rN   rO   rQ   r   r   s      r9   r  zPatternExpr._match  s    MPr;   c                    	 t        | g|j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wNr   )r   r   r   r   rQ   r   es      r9   r   zPatternExpr.match  s<    	djj9??dKK 	H	s   '* 	?:??c                     yr  rO   ri   s    r9   r"  zPatternExpr.has_multiple_users  r  r;   c                4    | j                   j                  dz   S )Nz())r   rL   ri   s    r9   r   zPatternExpr.__repr__  s    ~~&&--r;   c              #  L   K   | |j                   v r|j                   |     y y wrN   )r   rQ   r   searcheds      r9   find_anchor_nodeszPatternExpr.find_anchor_nodes  s,      3&&&%%d++ 's   "$c                .    t        || j                        S )z
        Compare two `PatternExpr`s and return true if they are the
        same. Note this is NOT matching a pattern - it is comparing the pattern
        structures (for debugging).
        )rD   r   )rQ   r   s     r9   
pattern_eqzPatternExpr.pattern_eq  s     %00r;   Nr   r   r   r   r=   r%  r   r   r=   r%  r  r  r   r   r2  OrderedSet[torch.fx.Node]r=   z+Generator[torch.fx.Node | None, None, None]r   r   r=   rE   )rL   rW   rX   r  r   r  r   r"  r   r3  r5  rO   r;   r9   r   r     sH     P P.,,+D,	4,1r;   r   c                      e Zd ZdZddZy)Argzn
    Capture an arg which will become an input to the handler.  Args are
    passed in depth first order.
    c                     t        || |g      S )N)rR   r   r(  s      r9   r  z
Arg._match  s    S$dV,,r;   Nr   r$  r   r   r=   r%  rL   rW   rX   r  r  rO   r;   r9   r<  r<    s    
-r;   r<  c                  (    e Zd ZdZddZddZddZy)	Ignoredz4
    Match an arg, but don't pass it to handler
    c                    t        ||       S rN   r>  r(  s      r9   r  zIgnored._match  s    S$r;   c                     y)N*rO   ri   s    r9   r   zIgnored.__repr__  s    r;   c                     y)Nz	Ignored()rO   )rQ   pps     r9   pretty_printzIgnored.pretty_print  s    r;   Nr?  r  rG  PatternPrettyPrinterr=   rK   )rL   rW   rX   r  r  r   rH  rO   r;   r9   rB  rB    s     r;   rB  c                  @     e Zd ZdZd fdZddZddZd	 fdZ xZS )

KeywordArgD
    Capture a kwarg which will become an input to the handler.
    c                0    t         |           || _        y rN   r   rj   namerQ   rP  r   s     r9   rj   zKeywordArg.__init__      	r;   c                "    d| j                   dS )NzKeywordArg(r   rP  ri   s    r9   r   zKeywordArg.__repr__#  s    TYYM++r;   c                6    t        || | j                  |i      S )NrS   )r   rP  r(  s      r9   r  zKeywordArg._match&  s    S$		4'899r;   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S rN   typingcastr   r   r5  rP  rQ   r   r   s     r9   r5  zKeywordArg.pattern_eq)  5    D%(w!%(DTYY%**-DDr;   rP  rK   r=   rl   r  r?  r:  	rL   rW   rX   r  rj   r   r  r5  r  r  s   @r9   rL  rL    s#    ,:E Er;   rL  c                  L     e Zd ZU dZded<   d fdZd	dZd
dZd fdZ xZ	S )ExclusiveKeywordArgrM  rK   rP  c                0    t         |           || _        y rN   rO  rQ  s     r9   rj   zExclusiveKeywordArg.__init__5  rR  r;   c                "    d| j                   dS )NzExclusiveKeywordArg(r   rT  ri   s    r9   r   zExclusiveKeywordArg.__repr__9  s    %dii]!44r;   c                    ||j                   v rt        d      S |j                   j                  |       t        || | j                  |i      S )Nzexclusive arg appears twicerV  )r  r   r   r   rP  r(  s      r9   r  zExclusiveKeywordArg._match<  sH    3)))<==%%d+S$		4'899r;   c                    t        j                  t        |      }t        |   |      xr | j
                  |j
                  k(  S rN   rX  r[  s     r9   r5  zExclusiveKeywordArg.pattern_eqC  r\  r;   r]  r  r?  r:  )
rL   rW   rX   r  rY   rj   r   r  r5  r  r  s   @r9   r`  r`  .  s*     I5:E Er;   r`  c                       e Zd ZU dZded<   ded<   	 d	 	 	 	 	 d fdZeedd              ZddZ	dd	Z
dd
Z	 	 	 	 	 	 ddZddZddZd fdZ xZS )_TargetExprz7
    Base class for filtering match by node.target
    zlist[FnsType]fnszOrderedSet[FnsType]fns_setc                Z   t         |           t        |      st        |t              r|gn
t        |      }|D ]O  t        t        j                  j                        s(|j                  fdj                         D               Q || _        t        |      | _        || _        y )Nc              3  6   K   | ]  }t        |        y wrN   )getattr)rx   overloadra   s     r9   r{   z'_TargetExpr.__init__.<locals>.<genexpr>W  s     PX72x0Ps   )r   rj   callablerD   rK   r   rs   _opsOpOverloadPacketr   	overloadsrg  r    rh  r   )rQ   rg  r   ra   r   s      @r9   rj   z_TargetExpr.__init__P  s     	}
3(<se$s) 	QB"ejj99:

PPP	Q !#
r;   c                     y rN   rO   ri   s    r9   opz_TargetExpr.op]  s    r;   c                   | j                   d   }t        |t              s|j                  }t	        | j                         dkD  rd| dS | j                   d   t        t        |d       u rd| S | j                   d   t        t        |d       u rd| S t        | j                   d   t        j                  j                        rt        | j                   d         S |S )Nr   r*   [z, ...]torch.z	operator.)
rg  rD   rK   rL   r   rk  rs   operatorrn  
OpOverload)rQ   
first_reprs     r9   fns_reprz_TargetExpr.fns_repra  s    XXa[
*c*#,,Jtxx=1zl&))XXa[GE:t<<J<((XXa[GHj$??zl++UZZ%:%:;txx{##r;   c                    | j                   t        u rd}n"| j                   dk7  rd| j                    d}nd}| j                  j                   d| j	                          | dS )Nz
, MULTIPLEr*   r   r    ()r   rg   r   rL   ry  )rQ   comma_userss     r9   r   z_TargetExpr.__repr__q  s^    ::!&KZZ1_tzzl!,KK..))*!DMMO+<[MKKr;   c                X    t        | j                  t              xs | j                  dkD  S )Nr*   )rD   r   re   ri   s    r9   r"  z_TargetExpr.has_multiple_usersz  s     $**h/A4::>Ar;   c                    t         rN   NotImplementedErrorr1  s      r9   r3  z_TargetExpr.find_anchor_nodes}  s
     "!r;   c                    t        |t        j                  j                        xr2 |j                  | j                  k(  xr t        |      | j                  v S rN   )rD   rs   rt   r   rr  extract_targetrh  )rQ   r   s     r9   
_match_fnsz_TargetExpr._match_fns  sD    tUXX]]+ 5477"5t$4	
r;   c                    | |j                   v xs6 | j                  t        u xs" t        |j                        | j                  k(  S rN   )r   r   rg   r   r(  s      r9   _match_usersz_TargetExpr._match_users  s=    CKK -zzX%-4::$**,	
r;   c                    t        j                  t        |      }t        |   |      xrO | j
                  |j
                  k(  xr4 | j                  |j                  k(  xr | j                  |j                  k(  S rN   )rY  rZ  r   r   r5  rr  rg  r   r[  s     r9   r5  z_TargetExpr.pattern_eq  sf    D%(Gu% *588#*EII%* 

ekk)		
r;   )r*   )rg  zFnsType | Sequence[FnsType]r   zMultiple | intr=   rl   r  r  r8  r   r   r=   rE   )r   r   r   r   r=   rE   r:  )rL   rW   rX   r  rY   rj   r  r   rr  ry  r   r"  r3  r  r  r5  r  r  s   @r9   rf  rf  H  s     
   IJ.7E	    LB""+D"	4"



 
r;   rf  .c                       e Zd ZdZdd	 	 	 	 	 	 	 	 	 d fdZe	 	 	 	 	 	 dd       Ze	 	 	 	 	 	 dd       ZddZddZ	dd	Z
	 	 	 	 	 	 dd
Zd fdZ xZS )_TargetArgsExprzE
    Base class for filtering match by node.{target,args,kwargs}
    r*   )_usersc               j   t         |   ||       t        |      | _        t	        |      | _        t        d t        j                  ||j                               D              r| j                  | _        n| j                  | _        | j                  | j                  | j
                        | _        y )Nc              3  R   K   | ]  }t        |t        t        t        f       ! y wrN   )rD   r  r   r   rx   xs     r9   r{   z+_TargetArgsExpr.__init__.<locals>.<genexpr>  s&      
 q4u-.
s   %')r   rj   r   rR   r  rS   any	itertoolschainvaluespytree_flattenflattensimple_flattenflat_args_kwargs)rQ   rg  r  rR   rS   r   s        r9   rj   z_TargetArgsExpr.__init__  s     	f%$K	6l 
__T6==?;
 
  ..DL..DL $TYY Dr;   c                n    g | |j                         }t        |       g|j                         }||fS rN   )r  r   r   )rR   rS   r  specs       r9   r  z_TargetArgsExpr.simple_flatten  s:     +4*&--/*D	*FKKM*t|r;   c                    t         t        t        t        t        t        idfdt        j                  | |ffd      }t        j                  |      \  }}||fS )Nc                    t        |       }j                  |      }|!t        j                   ||       fd      S | S )Nc                    t        |       v S rN   typer  type_mappings    r9   r   zF_TargetArgsExpr.pytree_flatten.<locals>.convert_type.<locals>.<lambda>  s    d1g&= r;   is_leaf)r  rA   pytreer   )r  cls
convert_fnconvert_typer  s      r9   r  z4_TargetArgsExpr.pytree_flatten.<locals>.convert_type  sG    q'C%))#.J% qM= 
 Hr;   c                    t        |       v S rN   r  r  s    r9   r   z0_TargetArgsExpr.pytree_flatten.<locals>.<lambda>  s    d1g5 r;   r  )r  r   r=   r   )r   r   r   r   r  r  r   tree_flatten)rR   rS   normalized_args_treeflatr  r  r  s        @@r9   r  z_TargetArgsExpr.pytree_flatten  s_    
 E%D*
		  &6N5 

 (()=>
dTzr;   c                   | j                         gt        t        | j                        | j                  j                         D cg c]  \  }}| d|  c}}}| j                  t        u r|j                  d       n-| j                  dk7  r|j                  d| j                          | j                  j                   ddj                  |       dS c c}}w )N=_users=MULTIPLEr*   _users=r|  r   r   )ry  mapreprrR   rS   r   r   rg   r   r   rL   join)rQ   ry   rz   rR   s       r9   r   z_TargetArgsExpr.__repr__  s    MMO
tyy!
 &*[[%6%6%89TQ1QCj9

 ::!KK)*ZZ1_KK'$**./..))*!DIIdO+<A>> :s   Cc           
        | j                         gfd| j                  D        | j                  j                         D cg c]  \  }}| dj	                  |        c}}}| j
                  t        u r|j                  d       n-| j
                  dk7  r|j                  d| j
                          d}| j                  j                   d|j                  |       dS c c}}w )	Nc              3  @   K   | ]  }j                  |        y wrN   )rH  )rx   r  rG  s     r9   r{   z/_TargetArgsExpr.pretty_print.<locals>.<genexpr>  s     4Qbooa 4s   r  r  r*   r  r   r|  r   )ry  rR   rS   r   rH  r   rg   r   r   rL   r  )rQ   rG  ry   rz   rR   
joiner_strs    `    r9   rH  z_TargetArgsExpr.pretty_print  s    MMO
4$))4
 7;kk6G6G6IJda1R__Q'()J

 ::!KK)*ZZ1_KK'$**./
..))*!JOOD,A+B!DD Ks    C'c                t   | j                  |      r+t        |j                        t        | j                        k7  rt        d||       S | j	                  ||      st        d|       S |j                  }|j
                  }t        |      t        | j
                        k  rddlm} t        |j                        sJ  ||j                  |j                  |j
                        }|t        d||       S |\  }}t        |      t        | j                        k(  rDt        |      t        | j
                        k\  r#|D ci c]  }|| j
                  v s|||    }}n/t        d||       S |D ci c]  }|| j
                  v s|||    }}| j                  ||      \  }}	| j                  \  }
}|	|k7  rt        d|	|      S t        |      t        |
      k(  sJ t        ||       }t        |
|      D ]  \  }}t        |t              r3|j!                  ||      }t#        |      s|c S |j%                  |       It        |t&        j(                  j*                        s||k7  sst        d|||      c S  |j,                  j/                  |       |j                  |j0                  | <   |S c c}w c c}w )Nz&function_mismatch: node={}, pattern={}zmultiple_users {}r   )normalize_functionzargs_structure {} {}z#constant_args: {} {!r}!={pattern!r})r   )r  r   rR   r   r  rS   torch.fx.operator_schemasr  rm  r   r  r  r   r   rD   r   r   r  r   rs   rt   r   r   r   r   )rQ   r   r   _args_kwargsr  normalized_args_and_kwargsi
node_items	node_spec
self_items	self_specr  r   
child_nodechild_matchs                   r9   r  z_TargetArgsExpr._match  sm   t$DII#dii.(HGtTT  s+2D99		++w<#dkk**DDKK((();TYY*& *1"#KTSWXX!;wu:TYY/CLCDT4T6=RdkkAQq'!*}RGR&@$  /6Jdkk9Iq'!*}JGJ $UG <
I $ 5 5
I	!5y)LL:#j/111#t#&z:#> 	GZ';/!ii<,&&%J6*:O"9#	 	 	
t++		$= S Ks    J0J00J5J5c              #    K   | |j                   v r|j                   |     y| j                  d   D ]  }t        |t              s|j	                  ||      D ]d  }t        |t
        j                  j                        s(|j                  D ].  }||vs| j                  |      s| |j                  |       0 f  yw)a  
        This is used when we are matching a pattern with multiple outputs.
        There is a partial match (stored in ctx) and we want to walk
        this pattern to find a connection to an already-matched node.

        Yields candidate nodes that `self._match` might like.
        Nr   )r   r  rD   r   r3  rs   rt   r   r   r  add)rQ   r   r2  r   
other_noder   s         r9   r3  z!_TargetArgsExpr.find_anchor_nodes*  s      3&&&%%d++,,Q/ 		3G';/")";";C"J 3J%j%((--@  * 0 0 3x/#t4&*
 (T 2	33		3s   ACACC&Cc                   t        j                  t        |      }t        |   |      xrW | j
                  d   |j
                  d   k(  xr6 t        d t        | j
                  d   |j
                  d         D              S )Nr*   c              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wrN   rD   r   r5  rx   abs      r9   r{   z-_TargetArgsExpr.pattern_eq.<locals>.<genexpr>H  6      Aq $.a#=Q16I   35r   )rY  rZ  r   r   r5  r  r5   r   r[  s     r9   r5  z_TargetArgsExpr.pattern_eqC  s    D%(Gu% %%a(E,B,B1,EE  5 5a 8%:P:PQR:ST 	
r;   )
rg  z*torch.fx.node.Target | str | Sequence[Any]rR   r   r  zint | MultiplerS   r   r=   rl   )rR   r  rS   zMapping[Any, Any]r=   z3tuple[Sequence[Any], _SimpleSpec | pytree.TreeSpec]r  rI  r6  r8  r:  )rL   rW   rX   r  rj   staticmethodr  r  r   rH  r  r3  r5  r  r  s   @r9   r  r    s     "#	E7E E 	E
 E 
E& %6	<  %6	< 8
?E4l33+D3	432	
 	
r;   r  c                      e Zd ZdZdZy)CallFunctionzR
    Matches a call_function node in the FX graphs: `fns[i](*args, **kwargs)`
    call_functionNrL   rW   rX   r  rr  rO   r;   r9   r  r  O  s     
Br;   r  c                      e Zd ZdZdZy)
CallMethodzW
    Matches a call_method node in the FX graphs: `fns[i].method(*args, **kwargs)`
    call_methodNr  rO   r;   r9   r  r  W       
Br;   r  c                      e Zd ZdZdZy)
CallModulezP
    Matches a call_module node in the FX graphs: `module(*args, **kwargs)`
    call_moduleNr  rO   r;   r9   r  r  _  r  r;   r  c                      e Zd ZdZddZy)_TargetExprVarArgsz[
    Matches a call_function node with any arguments which are passed into the pattern
    c                   | j                  |      st        d      S | j                  ||      st        d      S t        ||       }|j                  j                  |       |j                  |j                  | <   |j                  j                  |j                         |j                  j                  |j                         |S )Nfunction_mismatchmultiple_users)r  r   r  r   r   r   r   r   rR   r   rS   r   )rQ   r   r   r  s       r9   r  z_TargetExprVarArgs._matchl  s    t$233  s+/00#t	t++		$	dii 	$r;   Nr6  r@  rO   r;   r9   r  r  g  s    r;   r  c                      e Zd ZdZy)CallFunctionVarArgsr  NrL   rW   rX   rr  rO   r;   r9   r  r  {  s    	Br;   r  c                      e Zd ZdZy)CallMethodVarArgsr  Nr  rO   r;   r9   r  r        	Br;   r  c                      e Zd ZdZy)CallModuleVarArgsr  Nr  rO   r;   r9   r  r    r  r;   r  c                  B     e Zd ZdZdd fdZddZd	dZd
 fdZ xZS )ListOfz$
    Matches a repeated pattern
    c                b    t         |           t        |t              sJ || _        || _        y rN   )r   rj   rD   r   r   r   )rQ   r   r   r   s      r9   rj   zListOf.__init__  s,    ';///r;   c                N    | j                   j                   d| j                   dS Nr|  r   )r   rL   r   ri   s    r9   r   zListOf.__repr__  $    ..))*!DLL>;;r;   c                (   t        |t        t        f      rt        |      dk(  rt	        d      S t        ||       }|j                         }d}t        |      D ]  \  }}t        |j                  ||j                        }|j                  | j                  |      }	|j                         }t        |	      s| j                  st	        d||	      c S {d}|j                  |	j!                                 |st	        d      S |j!                         S )Nr   non_listFr+  zlist[{}]: {}Tzlist: no_match)rD   r   r   r   r   r   r#  	enumerater   r   r   r   r   r  r   r   r   )
rQ   r   r   r  r   matchedr  r  	child_ctxr  s
             r9   r  zListOf._match  s    $u.#d)q.z**#t 88:&t_ 	+MAz$_J4D4DI $//$,,
CK'BBDOK(||&~q+FFGHH['')*	+ /00xxzr;   c                    t        j                  t        |      }t        |   |      xr@ | j
                  j	                  |j
                        xr | j                  |j                  k(  S rN   )rY  rZ  r   r   r5  r   r   r[  s     r9   r5  zListOf.pattern_eq  sU    D%(Gu% .''6.-	
r;   F)r   r   r   rE   r=   rl   r  )r   r   r   r   r=   r%  r:  r^  r  s   @r9   r  r    s!    <0
 
r;   r  c                  ~     e Zd ZU ded<   d fdZedd       ZddZddZddZ		 	 	 	 	 	 ddZ
dd	Zd fd
Z xZS )MultiOutputPatternr  r   c                    t         |           t        |d   t              sJ t	        d |D              sJ |       t        |      | _        |d   j                  | _        y )Nr   c              3  H   K   | ]  }|d u xs t        |t                y wrN   )rD   r   r  s     r9   r{   z.MultiOutputPattern.__init__.<locals>.<genexpr>  s#     Lq19:
1k ::Ls    ")r   rj   rD   rf  r5   r   r   rr  )rQ   r   r   s     r9   rj   zMultiOutputPattern.__init__  sU    '!*k222LGLLUgULG}!*--r;   c                h    t        j                  t        | j                  d         }|j                  S Nr   )rY  rZ  rf  r   rg  )rQ   outputs     r9   rg  zMultiOutputPattern.fns  s&     [$,,q/:zzr;   c                N    | j                   j                   d| j                   dS r  )r   rL   r   ri   s    r9   r   zMultiOutputPattern.__repr__  r  r;   c                    | j                   D cg c]  }|j                  |       }}dd }| j                  j                   d|j	                  |       }| d}|S c c}w )Nz,
z  z([z
]))r   rH  r   rL   r  )rQ   rG  r  rR   r  str_outs         r9   rH  zMultiOutputPattern.pretty_print  sh    ,0LL9q"994&\
^^,,-R
0E/FGIT"	 :s   Ac                $   t        j                  t        | j                  d         }|j	                  ||      }t        |      s|S | j                  dd  D ]7  }|| j                  ||      }t        |      s|c S |j                  |       9 |S )Nr   r*   )rY  rZ  rf  r   r   r  _match_from_anchorsr   )rQ   r   r   r  r  r   r  s          r9   r  zMultiOutputPattern._match  s    [$,,q/:IIfd#{H||AB' 	"G227C@KK(""HH[!	" r;   c                    t        |j                        }t        d      }|j                  |t	                     D ]3  }|j                  ||      }t        |      r|c S t        |      |_        5 |S )Nzno anchor found)r  r   r   r3  r    r   r  )rQ   r   r   priorr  r   s         r9   r  z&MultiOutputPattern._match_from_anchors  sl     S(()$%67--c:<@ 	.D		'4(A{"&u+C	. r;   c                    	 t        | j                  |j                        j                  | |      S # t        $ r}|cY d }~S d }~ww xY wr*  )r   r   r   r   r   r,  s      r9   r   zMultiOutputPattern.match  s>    	DJJ?EEdDQQ 	H	s   03 	AAAAc                   t        j                  t        |      }t        |   |      xr] t        | j                        t        |j                        k(  xr0 t        d t        | j                  |j                        D              S )Nc              3  n   K   | ]-  \  }}t        |t              r|j                  |      n||k(   / y wrN   r  r  s      r9   r{   z0MultiOutputPattern.pattern_eq.<locals>.<genexpr>  r  r  )	rY  rZ  r   r   r5  r   r   r5   r   r[  s     r9   r5  zMultiOutputPattern.pattern_eq  sm    D%(Gu% DLL!S%77 emm< 	
r;   )r   zSequence[PatternExpr | None]r=   rl   )r=   z(Callable[..., Any] | str | Sequence[Any]r  rI  r6  )r   r   r   r   r=   r%  r7  r:  )rL   rW   rX   rY   rj   r  rg  r   rH  r  r  r   r5  r  r  s   @r9   r  r    s]    %%   
< ")5		
 	
r;   r  c                  J     e Zd ZdZd fdZedd       ZddZd	 fdZ xZ	S )
RepeatedExprzp
    Checks for a repeated pattern. Useful for repeated operations after a node such as `split` or `unbind`
    c                R    t         |           || _        |j                  | _        y rN   )r   rj   inner_patternrr  )rQ   r  r   s     r9   rj   zRepeatedExpr.__init__  s#    *""r;   c                .    | j                   j                  S rN   )r  rg  ri   s    r9   rg  zRepeatedExpr.fns	  s    !!%%%r;   c                   |j                  | j                  |      }t        |      s|S |j                  j	                  | j                         | j                  j                  |t        |g            D ]T  }t        | g|j                        j                  | j                  |      }t        |      s|c S |j                  |       V |S r*  )
r   r  r  r   popr3  r    r   r   r   )rQ   r   r   r  anchor_nodeanchor_ms         r9   r  zRepeatedExpr._match  s    IId(($/{H	
  --??TF#
 	K $TF$**=CC""KH H%HHX	 r;   c                    t        j                  t        |      }t        |   |      xr% | j
                  j	                  |j
                        S rN   )rY  rZ  r   r   r5  r  r[  s     r9   r5  zRepeatedExpr.pattern_eq   sF    D%(w!%( 
T-?-?-J-J.
 	
r;   )r  rf  r=   rl   )r=   zSequence[FnsType]r6  r:  )
rL   rW   rX   r  rj   r  rg  r  r5  r  r  s   @r9   r  r    s0    #
 & &&
 
r;   r  c                  Z    e Zd ZdZddZeej                  dd	d              Zd
dZ	ddZ
y)rJ  z
    Serializes Patterns to executable python.
    XXX: currently only used and tested for fuse attention patterns. May not cover
    all patterns.
    c                z    t         j                  j                  j                         | _        i | _        i | _        y rN   )rs   rt   r   
_Namespace	namespacememoized_objs_namesmemoized_objs_ppri   s    r9   rj   zPatternPrettyPrinter.__init__.  s*    224;= 8:r;   c                   t               }t        | d      sJ | j                  |      }|j                  D cg c]#  }|j                  |    d|j                  |    % }}|j                  | d|        dj                  |      S c c}w )zU
        Serializes obj to python code with obj written out to `output_name`
        rH  )rG  z = 
)rJ  hasattrrH  r  r  r   r  )objoutput_namerG  out_strr   r  s         r9   runzPatternPrettyPrinter.run3  s     "#sN+++""b") --
 %%c*+3r/B/B3/G.HI
 

 	S	23yy  
s   (B
c                    t        |t              r0| j                  j                  |      x}r|S | j	                  |      S t        |d      r|j                  |       S t        |      S )NrH  )rD   r  r  rA   memoizer  rH  r  )rQ   r  memoized_names      r9   rH  z!PatternPrettyPrinter.pretty_printG  sb    c?+ $ 8 8 < <S AA}A$$||C((3'##D))Cyr;   c                    |j                  |       }|j                         }dD ]  }|j                  |d      } | j                  j	                  |d       }|| j
                  |<   || j                  |<   |S )N)zaten.ru  zprims.r{  )rH  ry  replacer  create_namer  r  )rQ   r  obj_strobj_nameprefixtmp_names         r9   r  zPatternPrettyPrinter.memoizeR  s{    ""4(<<>3 	4F''3H	4 >>--h=(0  %%,c"r;   Nrk   )r  )r  r   r  rK   r=   rK   )r  r   r=   rK   )r  r  r=   rK   )rL   rW   rX   r  rj   r  r   cacher  rH  r  rO   r;   r9   rJ  rJ  '  s6    ;
 __!  !$		r;   rJ  c                      e Zd Z	 	 	 	 ddZy)_PassDictsTypec                     y rN   rO   )rQ   ry   s     r9   __getitem__z_PassDictsType.__getitem___  s     r;   N)ry    tuple[str, torch.fx.node.Target]r=   list[PatternEntry])rL   rW   rX   r)  rO   r;   r9   r'  r'  ^  s    !1!	!r;   r'  c                  F    e Zd ZU ded<   ded<   ddZ	 	 d		 	 	 	 	 	 	 d
dZy)PatternEntryr   r   Callable[[Match], bool]extra_checkc                    t         rN   r  rQ   r   r   r   s       r9   applyzPatternEntry.applyi  s    !!r;   Nc                   |Ht        | j                  d      sJ | j                  j                  D ]  }| j                  |||        y t	        |t
        t        f      rqt        | j                  d      sJ |r,|| j                  j                  |f   j                  d|        y || j                  j                  |f   j                  |        y t        j                  t        t           |      }|D ]  }| j                  |||        y )Nrg  prependrr  r   )r  r   rg  registerrD   r  PatternMatcherPassrr  insertr   rY  rZ  r   r'  )rQ   
pass_dictsr   r5  ra   r  s         r9   r6  zPatternEntry.registerl  s     >4<<///ll&& ?j"g>?
T+=$>?4<<...DLLOOV45<<QEDLLOOV45<<TBXn%=zJJ :a9:r;   r   r   r   r  r   r   r=   rl   r  )r9  )_PassDictsType | Sequence[_PassDictsType]r   ztorch.fx.node.Target | Noner5  rE   r=   rl   )rL   rW   rX   rY   r2  r6  rO   r;   r9   r-  r-  d  sF    ((" /3	:=: ,: 	:
 
:r;   r-  c                       e Zd ZU ded<   ddZy)LoweringPatternEntryCallable[..., Any]handlerc                    t        j                  | j                        t        j                  | j                  |            }|j	                  |      5  |j                  |t        |j                        |j                        }t        |j                  |       |j                  |       d d d        |j                  d   |u sJ |j                          y # 1 sw Y   -xY w)N)r   wrapsr?  r   inserting_beforer  r   rR   rS   r   r   replace_all_uses_withr   r   )rQ   r   r   r   r?  r   s         r9   r2  zLoweringPatternEntry.apply  s    /)//$,,/	0A0A$,,PU0VW##D) 	4--guUZZ7H%,,WK;++T2&&{3	4 {{2$&&&	4 	4s   ACC"Nr:  )rL   rW   rX   rY   r2  rO   r;   r9   r=  r=    s    r;   r=  c                  $    e Zd ZU dZded<   ddZy)GraphPatternEntryz8
    A pattern that runs a function on the FX graph
    r>  r?  c                    |j                  |      5   | j                  |g|j                  i |j                   d d d        y # 1 sw Y   y xY wrN   )rC  r?  rR   rS   r1  s       r9   r2  zGraphPatternEntry.apply  sE    ##D) 	=DLL<<u||<	= 	= 	=s   +AANr:  )rL   rW   rX   r  rY   r2  rO   r;   r9   rF  rF    s      =r;   rF  c                  ^    e Zd ZU dZded<   dZded<   e	 d		 	 	 	 	 	 	 	 	 	 	 d
d       ZddZy)r   z/
    The replacement pattern for the graph
    zCallable[..., list[Any]]normalize_argsNr0   pattern_namec           	        g  G fddt         j                  j                        }| j                         }t	        |      dk(  r|d   }n|d   sJ t        |d   j                  j                        }|D 	cg c]9  }	t        |	t         j                  j                        r|j                  |	      |	f; }
}	t        |
t        j                  d            d   }	 	 	 	 	 	 	 	 	 	 d
dj                  |      5  t        |t         j                  j                        sJ   ||      j                    }t        |t         j                  j                        r|g}dd	 	 	 	 	 	 dfdt	        |      t	        |      k(  rt#        ||      D ]  \  }} ||        nt	        |      dk(  sJ  |d   |       d	d	d	       | j%                          t'              D ]`  }|j(                  r|j+                         r!t        |j,                  t         j.                  j0                        rPj3                  |       b y	c c}	w # 1 sw Y   xY w)zT
        Inserts the replacement graph into the toplevel graph at the match
        c                  4     e Zd ZdZdZdZd fdZ xZS )<ReplacementPatternEntry.replace_with_graph.<locals>.ReplacerNc                   |j                   dv rt        | 	  |      S |j                  }| j	                  |      \  }}|j                   dk(  ryt        |      sJ j                  |||      }j                  |       t        |j                  |xs d       d|j                  v r|j                  d   |j                  d<   |S |j                   dk(  rddl
m} t        | 1  |||      }t        |t        j                  j                         st#        d	| d
| d      j$                  J d }j$                  j'                         D ]  \  }	}
||
u s|	} n |Dt        |t(              sJ  |j$                  |      \  }}j$                  j+                  ||       j                  |      }j                  |       |S t#        d|       )N)placeholderr  r  r{  r   r   get_attrr   )unique_graph_name_with_rootzNYI: replacement_graph.z is not a graph module. Got .z
unhandled )rr  r   run_noder   fetch_args_kwargs_from_envrm  r  r   r   r   torch._higher_order_ops.utilsrQ  rP  rD   rs   rt   GraphModuler  owning_modulenamed_modulesrK   register_module)rQ   r   r   rR   rS   resultrQ  sub_gm
graph_namer   mod_getattr_noder   added_replacement_nodesr   r   s                r9   rS  zEReplacementPatternEntry.replace_with_graph.<locals>.Replacer.run_node  s   7777 7+D11#>>tDf77o-#F+++"00vFF+226:"!'!%"+/r *TYY6:>))DV:W$67!M77j( #W-fdFCF%fehh.B.BC15fX=YZ`Yaabc  !..:::!%J"'"5"5"C"C"E "3!S=)*J!" "))&#666(C!//"): ++;;JO#(>>*#=L+22<@'')Jtf*=>>r;   r   r   r=   r   )rL   rW   rX   r  r  rP  rS  r  )r   r`  r   r   s   @r9   ReplacerrM    s    KKH5? 5?r;   rb  r*   r   )r   c                   | g}t        t        j                  j                            }|rc|j	                         }||vrK||vrGt        |d      r;|j                  |       ||j                  |<   |j                  |j                         |rby y )Nr   )
r    rs   rt   r   r
  r  r  r   r   all_input_nodes)r   tag_name	tag_valueinput_stopsqueuevisitedr   s          r9   percolate_tagszBReplacementPatternEntry.replace_with_graph.<locals>.percolate_tags  sy     FE /1Giikw&;.V,KK$)2CHHX&LL!4!45 r;   c                    | j                   dk7  ry | j                  t        j                  k7  ry t	        | j
                        dk(  sJ | j
                  d   S )Nr  r!   r*   )rr  r   rv  getitemr   rR   r   s    r9   maybe_getitemzAReplacementPatternEntry.replace_with_graph.<locals>.maybe_getitem  sJ    77o-;;("2"22499~***yy|#r;   c           	        dfd}| |J y t        | t        j                  j                        sJ |=| j	                  d |       t        | j                        dk(  r	j                  |        y t        |t        j                  j                        rt        |j                  | xs d       dD ]2  }|| j                  v s ||| j                  |   t                     4 | j	                  ||       t        | j                        dk(  r	j                  |        y t        | j                  j                               }|D ]*  } 
|      }|| j	                  |        |||          , 	j                  |        y )Nc                    | vS rN   rO   )r   r`  s    r9   !filter_nodes_in_newly_added_nodeszfReplacementPatternEntry.replace_with_graph.<locals>.replace.<locals>.filter_nodes_in_newly_added_nodes$  s      '>>>r;   )delete_user_cbr   r{  r   )	recomputeac_graph_idr  )rD   rs   rt   r   rD  r   r   r   r   r   r    r   r   )oldnewrq  re  old_usesuseridxr`  rR   r   rn  r   rj  r  s          r9   r  z;ReplacementPatternEntry.replace_with_graph.<locals>.replace   sy   ?
 ;;&;!#uxx}}555;--'H .  399~*((-c588==1"388SIOL %A #sxx/* #Xsxx/A:dCS --,M .  399~*((-:  		 01$ 0D'-C{ 11#6c#h/0   %r;   N)
r   r   re  rK   rf  rK   rg  r9  r=   rl   ra  )rv  ztorch.fx.Node | Nonerw  z.torch.fx.Node | Sequence[torch.fx.Node] | Noner=   rl   )rs   rt   Interpreterr   r   r   r   r   rD   r   indexminrv  
itemgetterrC  rV  r  r   r   r   r   	is_impurer   rn  HigherOrderOperatorr   )r   r   r   rR   r   rb  r   	last_noder   r   indicesr   rv  rw  r   r`  rn  rj  r  s    ` ``          @@@@r9   r   z*ReplacementPatternEntry.replace_with_graph  se    8::	? :	?uxx++ :	?x ))+|!$QI?"?a..445E &a/ Q#G 
 G)<)<Q)?@CI	6	6	6 	6 3		6
 	6( ##I. f	6/1E1EFFF9(#45994@K+uxx}}5*m$Q&)Q&CQ& Q& Q&f < C$44 #L+ > &HCC%& <(A---Q5Mf	6P 	 45 	'DJJ("4;;

0N0NO  &	'O6f	6 f	6s   >IC
II#c           
         |j                   J | j                  |||j                    | j                  |j                  i |j                  | j
                  xs d       y )Nr   rs  )r   r   rI  rR   rS   rJ  r1  s       r9   r2  zReplacementPatternEntry.apply  sb    &&222##D<u||<''?+? 	  	
r;   rN   )r   r   r   r  r   z%torch.fx.Graph | torch.fx.GraphModulerR   zSequence[torch.fx.Node]r   r0   r=   rl   r:  )	rL   rW   rX   r  rY   rJ  r  r   r2  rO   r;   r9   r   r     sw     -,#L*# !%]']']' A]' &	]'
 ]' 
]' ]'~
r;   r   c                     yr  rO   )r   s    r9   _return_truer    s    r;   c                F    t         j                  d| j                  |       y )Nz@Replacement pattern %s failed to apply due to shape mismatch: %s)loginforL   )	search_fnr-  s     r9   log_trace_failurer    s    HHJ	r;   Fc                p   t         j                  |       |j                        }|s'|   j                  |rt	        |             yd       y||ryt        j                  dfd       t	        |      }|D ]&  |k7  r	|r yt        j                  dfd       ( |j                  |       y)aB  
    Check if a pattern is a duplicate. Because we ignore certain types in searching, but not
    in matching, use the graph to distinguish equivalent search patterns.

    Returns True if a duplicate is found and `skip_duplicates=True` is passed in. Errors if
    `skip_duplicates` is False and a duplicate is found.
    NFTc                     d  dS )NDuplicate pattern: z with no graphrO   )pattern_reprs   r9   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>  s    ),~F r;   c                     d d  dS )Nr  z with duplicated match graph  rO   )	graph_strr  s   r9   r   z1check_and_add_duplicate_pattern.<locals>.<lambda>  s    ),7TU^T__`a r;   )rJ  r  rA   r   rK   rs   _check)r   r   seen_patternsskip_duplicatesequiv_pattern_reprsnew_graph_strr  r  s         @@r9   check_and_add_duplicate_patternr    s     (++G4L'++L9l#**3u:I EII}F	

 JM( 
	I%a	

 }-r;   rO   c                (    g t        j                         j                  j                         t        j                         rt                t        j                        r;g t        j                        j                  j                         }t        |      t        |t        t        f      st        dt        |             d
 f
d}dfd}t        u rt        j                         ryt        j                  d      5  |D cg c]*  }t        |t        j                         xr |j"                  , c}t%         |      \  }}n}d}t        |t&              r|n|gD ]D  }t        |t(              st+        ||r|j,                  nd|j.                  |	      s< ddd       y t1        ||||
	      }|j3                  |       |j4                  cddd       S c c}w # 1 sw Y   yxY w)a  
    Create a replacement rule based on example functions that get traced
    to create patterns.  This supports both training and inference when
    run on a joint forward+backward graph.

    Args:
        search_fn: traced to give original pattern
        replace_fn: traced to give replacement graph
        example_inputs: example inputs for initial trace
        trace_fn: fwd_only or joint_fwd_bwd
        pass_dict: dict of passes to register to
        extra_check: additional check to run on match(using real shapes)
    z,example_inputs must be a list or tuple, got c           
     	  
 t              }|D ]+  }|| j                  vst        d| d| j                          t        t        j                  j                  |D cg c]  }| j                  |    c}d             g }t        j                  j                  j                        }|J |5  t              D ]  \  }}t        |   t        j                        s%|r"t        |   j                        r
 ddd       yt        j                  |   j                         |   j!                         |   j                  |   j"                  |      |<   t%        j&                  |   j(                  |   j!                               D ]C  t        t        j*                        st-        fd|D              s3|j/                         E  }|s7|rdfd}	  ||z   	      }	g }t3        t5        t7        |      t7              z         |	j8                  j:                        D ]  \  }}|t7        |      k  r|j/                  |j<                         0|	j8                  j?                  |      5  |	j8                  jA                  ||t7        |      z
           }|jB                  |_        |jE                  |       |	j8                  jG                  |       ddd        ||z   }n	  	      }	tI        |	|
      }| jK                         d   }|J |jM                  |      }tO        |jB                        r#tP        jS                  d||jT                  ||       tW        |      r |      r| 	      | _,        t7        | j:                        dk(  rJ| jX                  j8                  j:                  D ]'  }t[        |j\                  | j:                  d   d       ) 	 ddd       y	 ddd       yc c}w # t        $ r}
t1        |
       Y d}
~
ddd       yd}
~
ww xY w# 1 sw Y   xY w# t        $ r}
t1        |
       Y d}
~
ddd       yd}
~
ww xY w# 1 sw Y   yxY w)z
        Often shapes get burned into the pattern, so our initial match ran with
        `ignore_types=(int, ...)`.

        Recheck the match with the correct shapes.
        z_Not all inputs to pattern found in match.kwargs. Perhaps one of the inputs is unused? argnames=z, match.kwargs=c                     | j                   d   S r   r   )r   s    r9   r   z8register_replacement.<locals>.check_fn.<locals>.<lambda>  s    QVVE] r;   NF)dtypedevicerequires_gradc              3  :   K   | ]  }t        |k7          y wrN   )r   )rx   r  rz   s     r9   r{   z9register_replacement.<locals>.check_fn.<locals>.<genexpr>  s       ?>?1!q&9?s   c                 >     | t        |       t              z
  d   S rN   )r   )args_newr^  rR   r  s     r9   search_fn_newz=register_replacement.<locals>.check_fn.<locals>.search_fn_new/  s#    ((3x=3t93L3N*OPPr;   get_decomp_fn)argnamesexclusive_arg_namesscalar_workaroundr   z"Specific pattern match: %s%s %s %sr*   r   r   T)r  r   r^  r   r=   r   )/r   rS   r  rs   rt   r   r   r   r   r  rD   Tensorr   r  empty_stridedsizestrider  r  r  shapeSymIntr5   r   r  r   ranger   r   r   r   inserting_afterrO  rP  rD  r   fx_to_patternr   r   rH   r  warningrR   r  r   r   r   )r   r  rP  sym_argsr   r  gradspecific_patternr  specific_graphr-  sym_arg_namesrO  r   r   specific_pattern_matchr   rR   rz   argnames_staticr  r/  r  
replace_fnr  r  r  search_fn_patternr   s                    @@r9   check_fnz&register_replacement.<locals>.check_fn  s}    ( 	D5<<'"99A
/RWR^R^Q_a 	 HH089d#9;R
 (*MM''88>	$$$ q	$]3 /4d1gu||4 0a ?$q	 q	 $11QQ("1gmm#Aw~~&*
DG '__T!W]]DGNN<LM /%a63 ?CK? < %OOA.	/%/6  1#Q%)1)$tO*7* %'M*-c(mc$i78&,,22+ I; s8},)001C1CD$+11AA+N I'5';';'G'G (S]): ;(H /7mmHO'==hG*00;;KHI II   -x7H%)1%t=* $1"%(;&7	$  %%'*D###%5%;%;D%A"!$)),8II*$ ./K@V4W*2M+' u{{#q("44::@@ &%&VV%*[[^&3 aq	 q	b cq	 q	 :n ( %))Q7$eq	 q	`%I I ( %))Q7$]q	 q	X%Yq	 q	s   !P50Q;&Q;
B*Q;5Q;
%Q;0P>BQ;	A(Q1Q;QC5Q;Q;	Q#P</Q;<QQ;Q	Q;	Q8Q3&Q;3Q88Q;;Rc                     D cg c]  }| j                  |       }}t        dt        |       dz         D ].  }d| | vr n%|j                  | j                  d|              0 | r
J d|        |S c c}w )Nr*   	tangents_zleftover kwargs: )r
  r  r   r   )rS   rP  rR   r  r  s       r9   rI  z,register_replacement.<locals>.normalize_args{  s    -<=T

4 ==q#f+/* 	5A1#f,KK

Yqc?34	5 9.vj99z >s   A7Ffunctionalize_rng_opsNr  )r  )r   r/  rI  rJ  r   r   r=   rE   )rS   r   r=   r   )inspect	signature
parametersr   ismethod_wrap_bound_methodrD   r   r   	TypeErrorr  joint_fwd_bwdrs   is_inference_mode_enabledfunctorch_configpatchr  r  gen_pattern_and_search_gmr   r7  r  r   r  r   r6  r   )r  r  example_inputsr   r9  r/  r  r  r  r  rJ  r  replace_argnamesr  rI  r  r   gmpattern_matcher_passr  r  s   `` ` ````  `       @@r9   register_replacementr    s   6 H)))4??DDFGO	"&y/B	
#MW..z:EEJJLM'
4DE
ntUm4:4;O:PQ
 	
I IV =  **, 
		e	< $ES%
@AJq%,,';AOO;%
 $3!#+KGR (GB %Z:J
	!  .0BC2 "BHH(66$3	 !7$ $"
	! * )%	
 	$I$ $%
$ $s1    H%/HAH'HH,HHHzOrderedSet[str]_serialized_patternsc                V   dd}t         j                         st        dt                |j                  }ddlm}  |j                  d      5  t        ||||      }d d d        t        j                  |       }	|t        vrd}
t        j                  |       nd	}
 |       }t        t         | d
z  |
      5 }|
dk(  r|j                  |       n|j                  d       |j                  |	       |j                  d       d d d        |S # 1 sw Y   xY w# 1 sw Y   |S xY w)Nc                    t        j                  d      } t        j                  d      j                  |       }g }t        t        j
                  j                        D ]^  }t        t        j
                  j                  |      }	 t        |t              r't        |t        t        f      r|j                  |       ` dj                  |      }d| d}| | S # t        $ r Y w xY w)Nz            # This is an auto-generated file. Please do not modify it by hand.
            # To re-generate, run:
            # cd ~/pytorch && python torchgen/fuse/gen_patterns.py
            a               # mypy: ignore-errors

            # noqa: F401, E501
            {msg}
            import torch
            import torch._inductor
            import operator

            aten = torch.ops.aten
            prims = torch.ops.prims

            )msgz,
   z1from torch._inductor.pattern_matcher import (
   z,
)
)textwrapdedentr  dirrs   	_inductorpattern_matcherrk  rD   r  
issubclassr   rf  r   r  r  )auto_generated_msgfile_templatepattern_matcher_importsrP  attrformatted_importss         r9   get_file_templatez-_serialize_pattern.<locals>.get_file_template  s    %__
 !
 &'&
( 	  #%778 		D5??::DADdD)j;4/ ,2248		 %MM*ABPQbPccij!2 344  s   7C	C*)C*z0Could not find serialized patterns directory at r   r"   Fr  )r  wr  z.pyz

r  r  )SERIALIZED_PATTERN_PATHis_dirr  rL   torch._functorchr#   r  gen_patternrJ  r  r  r  openwrite)unique_namer  r  r   r  r  rJ  r  r   serialized_pattern
write_moder  fs                r9   _serialize_patternr    s)   '5R #))+>?V>WX
 	
 %%L;				e	< ViCTUV .11'{1S//
  .
%'M	%<.(<<j	I QGGM"GGFO	"#	 N)V V Ns   D=ADDD(	fx_passesserialized_patternszvlist[tuple[Any, Iterable[Any], Callable[[Callable[..., Any], Iterable[Any]], torch.fx.GraphModule], Any, PatternExpr]]_known_precompiled_patternsc                   t        |      }dt        j                  v rt        | ||||      }nT|j                  }t        j                  d|       }|rt        ||       st        j                  d|        t        ||       }t        j                  |      D ]'  }t        |t              s|j                  !d |_        ) t         j#                  |||||f       t%        ||||||||||	| |
       y )NPYTORCH_GEN_PATTERNSz.torch._inductor.fx_passes.serialized_patterns.zDPrecompiled pattern %r not found. Run torchgen/fuse/gen_patterns.py.)r  r  rJ  r  )r   r?   r@   r  rL   	importlibimport_moduler  r  r  rk  r  	tree_iterrD   r'   constantr  r   r  )r  r  r  r  r   r9  r/  r  r  r  r  patrJ  r  r   s                  r9   gen_register_replacementr    s    >*N+ NH>O
 !))##<\NK
 ;/KKV a%/  c:&3<<+C
  CL   &&	NH.?E ' #r;   r  c                v   g t        j                  |       j                  j                         }|i }g }d}|D ]4  }	|	|v r|j	                  ||	          |j	                  ||          |dz  }6  || ||      }
t        |
t        t        t        t        j                  t        j                  f|||      |
fS )Nr   r*   r  )ignore_typesr  r  r  )r  r  r  r   r   r  intfloatr   rs   r  r  )r  r  r   r  r  r  r  flat_inputs	input_idxargname	search_gms              r9   r  r  P  s     A""9-88==?@H KI ''09:~i89NI K}MIudELL%++F/ 3	
 		 	r;   c                &    t        | ||||      d   S r  )r  )r  r  r   r  r  s        r9   r  r  t  s&     %>8->@S	 	r;   r4  c                     d fd}|S )z
    Register an aten to inductor IR replacement pattern.  The decorated
    function is saved and then called a lowering time allowing direct
    pattern to inductor IR conversion.
    c                n    t        |       sJ t        |       j                         d| _        | S )Nr   r/  r?  r4  T)rm  r=  r6  _inductor_lowering_functionr?  r/  	pass_dictr   r5  s    r9   	decoratorz,register_lowering_pattern.<locals>.decorator  s;       g	

(9g(
..2+r;   r?  r>  r=   r>  rO   r   r/  r  r5  r  s   ```` r9   register_lowering_patternr	    s      r;   c                     d fd}|S )zk
    Register a pattern that runs a function on the FX graph, allowing
    custom transformation code.
    c                `    t        |       sJ t        |       j                         | S )Nr  r4  )rm  rF  r6  r  s    r9   r  z)register_graph_pattern.<locals>.decorator  s3       g	

(9g(
.r;   r  rO   r  s   ```` r9   register_graph_patternr    s      r;   c                B    |t        t        | j                              u S rN   )r   iterr   )r   r   s     r9   is_start_of_fx_graphr    s    4U[[)***r;   z6(?<!_)(_$|_[.]|(\b|_)(set|enter|exit|seed)(\b|_))(?!_)c                    | j                   dk7  ry| t        j                  j                  j                  j
                  t        j                  j                  j                  j
                  fv S )Nr.   F)r  rs   r   r.   accumulate_grad_defaultresize_storage_bytes_)rr  s    r9   "fixme_incorrect_inductor_schema_opr    sW    	||z! 		++33		0088  r;   c                   t        | j                  t        j                  j                        r5t        | j                        s | j                  j                  j                  S t        | j                  t        j                  j                  j                        ry| j                  dk(  rAt        | j                        sJ t        j                  | j                  j                        rLy| j                  dk(  r<t        | j                  t               sJ t        j                  | j                        ry| j"                  j%                  d      d uS )NFr  Tr  out)rD   r   rs   rn  rw  r  _schema
is_mutable_higher_order_opsauto_functionalizeAutoFunctionalizedrr  rm  _mutation_op_researchrL   rK   rS   rA   rm  s    r9   is_mutation_opr    s    UZZ**0={{""---	U,,??RR
 ww/!$$$!!$++"6"67	M	!$++s+++!!$++.;;??5!--r;   c                    d| j                   v sJ d|j                   v sJ | j                   d   |j                   d   k(  S Nmutation_region_idr   )r  r  s     r9   same_mutation_regionsr"    sE    166)))166)))66&'1662F+GGGr;   c                ,   |}d|j                   vr3t        | |      s'|j                  }d|j                   vrt        | |      s'|j                   j                  dd      }||ur0|j                  }t        |      r|dz  }||j                   d<   ||ur0|S )Nr!  r   r*   )r   r  prevrA   r   r  )r   r   r   r!  s       r9   get_mutation_region_idr%    s    A
aff
,5I%QR5SFF aff
,5I%QR5S$8!<
4-FF!!#'9#$	 4-
 r;   c                V    dt        t        | j                              j                  vS r   )r   r  r   r   r+  s    r9   "should_compute_mutation_region_idsr'    s!    tD,='>'C'CCCr;   c                h    d}| j                   D ]!  }t        |      r|dz  }||j                  d<   # y )Nr   r*   r!  )r   r  r   )r   r!  nds      r9   compute_mutation_region_idsr*    s=    kk ;"!#(:$%;r;   c                     d fd}|D cg c]0  }t        j                  |t         j                  j                        2 }}t        j                  |      |_        |S c c}w )zQ
    Wrap a bound method to remove 'self' from its signature for FX tracing.
    c                      | i |S rN   rO   )rR   rS   ra   s     r9   wrapperz#_wrap_bound_method.<locals>.wrapper  s    4"6""r;   rV   )r  	ParameterPOSITIONAL_OR_KEYWORD	Signature__signature__)ra   r  r-  rP  paramss   `    r9   r  r    s_    
#
  	$ 1 1 G GHF  $--f5GNs   5Ac                  L     e Zd ZdZ	 	 d	 	 	 	 	 d fdZddZd	dZd
dZ xZS )r7  zA
    Registry of patterns to match and replace in FX graphs.
    c                    t         |           t        t              | _        || _        || _        t        t              | _        y rN   )r   rj   r   r   patternsr   	subsystemr  )rQ   r   r6  r   s      r9   rj   zPatternMatcherPass.__init__  sA    
 	  	 #" ;Fd:Kr;   c                     | j                   |   S rN   )r5  )rQ   items     r9   r)  zPatternMatcherPass.__getitem__  s    }}T""r;   c                *   | j                   syt        |t        j                  j                        r|j
                  }nJt        |t        j                  j                        r|}|j                  }nt        dt        |             t        |      rt        |       t        j                  t        |      }d}g }d}| j                   D ]0  \  }}|dk(  rd}|j                  |j!                  ||d             2 |r"|j                  |j!                  dd             | j"                  | j"                  nd	}	t        |t        j                  j                        sJ t%        ||	| j&                        5  t)        t*        j,                  j/                  |      d
      D ]U  }
t1        |
      }|
j2                  dk(  r|
j2                  |f| j                   vr9t5        |
d      rG| j                   |
j2                  |f   D ]  }|
j6                  r t|j8                  j;                  |
      }t=        |      r,t?        tA        tC        ||jD                                    dk7  rdt=        |      r)t?        tA        d |jD                  D                    dk7  rtG        |
jH                        r-tJ        jM                  d|
|
jN                  ||j8                         t=        |      stQ        |jS                  |            s|dz  }|jU                  |||
       tV        tX           dxx   dz  cc<   tV        tX           dxx   t?        |jD                        z  cc<   tZ        j\                  j_                  d      s~ta        |dd      r|jb                  }n<|j8                  jd                  jf                  }ti        |
jj                        }| d| }tX         d}tV        |   |xx   dz  cc<    X 	 ddd       |S # 1 sw Y   |S xY w)zLApply all registered patterns to the graph, returning the number of matches.r   zJThe input to PatternMatcherPass must be a GraphModule or a Graph, but got Fr  T)rr  r   sort)rr  r:  Nr  )reverse)allow_cpu_inputsr*   c              3  t   K   | ]0  }|j                   j                  d i       j                  dd       2 yw)customstreamr   N)r   rA   )rx   r   s     r9   r{   z+PatternMatcherPass.apply.<locals>.<genexpr>]  s4      '$% !"

8R 8 < <Xq I's   68z
%s%s %s %spattern_matcher_countpattern_matcher_nodesr<   rJ  r^  _pattern_matcher_per_pattern)6r5  rD   rs   rt   rV  r   GraphrW  r  r  r'  r*  r   r   r%  r   
find_nodesr   r   r6  sortedr  r  from_iterabler  rr  r,   r   r   r   r  r   r    r  r   rH   rP  r  r  rR   r   r/  r2  r   backendr?   r@   rA   rk  rJ  r   rL   rK   r   )rQ   r  r   get_mutation_region_id_partialcountr   has_call_modulerr  r   r   r   entryr  rJ  pattern_classpattern_keys                   r9   r2  zPatternMatcherPass.apply!  s   }}b%((../HHEEHHNN+E$$B\]abd]e\fg  .e4'.)2):):"E*
& -- 	QJB]""&U--F-OP		Q
 LL))])GH&*nn&@DNNFW	"ehh22333#B	4>>B :	Ey<<UCTR 9E'-77m+(= 
 9PUV!]]DGGV+<= -EE||++D1A !&s+I177'ST 
 ! !& ')*'   !)$))4L$		1emmT{~e6G6G6J'K
Aud3 )*ABaGB )*ABc!''lRB ::>>*MN&undC/4/A/A 160G0G0P0P),T[[)92?&/J-4I5Q*RK$[1,?1D?[-E9E:	Ev w:	Ev s!   9E2P,PA:PA9PPc                8    | j                   j                          y rN   )r5  clearri   s    r9   rO  zPatternMatcherPass.clear|  s    r;   r   )r   r0   r6  r0   r=   rl   )r8  r*  r=   r+  )r  %torch.fx.GraphModule | torch.fx.Graphr=   r  rk   )	rL   rW   rX   r  rj   r)  r2  rO  r  r  s   @r9   r7  r7    sF     !% $LL L 
	L$#Yvr;   r7  c                     t         rN   r  r   s     r9   _not_implementedrR    s    
r;   c                  	
 |xs i }|j                         D ci c]  \  }}||
 c}}
t        
      t        |      k(  sJ 	 d	 	 	 	 	 d
fdt        j                         	 G 	fddt        j
                  j                        }t        | t        j
                  j                        sJ  ||       j                         }t        |t              st        t        j                  |            S |S c c}}w )z
    Convert an FX graph into a PatternExpr.  This is useful for simple
    patterns that can only match single functions and fixed-length lists.
    c                    ||n}t        | t        t        f      r| v rt        |          S t	        |       |v r
t               S t        | t              rt        d | D              r| r
t               S | S )Nc              3  <   K   | ]  }t        |t                y wrN   )rD   rB  )rx   ys     r9   r{   z5fx_to_pattern.<locals>.process_arg.<locals>.<genexpr>  s     &I!z!W'=&Is   )rD   r  r  rL  r  rB  r   r5   )r  ignore_types_overridecurrent_ignore_typesr  inv_scalar_workarounds      r9   process_argz"fx_to_pattern.<locals>.process_arg  sy     &;%F!L 	 a%&10E+E3A6777**9a3&Iq&I#Ia9r;   c                  l     e Zd ZeZeZeZ	 	 	 	 	 	 	 	 dfdZ	 	 	 	 	 	 	 	 dfdZd fdZ	 xZ
S ) fx_to_pattern.<locals>.Converterc                    t              }|t              k  r|   }n1r|j                  d      sJ |}nt        j                  dd|      }|}|v rt        |      S t        |      S )Ntangentz_\d+$r{  )r   r   
startswithresubr`  rL  )	rQ   r   rR   rS   r   rP  r  argnumr  s	         r9   rO  z,fx_to_pattern.<locals>.Converter.placeholder  su     VA3x= {((333"f5***400!$''r;   c                `   	}|t         j                  u rt        d D              f	 	 	 	 	 d	fd}|}t        j                  |||f      \  }}t
        v r>|D cg c]
  } ||       }}|j                         D ci c]  \  }}| ||       }}}t        |g|i |S c c}w c c}}w )Nc              3  2   K   | ]  }|t         us|  y wrN   )r  )rx   ts     r9   r{   zAfx_to_pattern.<locals>.Converter.call_function.<locals>.<genexpr>  s      N1C<Ns   c                     | |      S rN   rO   )r  rW  rZ  s     r9   process_arg_fn_implzKfx_to_pattern.<locals>.Converter.call_function.<locals>.process_arg_fn_impl  s     'q*?@@r;   r  rc   rW  zSequence[type[Any]] | Noner=   zT | KeywordArg | Ignored)rv  rl  r   r  r   r   r   r  )
rQ   r   rR   rS   process_arg_fnrg  r  ry   r  rZ  s
           r9   r  z.fx_to_pattern.<locals>.Converter.call_function  s     )N))) IN N#/N IAA+EA
 .A "5!??>D&>JLD&|#378aq)88;A<<>J41a!^A..JJ8888 9Js   B% B*c                l   t         |   |      }|j                  dk(  rzt        |t              rj|j
                  d   }t        |t              sJ t        |      t        |      k(  sJ t        ||      D ]  \  }}t        |j                        |_	        ! |S t        |j                        |_	        |S )Nr  r   )
r   rS  rr  rD   r   rR   r   r   r   r   )rQ   r   rvrR   rr   r   s         r9   rS  z)fx_to_pattern.<locals>.Converter.run_node  s    !!$BttxJr5$9vvay!$
3332w#d)+++!"dm -FAs!#))nAG-
 I qww<Ir;   )r   rK   rR   r  rS   Mapping[str, Any]r=   z ExclusiveKeywordArg | KeywordArg)r   rK   rR   r  rS   rm  r=   r   )r   r   r=   r   )rL   rW   rX   rR  r  r  rP  rO  r  rS  r  )r   r  rb  r  r  rZ  s   @r9   	Converterr\    sr    &&#	(	(  	( &		(
 .	(*	9	9  	9 &		9
 	96	 	r;   rn  rN   rh  )r   r   r  rI  rs   rt   r{  rD   rV  r  r   r  r  tree_leaves)r  r  r  r  r  ry   rz   rn  r   rb  rY  rZ  s    `` `    @@@r9   r  r    s     */R.?.E.E.GHdaQTH$%->)???? CG%?	! __FD DEHH(( DL b%((..///m!Gg{+!&"4"4W"=>>N{ Is   DT)r   r  c               6   t               5  t               5   t        |  |       d      | }ddd       ddd       ddlm} |r, |j
                         |j
                  j                          j                          |S # 1 sw Y   WxY w# 1 sw Y   [xY w)z>Build a normalized inference graph, for use with fx_to_patternreal)tracing_modeNr*   remove_noop_ops)r   r   r   fx_passes.post_gradrt  r   eliminate_dead_code	recompile)ra   rR   r   r  r  rt  s         r9   r   r     s     
"	# F%7%9 F>WRv>EF F 5! 	$$&LLNI%F F F Fs!   BBBB	BBr  c          
         d	 	 	 	 	 	 	 	 d
fd}t         j                  j                  d      5   t        | d | |       dd      |  ddd       sJ ddlm}  |j                         dd	lm} |j                  j                         t         j                  j                  j                         j                  _        j                  j                          j                          S # 1 sw Y   xY w)z=Build a normalized training graph, for use with fx_to_patternNc                <    rJ t        |       t        | |fi |S rN   )clone_graphr&   )joint_graphinputsrS   r  s      r9   record_joint_graphz)joint_fwd_bwd.<locals>.record_joint_graph	  s(     v% f???r;   c                    t        |       S rN   )r%   )r  r  s     r9   r   zjoint_fwd_bwd.<locals>.<lambda>'	  s    r': r;   TF)partition_fndecompositionskeep_inference_input_mutations
enable_logr*   rs  )early_patterns)r{  rb   r|  r  rS   r   r=   z1tuple[torch.fx.GraphModule, torch.fx.GraphModule])rs   _guardstracingr$   ru  rt  r   fx_passes.joint_graphr  r2  rt   CodeGen_codegenrv  rw  )ra   rR   r  r}  rt  r  r  s         @r9   r  r  	  s     '+B@)@3@@LO@	:@ 
		t	$ 		
:+(?+/	
 		 I24BHH5" ..0BHHHH  "LLNI1	 	s   C44C=c                    g }t         j                  j                  | j                  | j                  f|j
                         |S rN   )rs   rt   r   rR   rS   r   )r   rR   s     r9   r  r  >	  s1    )+D	HHaffahh'5Kr;   c                T   t        t        | j                              }t        t        j
                  j                            }t        t               }d }|r|j                         }t        |      D cg c]	  }||vs| }}|r||d      j                  |       n^|j                  |       |r|j                  |ur|j                  |       |}|j                  t        |j                  |d                   |r|s!t        |      t        | j                        k(  sJ y c c}w )NrA  rO   )r   r   r   r    rs   rt   r   r   r
  r  r   r  r   r   r   )r   pendingreadywaitingcursorr   r  waiting_fors           r9   stable_topological_sortr  D	  s     8EKK()G uxx}}%'E $G F
{{}"'+@Q%q@@ KO$++D1IIdO&++T1d#F NN8GKKb$9:;   3u:U[[)99999 As   4	D%>D%c                     t        j                         j                  t        j                  t        j
                         dt        f	 	 	 	 	 d fd              }|S )z0Wrapper around lazy init functions in fx_passes/Nc                t   t         t           j                         }i }dv r| |d<   dv r||d<   t        j                  j                  d       5  t               5  t               5   di |}d d d        d d d        d d d        |t         t        <   S # 1 sw Y   (xY w# 1 sw Y   ,xY w# 1 sw Y   0xY w)Ninput_devicer  rO   )r   rG  r   rs   r  r  r   r(   )r  r  counters_refrS   rZ  
_fn_paramsra   s        r9   	lazy_initz%init_once_fakemode.<locals>.lazy_initm	  s      (--/!#Z'%1F>"j(&3F?#]]""4( 	"*@*B 	"NDT 	"\&\F	" 	" 	" )	" 	" 	" 	" 	" 	"s<   B.B"&	B/B"7B.BB""B+	'B..B7)r  z
Any | Noner  ,Callable[..., dict[Any, Callable[..., Any]]]r=   r   )r  r  r  r   r%  rB  r+   )ra   r  r  s   ` @r9   init_once_fakemoder  h	  sn     ""2&11J____R#'  	 

 
  , r;   c                     d fd}|S )z2Function for extra_check to put pass behind a flagc                $    t        t              S rN   )rk  r#   )r   rP  s    r9   
flag_checkzconfig_flag.<locals>.flag_check	  s    vt$$r;   )r   r   r=   r   rO   )rP  r  s   ` r9   config_flagr  	  s    % r;   c                L     G d dt               } ||       j                         S )Nc                        e Zd Zd fdZ xZS )clone_graph.<locals>.CopyGraphc                J   t         |   |      }t        |t        j                  j
                        rn|j                  j                  j                  |j                         | j                  j                  j                  |j                  d       |j                  _        |S rN   )r   rS  rD   rs   rt   Proxyr   r   r   	new_graph_graph_namespacer   rP  )rQ   r   r   r   s      r9   rS  z'clone_graph.<locals>.CopyGraph.run_node	  sq    w'1H(EHHNN3""))(--8%)^^%D%D%P%PMM4&" Or;   )r   r   r=   r   )rL   rW   rX   rS  r  r  s   @r9   	CopyGraphr  	  s    	 	r;   r  )r)   	transform)input_graphr  s     r9   rz  rz  	  s$    	K 	 [!++--r;   _seen_patternsc                    t        | j                        |kD  r| j                  |   S |y | j                  j                  |      S rN   )r   rR   rS   rA   )r   
arg_number
kwarg_names      r9   get_arg_valuer  	  s?     499~
"yy$$		{{z**r;   c           	        |g}t        |t        j                  j                        r7|j	                  |j                         D cg c]  }t        ||       c}       | D cg c]  }|j                  |v s| c}S c c}w c c}w rN   )rD   rs   rn  ro  r   rp  rk  r   )r   ra   rg  rl  r   s        r9   filter_nodesr  	  sf    $C"ejj112

",,.IhGB)IJ"9TdkkS&8D99 J9s   A>#B7Bc                    | j                   dk(  rPt        | j                  t              sJ t	        | j
                  j                  | j                        j                  S | j                  S )zFor call_function and call_method, we directly use the target function;
    For call_module, the target is string, and we treat the module class
     as a function.
    r  )rr  rD   r   rK   r   r   rW  r   rm  s    r9   r  r  	  sO    
 ww-$++s+++114;;?III;;r;   )rF   rK   r=   rE   )r{  )r   r   r   r   r   rK   r=   rl   )r  r%  r=   zTypeIs[Match]r  )r  r>  r-  r  r=   rl   r  )
r   r   r   ztorch.fx.Graph | Noner  zdict[str, list[str | None]]r  rE   r=   rE   )r  rJ   r  r[   r  zlist[Any] | tuple[Any, ...]r   r_   r9  r;  r/  r.  r  dict[str, float | int] | Noner  Sequence[str]r  zPatternExpr | Noner  rE   rJ  r0   r  r  r=   rE   )r  rK   r  rJ   r  r  r   r_   r  r  r=   r   )r  rK   r  rJ   r  r[   r  zIterable[Any]r   r_   r9  r;  r/  r.  r  r  r  r  r  rE   r  r  r=   rl   )r  rJ   r  r  r   r_   r  r  r  r  r  r  r=   z(tuple[PatternExpr, torch.fx.GraphModule])NrO   )r  rJ   r  r  r   r_   r  r  r  r  r=   r   )
r   r   r/  r.  r  r'  r5  rE   r=   z2Callable[[Callable[..., Any]], Callable[..., Any]])r   r  r   r   r=   rE   )rr  ztorch._ops.OpOverloadr=   rE   r  )r  r   r  r   r=   rE   )r   r  r   r   r=   r  )r   r  r=   rE   )r   r  r=   rl   )ra   r   r  z	list[str]r=   r   )rR   r   rS   r   r=   r   )rO   rO   NrO   )r  rP  r  zSequence[type[Any]]r  r  r  r  r  r  r=   r   )
ra   r>  rR   r  r   rE   r  r>  r=   rb   )ra   r>  rR   r  r  r>  r=   rb   )r   r   r=   zlist[torch.fx.node.Argument])ra   r>  r=   r>  )rP  rK   r=   zCallable[[Match], Any])r  rb   r=   rb   rN   )r   r   r  r  r  r0   r=   r   )r   zIterable[torch.fx.Node]ra   r   r=   r   )r   r   r=   ztorch.fx.node.Target)r  
__future__r   r   dataclassesr   r  r  r  loggingrv  r?   r`  r  rY  abcr   r   collectionsr   collections.abcr   r   r	   r
   r   r   pathlibr   r   r   r   r   typing_extensionsr   r   rs   torch._guardstorch.fxtorch.utils._pytreer   r   r  torch._dispatch.pythonr   torch._dynamo.utilsr   torch._prims_commonr   torch._subclasses.fake_tensorr   "torch.fx.experimental.proxy_tensorr   %torch.fx.experimental.symbolic_shapesr   r   torch.fx.graph_moduler   torch.fx.immutable_collectionsr   r   (torch.fx.passes.graph_transform_observerr   r   r   torch.utils._ordered_setr    
_functorchr#   r  _functorch.aot_autogradr$   r%   _functorch.partitionersr&   _subclassesr'   r(   rt   r)   r{  decompositionr+   loweringr,   	getLoggerrL   r  r   atenprimsConstantr   r$  r@   rA   rG  r/   rY   r1   rH   rJ   r[   r_   rc   r   TargetrK   FnsTypere   rg   r   r   r  r   r%  r  r   r   r<  rB  rL  r`  rf  r   _SimpleSpecr  r  r  r  r  r  r  r  r  r  r  rJ  r'  	dataclassr-  r=  rF  r   r  r  r  r  r  r  __file__parentr  r  r  r  r  r  r	  r  r  compiler  r  r  r"  r%  r'  r*  r  r7  rR  r  no_gradr   enable_gradr  r  r  r  r  rz  r  r  r  r  rO   r;   r9   <module>r     s)  !F #         	 	   # # X X  3 3 *    $ $ ; ( 0 @ 6 W + I K 1 / 3 C 7 4   . ; g!yy~~		EHHMM)
**..>

K 59 1 8+/ j /$4>x >> >#h # CL ((--


$? ? : IK??(5?BE?	?BN Nb, 8 k!'
 '
T 1#  1F-+ -k E E(E+ E4O
+ O
d CHoo
k o
d?    (, * * ,
[ ,
^F
 F
R%
; %
P4 4n!X ! : : :: 
< 
 
 	= 	= 	= p
l p
 p
f "	'' ' /' 	'
 
'` ,87;)+,0!#BUhhh 0h 	h
 :h )h 5h 'h *h h h @h 
hV )3 o 4MMM "M 	M
 5M M` x.//+=@UU     $ ,87;)+!BU888 8 "	8
 8 :8 )8 58 '8 8 @8 
8v e4
 8<)+BU  !    5	 
 '  @  .  5 N 8<)+				!		 		 5			
 '		 		 ,8
 ( 	
  84 ,8
 ( 	
  8,+ "**VW	.(H
D; v vr )+ 7;)+k-k%k k 5	k
 'k k\ 
 #'(;
  	
 &  < 
 )<	))
) &	)
 ) )X!:H@.  #-, . DH+
+%(+6@++:r;   