
    9j'                        d Z ddlZddlZddlZddlZddlZddlmZmZ ddl	m
Z
mZ ddlmZmZ ddlZddlmZ ddlmZmZmZ dd	lmZmZmZmZmZmZmZ dd
lm Z m!Z! ddl"m#Z# ddl$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*m+Z+ ddl,m-Z-m.Z.m/Z/m0Z0m1Z1m2Z2m3Z3m4Z4m5Z5m6Z6m7Z7m8Z8m9Z9m:Z:m;Z; ddl<m=Z=m>Z>m?Z? ddl@mAZA ddlBmCZC ddlDmEZE erddlFmGZG ddlHmIZI ddlJmKZK dddej                  j                  dee?   deNeOe?f   ddf
dZPe
deOdedddej                  j                  def
d       ZQdedz  dej                  j                  ddfd ZR G d! d"e?      ZS G d# d$eE      ZT G d% d&eT      ZU G d' d(eT      ZVy))aQ  
This module implements variable tracking for PyTorch nn.Module instances during Dynamo tracing.

It provides specialized handling for different types of nn.Module instances through several key classes:

- NNModuleVariable: Handles instance-specific module tracing, specializing on module id() and placing
  parameters directly on the torch.fx.GraphModule. This creates one graph per module instance.

- UnspecializedNNModuleVariable: Provides class-level module tracing, treating nn.Modules like other
  user-defined objects and passing parameters as inputs to the FX graph. This creates one graph per
  module class.

- UnspecializedBuiltinNNModuleVariable: Specifically handles built-in PyTorch modules (e.g. nn.Linear)
  with appropriate optimizations.

- FSDPManagedNNModuleVariable: Special handling for FSDP-wrapped modules with modified guarding behavior
  and parameter handling.

The module integrates with Dynamo's broader tracing functionality to handle module method calls,
parameter access, hooks, and other nn.Module behaviors while maintaining proper scoping and guarding
of module state.
    N)IterableSequence)contextmanagernullcontext)AnyTYPE_CHECKING)Source   )graph_break_hintstrace_rules	variables)handle_observed_exceptionObservedAttributeErrorraise_observed_exceptionraise_type_errorunimplementedUnspecializeRestartAnalysisUnsupported)GuardBuilderinstall_guard)GenerationTracker)
AttrSourceConstDictKeySourceDictGetItemSourceFSDPNNModuleSourceGetItemSourceNNModuleSourceUnspecializedNNModuleSource)"enumerate_items_with_dict_positionget_custom_getattrget_fake_valueis_lazy_moduleis_namedtupleis_safe_constantistensoristypennmodule_has_hooksobject_has_getattributeproxy_args_kwargsraise_args_mismatchset_example_valueunpatched_nn_module_callunpatched_nn_module_call_impl   )typestrValueMutationNewVariableTracker)invoke_and_store_as_constant)LazyVariableTracker)UserDefinedObjectVariable)InstructionTranslatorConstantVariable)DunderDictVariabletxr5   modargskwargsreturnc                 d    t        |d      rudt        dt        f fdt        ||      \  }}|D cg c]
  } |       }}|j                         D 	ci c]  \  }}	| |	       }
}}		 |j	                  |||
       yyc c}w c c}	}w # t
        $ r t        t
         dg       Y yw xY w)a~  
    Fairly coupled helper used by NNModuleVariable and UnspecializedNNModuleVariable.

    Used to cause lazy module to be initialized (and delete its init hook) before tracing. Especially
    useful now that 'allowed' modules graph-break on hooks, calling this first ensures there is no hook
    by the time we trace __call__ and thus no graph-break for lazy allowed modules.
    _initialize_hookxr=   c                    t        |       r t        |       fd| D         S t        | t              r*| j	                         D ci c]  \  }}| |       c}}S t        | t
        t        t        f      r t        |       fd| D              S t        | t        j                  j                        rt        | j                        S | S c c}}w )Nc              3   .   K   | ]  } |        y wN .0elemconvert_to_fakes     a/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/_dynamo/variables/nn_module.py	<genexpr>zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>h   s      E4!6 E   c              3   .   K   | ]  } |        y wrC   rD   rE   s     rI   rJ   zBinitialize_lazy_module.<locals>.convert_to_fake.<locals>.<genexpr>l   s     Ct4CrK   )r#   type
isinstancedictitemslisttuplesettorchfxProxyr!   node)r@   kvrH   r9   s      rI   rH   z/initialize_lazy_module.<locals>.convert_to_fakef   s    QtAw E1 EFFAt$:;'')D$!Q?1--DDAeS12tAwCCCCAuxx~~.%affb11 Es   Cz0AttributeError during lazy module initializationr;   N)hasattrr   r)   rP   _infer_parametersAttributeErrorr   )r9   r:   r;   r<   
proxy_argsproxy_kwargsarg	fake_argsrX   rY   fake_kwargsrH   s   `          @rI   initialize_lazy_modulerc   W   s     s&'
	s 
	s 
	 $5T6#B 
L5?@c_S)@	@9E9K9K9MNAq/!,,NN	!!#y+>% ( AN  	$HI	s   BB
0B B/.B/
module_keysourcec              #   H  K   |j                   }t        j                  dd|      }|j                  j	                  |d      }|dkD  r|  d| n| } 	 ||j
                  f|j                  | <   |dz   |j                  |<   d  |j                  | = y # |j                  | = w xY ww)Nz9\._(?:modules|parameters|buffers)\[(['\"])([^'\"\]]+)\1\]z.\2r   @r.   )nameresub	num_callsget	__class__nn_module_stack)rd   re   r9   r:   fully_qualified_namerk   s         rI   record_nn_module_stackrp      s      ";;66D
   !5q9I09AJ<q,:J+*>)N:&-6])*z*Bz*s   AB"1B B"BB"c                    | rd|j                   v rt        |j                   d         rk|j                   d   }t        | d      }t        |      t        j
                  u rt        |d      }t        |j                  t        j                               y t        | j                  t        j                  t        j                  d                   y y )Nforward__func__attr)__dict__callabler   rM   types
MethodTyper   
make_guardr   CLOSURE_MATCH	functoolspartialNOT_PRESENT_IN_GENERIC_DICT)re   r:   fwdforward_sources       rI   &guard_to_detect_forward_monkeypatchingr      s      $#,,y2I)J,,y)C'	:NCyE,,,!+NJ!G.33L4N4NOP !!%%$@@y     c                   4    e Zd Zddddhej                  Zdededej                  j                  de
ddf
 fdZd&dZdefdZdeddfdZdefdZdefdZd	d
defdZd	d
dedej                  j                  de
de
ddfdZd	d
dee   fdZd	d
deddfdZd	d
defdZd'dZdej                  j                  d	d
dedededz  f
dZd	d
dedef fdZd	d
d ee   deeef   defd!Z	 	 	 	 	 	 d(d"Z 	 d)d	d
ded ee   deeef   d#edef fd$Z!d*d%Z" xZ#S )+NNModuleVariablemodule_typerd   valuenn_module_stack_sourcer<   r=   Nc                     t        |   di | || _        || _        || _        | j
                  | _        | j
                  | _        y )NrD   )super__init__r   rd   r   re   r   )selfr   rd   r   r<   rm   s        rI   r   zNNModuleVariable.__init__   sD     	"6"&$
 #kk&*kk#r   r9   r5   c                 |    t        | d      s%t        j                  j                  ||       | _        | j                  S )Ndict_vt)r[   r   r8   creater   )r   r9   s     rI   get_dict_vtzNNModuleVariable.get_dict_vt   s0    tY'$77>>r4HDL||r   c                 B    | j                   xs | j                  }|sJ |S rC   r   re   r   ress     rI   get_nn_module_stack_sourcez+NNModuleVariable.get_nn_module_stack_source   "    ))8T[[
s
r   re   c                     || _         y rC   r   r   re   s     rI   set_nn_module_stack_sourcez+NNModuleVariable.set_nn_module_stack_source   
    &,#r   c                     | j                   S rC   )r   r   s    rI   python_typezNNModuleVariable.python_type   s    r   c                     | j                   S rC   )r   r   s    rI   get_real_python_backed_valuez-NNModuleVariable.get_real_python_backed_value   s    zzr   c                     ddl m} |j                  j                  | j                        } |j
                  t        |            S )aY  nb_bool for nn.Module.

        nn.Module itself has no __bool__ or __len__, so bare modules are always
        truthy.  Subclasses like ModuleList/ModuleDict define __len__, so
        bool(module) calls PyObject_IsTrue which falls through nb_bool (NULL)
        to sq_length/mp_length.  We evaluate on the real module to capture this.
        r.   r6   )constantr7   outputget_submodulerd   r   bool)r   r9   r7   r:   s       rI   	bool_implzNNModuleVariable.bool_impl   s7     	/ii%%doo6&&&tCy11r   submod	key_extraoptionsc                      y rC   rD   )r   r9   re   r   r   r   s         rI   _wrap_submodulez NNModuleVariable._wrap_submodule   s     	r   c                 H   |j                   j                  | j                        }g }t        |t        j
                  j                        r|j                         D ]r  \  }}t        j                  ||      }|j                   j                  || j                  |t        t        | j                  |                   |j                  |       t |S t        |t        j
                  j                  t        j
                  j                   t        j
                  j"                  f      sJ t%        |             t'        |      D ]Z  \  }}|j                  |j                   j                  || j                  |t        t        | j                  |                         \ |S Nre   )r   r   rd   rN   rT   nn
ModuleDictrP   r1   buildregister_attr_or_moduler   r   re   append
ModuleListParameterList
Sequentialr/   	enumerate)r   r9   baseresultrh   r   name_varidxs           rI   unpack_var_sequencez$NNModuleVariable.unpack_var_sequence   sU   yy&&t7(*dEHH//0 $

 (f*00T:		11OO)-T*JK	 2  h'( M588&&(>(>@S@ST
 	4=	 
 %T? 	KCMM		11OO)-S*IJ	 2 	 r   rh   r7   c                    |j                   j                  | j                        }t        ||      }t	        | j
                  j                  t        j                  t        j                  |                   t        j                  ||      S )Nrt   )r   r   rd   r[   r   re   rz   r|   r}   r   HASATTRr1   r   )r   r9   rh   r:   r   s        rI   call_obj_hasattrz!NNModuleVariable.call_obj_hasattr  sk     ii%%doo6d#KK""9#4#4\5I5IPT#UV	
 $$R00r   c                 f    |j                   j                  | j                        }t        |dd      S )NtrainingF)r   r   rd   getattrr   r9   r:   s      rI   is_trainingzNNModuleVariable.is_training   s)    ii%%doo6sJ..r   c                     |j                   j                  | j                        }t        j                  |       |j
                  j                  dk7  rt        j                  t        |             t        )zIRestart analysis treating this module as an UnspecializedNNModuleVariabler   )
r   r   rd   r   tagf_codeco_namemark_class_dynamicrM   r   r   s      rI   convert_to_unspecializedz)NNModuleVariable.convert_to_unspecialized$  sT    ii%%doo6c" 99
*00c;))r   r   
obj_sourcec                    t        |      rwt        j                  t        |      d      }|rt	        |d      nd}	 t        j                  || |      j                  |t
        j                  j                  |      gi       S t!        |d
      }|yt#        |t$        j&                        s&t        dd|  d| ddgt        j                  	       dt	        |d      i}t        j                  || fi |j                  |t)        j*                  ||      gi       S # t        $ r t        |       Y t        $ r) t        dd|  d| ddgt        j                  	       Y w xY w)zDCheck for a __getattr__ and handle it specially if it is implemented__getattribute__Nr   z5Custom __getattribute__ in nn.Module attribute accessvar_getattr  zXDynamo could not trace through the custom `__getattribute__` method on this `nn.Module`.z[Simplify your `__getattribute__` implementation, or replace it with a targeted `@property`.gb_typecontextexplanationhintsT)ignore_nn_module_getattrz6torch.nn.Module with a non-function custom __getattr__zDynamo detected a nn.Module object with a custom `__getattr__` method, but this method is not a standard Python function (e.g., it might be implemented in C/C++). Dynamo cannot currently trace into such non-standard `__getattr__` methods.zAvoid using objects with non-standard __getattr__ methods within the compiled region. If possible, implement __getattr__ as a standard Python function.re   __getattr__)r(   inspectgetattr_staticrM   r   r   UserMethodVariablecall_functionr7   r   r   r   r   r   r   SUPPORTABLEr    rN   rx   FunctionTyper1   r   )	r   r   r9   rh   r   getattribute_fn
new_source
getattr_fnr   s	            rI   _custom_getattr_fallbackz)NNModuleVariable._custom_getattr_fallback.  s    #4(%44T$ZASTO>H
:'9:d  33#%  -Y%?%?%F%Ft%L$MrR	S( (tL
*e&8&89P&tfAdV4-A '22	$ Z
MBC++JHHVV&&r4012
 	
Q * .)"- S*4&$8!EE +66
s   AD E%4.E%$E%c                    | j                   xr t        | j                   |      }|j                  j                  | j                        }t
        j                  |d      }d}t               }t        j                  |j                        D ]+  }|j                  |j                  j                                - | j                   s%t        dd|  d| dg t        j                          |dk(  r| j#                  |      S d }	||v r||   }	nYd|v r||d   v r||vr	|d   |   }	nAd	|v r||d	   v r	|d	   |   }	n-d
|v r||d
   v r	|d
   |   }	n	 t        j$                  ||      }	d}|dk(  rt1        | j                   |       |dk(  r$|s"t3        j4                  ||j                  |      S |rat3        j4                  ||	t7        |            }t9        |t:        t<        f      r)|j?                  t        | jA                         |             |S tC        |	tD              rk| j                   r,t        t        | j                   d      |      }t        |d      }tG        jH                  |	jJ                  |      jM                  || gi       S tC        |	tN              r>tG        jP                  |	jR                  tG        jT                  t-        |            |      S tC        |	tV              r&tG        jH                  |	jY                  |      |      S tC        |	tZ        j\                        rtG        jP                  |	| |      S t_        |	      sta        |	      r t3        j4                  ||	t7        |            S t        ddtc        |       d| dtc        |	       dtc        |	       dd| dtc        |	       dtc        |       ddgt        jd                         tf        |   ||      S # t&        $ rV | j)                  |||| j                         }
|
|
cY S t+        t&        |dt-        |      j.                   d| dg       Y w xY w)Nrv   Tzgetattr with no sourcer   r   zDynamo does not know how to access an attribute on an `nn.Module` instance that lacks a source. This is usually an internal error in Dynamo.r   _modules_parameters_buffersF)r   r9   rh   r   '' object has no attribute 'rZ   rr   rm   r   fgetz$Unsupported nn.Module attribute typeznn.Module subclass: z, name: z, attribute type: z>Dynamo does not support tracing nn.Module attributes of type ``zRefactor your code so that `z	` (type `z`) is not an attribute of `zqCurrently supported attribute types are methods, classmethods, staticmethods, properties, constants, and tensors.)5re   r   r   r   rd   objectr   rS   r   getmrorm   updaterv   keysr   r   
DYNAMO_BUGr   r   r]   r   r   rM   __name__r   r1   r   r   rN   r   UnspecializedNNModuleVariabler   r   r&   propertyr   UserFunctionVariabler   r   classmethodr   rs   r4   staticmethod__get__rx   r   r$   r%   r/   r   r   var_getattr)r   r9   rh   re   r   	base_dictobject_memberall_class_attribute_namesr@   subobjr   outrm   s               rI   r   zNNModuleVariable.var_getattrm  sO   >DKK!>yy&&t7 ++D*=	$'E!/ 	@A%,,QZZ__->?	@ {{0&tfAdV47 6)445 :##B''9t_F)#	*--55z*40Fi'DIm4L,L}-d3F9$:1F)Fz*40F //d; % 924;;E;}"((T^^FKK!''FN64JKC# 02OPQ ..t>>@$G J fh';;'
4;;(LdSF'7F 55KK!  -dHb12 , 33OO77T
C! 
 - 55NN4(!   2 23 33FDPP!&)Xf-=&,,R9OPPB274=/$Oabijpbqars"`ahioap`qqr s6tfIgfoEVVqryz~r  rA  AB  C> +66		
 w"2t,,Q " 66"4DKK 7  %!M("d4j1122MdVSTUV s   O) ),Q-QQr;   c                    |j                   j                  | j                        }t        | j                  | j	                         ||      5  t        |      }t        |t        j                  j                        r|j                  j                  t        j                  j                  j                  u rt        |      r| j                  |       |rJ d       |rt        |ddt        |       d       |\  }|j                   j#                         D ]m  \  }}|j%                  |j                   j'                  || j                  |t)        t+        | j,                  |                  |gi        |j/                         }o |cd d d        S |r+|j0                  |j0                  | _        t5        ||||       |j                   j7                         r|j8                  j;                  d      rw|j8                  dk7  rht        |dd	      r| j                  |       d
dlm}	  |	| |j                   j@                  d| j                  gtC        ||             cd d d        S t        |t        jD                  jF                        r#|j                  }
t+        | j,                  d      }n"|jH                  }
t+        | j,                  d      }tK        |
tL        jN                        r(|
jP                  }
t+        |d      }| gtS        |      z   }ntK        |
tL        jT                        sJ |jW                  tY        jZ                  |
|      ||      cd d d        S # 1 sw Y   y xY w)Nz3Expected lazy sequential isn't a valid combination?ztorch.nn.Module.Sequentialz0 kwargs kwargsr   )z	torch.nn.z	torch.ao.ztorch.nn.utils.parametrizeT)check_forward_hookscheck_backward_hooksr.   wrap_fx_proxycall_moduler9   proxyrr   
_call_implrs   ).r   r   rd   rp   r   r"   rN   rT   r   r   rm   rr   r'   r   r*   lenr   rP   r   r   r   r   re   popcls_to_becomer   rc   is_root_tracer
__module__
startswithbuilderr   create_proxyr)   rU   GraphModuler  r&   rx   ry   rs   rQ   r   inline_user_function_returnr   r   )r   r9   r;   r<   r:   is_lazyr`   
child_namer   r   fn	fn_sources               rI   r   zNNModuleVariable.call_function  s    ii%%doo6#OOT<<>C
 c	 %S)G3 3 34MM))UXX-@-@-H-HH%c* 11"5 # I{ '4"v;-w/	 *-,,*<*<*> #&J$$		99" OO&#1*T[[*2U#V	 :  	 &&(C# Oc	 c	R $$0'*'8'8D$
 'r3f= 		((*NN--.HINN&BB%T 11"52$0"))00% +48Oc	 c	` c588#7#78 B *4;;	 BIB *4;; EI"e../B *9j AI 6DJ.D!"e&8&899955222iHc	 c	 c	s   D;M"CM"*C.M""M+c                 T   ddl m} ddlm} |j                  j                  | j                        }t        j                  j                  j                  t        j                  j                  j                  t        j                  j                  j                  t        j                  j                  j                  t        j                  j                  j                  f}t        |      j                  |vr|j!                         r$t#        |j%                         t&        t(        f      st+        dd|  d| d| dd	g
       |j                  j,                  }t#        |t.        j0                        sJ t3        t3        | j4                  d      d      }|j7                  t9        j:                  ||      | |gi       S t#        ||      r.|j                  j<                  rg }	t?        tA        tC        |                  |j%                            }
tE        ||j%                                  D ]V  \  }}|
|   }tG        tI        | j4                  |            }|	jK                  |j                  jM                  |||             X ||j%                            }|j                  jM                  ||  dtG        tI        | j4                  |j%                                           }|S | jO                  |       d}t#        ||      r|jQ                  |j                        }n6|j!                         r|j%                         }nt+        dd|  d| dg 
       ||   }|j                  jM                  || j                  |tG        tI        | j4                  |                  S )Nr.   )SliceVariable)SymNodeVariablez6Invalid or non-const argument in nn.Module __getitem__zmp_subscript_impl: r   zFDynamo does not support calling method `__getitem__` of ``nn.Module`` z+ with a non-constant or non-(str, int) key.z9Use constant arguments of type str or int for __getitem__r   __getitem__rs   r   z.__getitem__(slice)r   z.Unsupported key type for nn.Module.__getitem__zEDynamo does not support getitem on `nn.Module` with non-constant key.))listsr  tensorr  r   r   rd   rT   r   r   r  r   ParameterDictr   r   rM   is_python_constantrN   as_python_constantstrintr   rs   rx   r   r   re   r  r   r   exportrQ   ranger  r   r   r   r   r   r   evaluate_expr)r   r9   keyr  r  modulebuiltin_supportedr  srcr   r   r   r   rX   
new_modulenew_module_variable	key_values                    rI   mp_subscript_implz"NNModuleVariable.mp_subscript_implO  s2    	)+((9 HH++HH++HH""..HH""..HH++
 <##+<<&&(s557#sDT1$q>!==CHDo!qVW ##,,Bb%"4"4555Z]CZPC11..r#>s  c=) yyE#f+./0F0F0HI#,VC4J4J4L-M#N 	KCS	A(t{{A)FGCMM		99"#& : 	 $C$:$:$<=
&(ii&G&Gf/0)%dkk33I3I3KL 'H '# +* --b1	c?+))"))4I##%..0IH-dV1SE:5 	"yy00OO!-Y"GH	 1 
 	
r   r   c                     ddl mm ddlm}  j
                  j                  j                        dt        dt        f fd}dv r j                        S dk(  r8t         j
                   j                               5   |      cd d d        S d	k(  rNt        j                  t        j                   j"                  j$                              r |j&                  d
      S dk(  r܉d   j)                         st+        d d d       t-        d         st+        d d d       d   j.                  d   j0                     }t-        |t2              r|S |j
                  j                  j                        }	j                  j5                  |	t7        t9         j:                                    S |r7t=              }
j"                  j>                   d dtA        |
      S d; fddt        dtB        t        tD        f   ffd}dtF        tH        tD        tD        f      ddf fd}dt        dtD        ddf fd}dtJ        dt        dtJ        fddk(  rj                  jL                  jO                  tQ         j:                  d              sr&tS        d!tU               d"tU               d#       g }jW                         D ]  \  }	|jY                   ||	               |t[               $      S d%k(  rj                  jL                  jO                  tQ         j:                  d&             g } j\                  d<i  |d'd(      D ]  \  }|jY                   ||               |t[               $      S d)k(  rj                  jL                  jO                  tQ         j:                  d*             g } j^                  d<i  |d'd(d+      D ]  \  }|jY                   ||               |t[               $      S d,k(  rj                  jL                  jO                  tQ         j:                  d              g } j`                  d<i  |d-d'd+      D ]  \  }	|jY                   ||	               |t[               $      S d.k(  ryj                  jL                  jO                  tQ         j:                  d              sr&tS        d!tU               d"tU               d#        |jW                               S d/k(  rOj                  jL                  jO                  tQ         j:                  d               |ja                               S d0k(  rWj                  jL                  jO                  tQ         j:                  d&              | j\                  d<i  |d(            S d1k(  rWj                  jL                  jO                  tQ         j:                  d*              | j^                  d<i  |d(            S d2k(  rjsr&tS        d!tU               d"tU               d#       g }D ]'  }|jY                  t        jb                  |             )  |t[               $      S d3k(  r@sr&tS        d!tU               d"tU               d#        |j/                               S dk(  rnsr&tS        d!tU               d"tU               d#       g }j/                         D ]  \  }	|jY                   ||	               |t[               $      S d4k(  r!  je                        t[               $      S d5k(  rt-        tf        jh                  jj                  tf        jh                  jl                  f      rHrFd   j)                         r3t        jb                  d   jo                         jp                  v       S d6k(  szt-        tf        jh                  jr                  jt                  jv                        rd7k(  s=t-        tf        jh                  jr                  jt                  jx                        rpd8k(  rkt=              jz                  }
tQ        tQ         j:                        d9      }j}                  t        j                  |
|       gt              z         S j"                  j                  v r^t        j"                  j                           r<t        d: t        j                  j                               D              r |      S t         %  t                    S # 1 sw Y   	%xY w)=Nr.   )ListIteratorVariableTupleVariabler6   rh   r=   c           	          	j                   j                  dj                  di       }t        |j                         t              \  }}ddlm}  |		j                   j                  d| |g||            S )Nget_attrrD   r.   r   call_method)r;   r<   r  )r   r  rd   r+   rW   r)   r
  r   )
rh   	mod_proxyr^   r_   r   r;   r<   r!  r   r9   s
        rI   generic_call_method_helperz@NNModuleVariable.call_method.<locals>.generic_call_method_helper  s     		..	I innf5'8v'F$J. ii,,!#1j1'	 -  r   r  _wrapped_call_implrr   _check_input_dimT_get_item_by_idxz``nn.Module`` z's call method z# requires a constant index argumentr   z# requires a tuple as first argumentr   __resultc                      t        d t        j                   j                               D              s#t	        dd d d  d d d dg        y y )	Nc              3   <   K   | ]  }|j                           y wrC   )r  rF   r@   s     rI   rJ   zUNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_const.<locals>.<genexpr>  s      +,$$&   z&non-const argument in nn.Module methodcall_method: r   z(Dynamo does not support calling method `z` of ``nn.Module`` z with non-constant arguments.r   )all	itertoolschainvaluesr   )r;   r<   r!  rh   r   s   rI   assert_all_args_kwargs_constzBNNModuleVariable.call_method.<locals>.assert_all_args_kwargs_const  st     09fmmo0V  D+D64&$qI!#f$7x?\!^r   namesc                              t        	
      } t        j                  |      j                  D cg c]  }|j	                          c}i j                         D ci c]  \  }}||j	                          c}}}|j                          |j                  }| D ci c]  }|||   
 c}S c c}w c c}}w c c}w rC   )r   r   	signaturebindr  rP   apply_defaults	arguments)r@  r  r@   rX   rY   
bound_argsr;   r?  r<   r!  rh   s         rI   
get_kwargsz0NNModuleVariable.call_method.<locals>.get_kwargs  s    (*&B3**2.33378a1'')89?HA1a**,,HJ %%'#--J.34Az!}$44 9H 5s   B6%B;&CrP   zvariables.ListIteratorVariablec                     g }| D ]M  \  }}|j                  j                  j                  ||t         j                  |                         O  |t                     S )Nr   mutation_type)r   r   r   r   re   r0   )	rP   named_childrenrh   r   r)  
gen_sourcer   r   r9   s	       rI   wrap_valuesz1NNModuleVariable.call_method.<locals>.wrap_values  sv     57N % f%%II55-jd.KL	 6  (.>.@ r   objzvariables.TupleVariablec                      t        j                  |       j                  j                  || t	         j
                  |                   g      S r   )r1   r   r   r   r   re   )rh   rN  r*  rL  r   r   r9   s     rI   named_embedz1NNModuleVariable.call_method.<locals>.named_embed/  sX     #))"d3II55-jd.KL	 6 
 
r   re   c                     |j                  d      }|d   dk(  r| S t        |      dkD  r,|j                  d      }t        | |      } t        |      dkD  r,| S )N.r    )splitr  r  r   )re   rh   
name_splitr@   s       rI   rL  z0NNModuleVariable.call_method.<locals>.gen_source<  s[    CJ!}"j/A%NN1%#FA. j/A% Mr   rK  r   z0 args and 0 kwargsz
 args and r   rI  named_parametersr   prefixrecursenamed_buffersr   remove_duplicatenamed_modulesmemochildrenmodules
parametersbuffersr   r>  __iter____contains___get_abs_string_index_conv_forward_output_paddingrs   c              3   <   K   | ]  }|j                           y wrC   )	is_tensorr8  s     rI   rJ   z/NNModuleVariable.call_method.<locals>.<genexpr>  s     RaAKKMRr9  )r=   NrD   )JrS  r)  r*  r   r7   rd   r   r   r  r1   r   rp   r   r   is_torch_inline_allowedr   getfilerm   r2  r   r  r   rN   rP   r   r   r   r   r   re   r   r   r2   rO   r   r   rR   r	   guard_on_key_orderaddr   r*   r  rK  r   r0   rV  rY  r[  r   r   rT   r   r   r  r  r   r^  conv_ConvNd_ConvTransposeNdrs   r  r   r   rQ   rv   rw   r;  r<  r=  r>  r   r-  )r   r9   rh   r;   r<   r   r7   r/  mod_varr   r  rG  rM  rP  rK  rV  paramrY  buffernamed_modules_listr   tmpitems_resultr  r)  r*  r?  rL  r   r!  rm   s   `````                   @@@@@@rI   r-  zNNModuleVariable.call_method  sX	    	:.oo((-	S 	_ 	 	2 77 %%b$77Y (!@!@!BB 8 2$78 8
 %%+*M*MOOF,,==>+
 +#**400%%7--/ $VHOD6Ade d1g}5 $VHOD6Ade 1gmmDGMM2G'#@A$$CYY,,S1F9944%mDKK&EF	 5   &B&&//0$w?D/BdFKK
	 
			5s 		5tCH~ 		5 		5	E#s(O,	-	 	"	c 	 	0I 	 		v 	S 	V 	 ##II((,,ZZ-PQv#)4ykCK=@	 57N & 5 5 7 Af%%k$&?@A'.>.@  ''II((,,Z]-ST686v66  Xy1  Be !''D%(@AB ( 0@0B  _$II((,,ZZ-PQ35M 4 4 4 !Xy2DE! @f $$[v%>?@ (EUEWXX_$II((,,ZZ-PQ8: 4 4 4 !VX/AB! Ef #))+dF*CDE ("2B2D  ZII((,,ZZ-PQv#)4ykCK=@	 v44677YII((,,ZZ-PQv33566\!II((,,Z]-ST6v66OI9NOPPYII((,,ZZ-PQ3v33Lj6KLMMV^v#)4ykCK=@	 F >o33B<=>'>N>PQQXv#)4ykCK=@	 v||~..W_v#)4ykCK=@	 35L & ?f##Kf$=>?'DTDVWWZ'((,<L<N  N"6EHH$7$79O9O#PQQ**,"((DG..0FOOC  ++6588#3#3#8#8#@#@AO+ 6588#3#3#8#8#I#IJ-- &//B":dkk4#@*MI11..r)Dd#  F$$---))22489R9??4+QRR-d337&r4dVDD[8 8s   ggc                     |j                   j                  | j                        }t        j                  |t        |            S )z<Sequence length for container modules (e.g., nn.Sequential).)r   r   rd   r1   r   r  )r   r9   r!  s      rI   	sq_lengthzNNModuleVariable.sq_length  s1    ((9$$RV55r   )r9   r5   r=   r8   )r9   r5   r=   N)r9   r5   r   r1   r=   r1   )F)r9   r5   r=   r1   )$r   r  __qualname__r1   _nonvar_fieldsrM   r  rT   r   Moduler   r   r   r	   r   r   r   r   r   r   r   rQ   r   r   r   r   r   r   r   r   rO   r   r'  r-  rv  __classcell__rm   s   @rI   r   r      sq    	
 
	'	'N22-029>2TW2	2
F 
- -D - T  f 23 2 2#  	
   
&= $BW <1)1141	1/5 /$ /*=
hhoo=
 $=
 	=

 =
 
4	=
~s-5 s-S s-_ s-jk#k 'k S/)*	k
 
kZ`
#`
 `
 
	`
P ]E#]E ]E '	]E
 S/)*]E ]E 
]E~6r   r   c            
           e Zd Zdddhej                  Z	 dej                  j                  deddf fdZ	d	e
de
fd
Zde
fdZde
ddfdZeej                   dee   fd              Zdddee   f fdZdddee   deeef   defdZdddedee   deeef   def
 fdZdddedededz  fdZdddedef fdZdddedefdZ xZS )r   
value_typeis_state_mutatedr   r   r<   r=   Nc                     d|v r+t        |d   dd       }t        |      |u rt        |      |d<   t        |   dd|i| d| _        | j
                  | _        y )Nr}  r  r   FrD   )r   rM   r   r   r~  re   r   )r   r   r<   lazy_value_to_becomerm   s       rI   r   z&UnspecializedNNModuleVariable.__init__  sh    6!#*6,+?RV#W E{22
 (,E{|$/u// % '+kk#r   attr_sourcec                     |S rC   rD   r   r  s     rI   _wrap_sourcez*UnspecializedNNModuleVariable._wrap_source      r   c                 B    | j                   xs | j                  }|sJ |S rC   r   r   s     rI   r   z8UnspecializedNNModuleVariable.get_nn_module_stack_source  r   r   re   c                     || _         y rC   r   r   s     rI   r   z8UnspecializedNNModuleVariable.set_nn_module_stack_source  r   r   c                     t         j                  j                  j                  t         j                  j                  j                  t         j                  j                  j
                  h} t         j                  j                  j                  j                         D ch c]'  }t        |d      r|| vrt        |j                        ) c}S c c}w )N__code__)rT   r   ry  __setattr__r   __delattr__rv   r>  r[   idr  )	supportedr@   s     rI   _nn_module_method_idsz3UnspecializedNNModuleVariable._nn_module_method_ids  s    
 HHOO''HHOO$$HHOO''
	 XX__--446
q*%!9*< qzzN
 	
 
s    ,Cr9   r5   c                    	 t        j                  | j                  d      }|t
        j                  j                  j                  t
        j                  j                  j                  t
        j                  j                  j                  fv r7|j                  t        j                  ||      | gi       j                  |      S t        | 9  |      S # t        $ r}t        |d }~ww xY w)Nra  )r   r   r}  r]   NotImplementedErrorrT   r   r   ra  r   r   r  r1   r   r   r   )r   r9   r  erm   s       rI   r   z1UnspecializedNNModuleVariable.unpack_var_sequence  s    	-''DB HH((HH""++HH((
 
 11%%b"-  "!"%& w*2..'  	-%1,	-s    C 	C+C&&C+r;   c                 	   | j                   }t        |      r+|j                  |j                  | _        t	        ||||       t        |t        j                  j                        sd}t        | j                  |      }nd}t        | j                  |      }t        |j                  t        j                        rt        |j                  t        j                        r|j                  j                  t         u r|j                  j                  t"        u r| j%                  |d      sot'        j(                  |d      }t        |t        j*                        r>|j,                  }| j/                  |d      j1                         j3                         s| j/                  |d      j1                         j3                         s| j/                  |d      j1                         j3                         s| j/                  |d      j1                         j3                         sv| j/                  |d      j1                         j3                         sG| j/                  |d	      j1                         j3                         s|j/                  |d
      j3                         s|j/                  |d      j3                         s|j/                  |d      j3                         s|j/                  |d      j3                         s|j/                  |d
      j3                         sx|j/                  |d      j3                         sX|j/                  |d      j3                         s8|j/                  |d      j3                         sd}| j                  j4                  }| j6                  r| j9                  ||      }	nd }	t;        | j6                  |       | j6                  r.t=        t?        tA        |            | jC                         ||      n	tE               }
|
5  t        |t        j*                  t        jF                  jH                  f      sBtK        jL                  |||	d      }|jO                  || gtQ        |      z   |      cd d d        S tS        jT                  ||	      jO                  || gtQ        |      z   |      cd d d        S # 1 sw Y   y xY w)N__call__r  rr   _backward_hooks_backward_pre_hooks_forward_hooks_forward_hooks_with_kwargs_forward_pre_hooks_forward_pre_hooks_with_kwargs_global_backward_pre_hooks_global_backward_hooks_global_forward_hooks_global_forward_pre_hooksTre   realizer   )+r   r"   r  r}  rc   rN   rT   rU   r  r   r&   r  rx   ry   r  rs   r,   r-   has_key_in_generic_dictr   r   r   nn_modules_globals_vtr   r  r  rr   re   get_source_by_walking_mror   rp   r  r  r   r   jitScriptFunctionr1   r   r   rQ   r   r   )r   r9   r;   r<   r:   rh   r  forward_method
globals_vtre   ctxfn_vts               rI   r   z+UnspecializedNNModuleVariable.call_function-  s    jj#  ,"%"3"3"2sD&9#uxx334D$/BD$/B
 3<<!1!12s~~u'7'78%%)AA''+HH 00Y?$33CCN.%*<*<=55
$$R):;CCEIIK'',ABJJLPPR'',<=EEGKKM'',HIWYSU '',@AIIKOOQ'',LMWYSU "--b2NOSSU!--b2JKOOQ!--b2IJNNP!--b2MNRRT!--b2NOSSU!--b2JKOOQ!--b2IJNNP!--b2MNRRT$D00B;;33B=FF.t{{C@ {{ #BsG//1	  	  	b5#5#5uyy7O7O"PQ'--b"VTR**2vT
/BFK	 	 !55bHVVd+V	 	 	s   A-S4SSrh   c                    |dv rpt        | j                  |      }| j                  r| j                  ||      }nd }t	        j
                  |||d      }|j                  || gt        |      z   |      S | j                  ||      s	 t        j                  t        | j                        |      }t        |t              rRt!        | j                  ||      d      }t	        j
                  ||j"                  |d      }|j                  |||      S t%        |d      rat'        |j(                        | j+                         v r<t-        dd|  d| d| d| d	| d
| j                   dgt.        j0                         |dk(  rd| j                  j2                  v r|d   j5                         }	|d   }
|
j7                         r*|
j9                         t:        j<                  j>                  u s|	| j                  j2                  d   v rd| _         no|	| j                  j2                  d   v rd| _         nLt        |
tB        jD                  tB        jF                  f      s|	| j                  j2                  d   v rd| _         |t:        j<                  jH                  jJ                  u rt        |d   tB        jL                        s&|t:        j<                  jH                  jN                  u rPt	        j
                  |t:        j<                  jH                  jN                        }|j                  || |d   g|      S tP        |   ||t        |      |      S # t        $ r d }Y w xY w)Nr0  Tr  rs   r  z,UnspecializedNNModuleVariable missing methodr:  r   z'Dynamo does not support tracing method z of nn.Module z@Dynamo does not really define unspecialized nn.Module very well.r   r  r   r   r.   r   r   )*r   r}  re   r  r1   r   r   rQ   r  r   r   rM   r   r]   rN   r   r   rs   r[   r  r  r  r   r   	DIFFICULTrv   r  rg  r   rT   r   	Parameterr~  r   r   r   ry  r  DeletedVariabler  r   r-  )r   r9   rh   r;   r<   r  re   r  method	attr_namer   rm   s              rI   r-  z)UnspecializedNNModuleVariable.call_method  s    77$/B{{77DA#))"bNE&&rD6DJ+>GG++B5 //TZZ0@$G &,/#222t<j (-- **2tV<< 
+v'4+E+E+GGJ+D64&$qI"I$~^b^h^h]i jZ*44	 }$$**:M:M)M !G668	Q
 OO%%*;*;*=ASAS*S$**"5"5m"DD,0D)$**"5"5j"AA,0D)%66%CC !DJJ$7$7
$CC -1D) %((//555tAw	(A(AB588??666 (--b%((//2M2MN**2d1gGGw"2tT$Z@@Q " s   
)M M&%M&fieldname_vtc                 ~    | j                  ||      }t        |t        j                        r|j	                  |      S y rC   )r   rN   r   ConstDictVariablemaybe_getitem_const)r   r9   r  r  r   s        rI   getattr_helperz,UnspecializedNNModuleVariable.getattr_helper  s:     ""2u-gy::;..w77r   c           
         |dv rj                   j                  j                  | |      st        | j                  |      }t        |t              rmt        |      dk(  r_| j                  r>t        | j                  |      }t        |j                  t        j                               t        j                  i       S | j                  r|dv r	j                   j                  j                  | |      st        | j                  |      }t        | j                  |      t        j                  t        j                                j                   j"                  j%                         dt&        dt(        dt(        dt*        t,        t,        f   ffdt        fd	t/        |      D              }t        j0                  |t3        |      
      S t4        | m  |      S )N)r  r  r  r  r  r  r   )r  r  r  r  irX   rY   r=   c                     t        j                  |      }t        |       }t        |      }t	        j
                  ||      }||fS rC   )r1   r   r   r   r3   r   )	r  rX   rY   r   
source_keysource_valuer   hooks_dict_sourcer9   s	          rI   build_key_valuezBUnspecializedNNModuleVariable.var_getattr.<locals>.build_key_value  sO     &++B2 00A1E
01BJO+221lCEz!r   c              3   :   K   | ]  \  }}} |||        y wrC   rD   )rF   r  rX   rY   r  s       rI   rJ   z<UnspecializedNNModuleVariable.var_getattr.<locals>.<genexpr>$  s(      Aq!  1a(s   r   )r   side_effectshas_pending_mutation_of_attrr   r   rN   rO   r  re   r   r   rz   r   EMPTY_NN_MODULE_HOOKS_DICTr   r  SEQUENCE_LENGTHrj  rk  r  r   rR   r1   r   NNModuleHooksDictVariablerM   r   r   )	r   r9   rh   
hooks_dicthooks_sourcer   r  r  rm   s	    `    @@rI   r   z)UnspecializedNNModuleVariable.var_getattr  s    
 
 99))FFtTR$TZZ6
j$/C
Oq4H{{'1$++t'D%(33 , G G
 %66r:: KK II**GGdS T2J *4;; =+66|7S7STUII((,,->?"""#&"78"  A*M F
 66Z(1B  w"2t,,r   c           	      *   ddl m} |j                  ||      }| j                  |d|      }|| j                  |d|      }|| j                  |d|      }|7t	        t
        |dt        | j                        j                   d| dg       |J |S )	a  
        Dynamo tracing of nn.Module __getattr__ can be expensive if the model
        has deep submodule hierarchy. Since the __getattr__ is stable, we can
        directly look into the underlying datastructures. This saves a lot of
        compilation time.
        r.   )SourcelessBuilderr   r   r   r   r   rZ   )	r
  r  r   r  r   r]   rM   r   r   )r   r9   rh   r  r  r   s         rI    manually_trace_nn_module_getattrz>UnspecializedNNModuleVariable.manually_trace_nn_module_getattr.  s     	/#**2t4!!"mW=;%%b*g>C;%%b*g>C;$TZZ(1122MdVSTU 
r   ) r   r  rw  r4   rx  rT   r   ry  r   r   r	   r  r   r   r   r|   cacherS   r  r  rQ   r1   r   r   rO   r  r   r-  r  r   r  rz  r{  s   @rI   r   r     s     
#	1	1	N2ehhoo 2 2 2( 6 F 
- -D - __
3s8 
  
/&= /$BW /0U#U 'U S/)*	U
 
Un\A#\A \A '	\A
 S/)*\A 
\A|)25@O	4	B-5 B-S B-_ B-H)14	r   r   c                        e Zd ZdZdedefdZy)$UnspecializedBuiltinNNModuleVariablezg
    Differentiates between builtin nn modules (e.g. torch.nn.Linear) and user defined nn modules.
    r  r=   c                     |S rC   rD   r  s     rI   r  z1UnspecializedBuiltinNNModuleVariable._wrap_sourceP  r  r   N)r   r  rw  __doc__r	   r  rD   r   rI   r  r  K  s     6 r   r  c                   h     e Zd ZdZdej
                  j                  deddf fdZdedefdZ	 xZ
S )	FSDPManagedNNModuleVariablea  
    Tracing behavior: trace into submodules and treat them as Unspecialized, do not
    register parameters to the top-level, treat them as function inputs.

    Guards behavior: if 'skip_fsdp_guards', many guards that would be installed
    by a vanilla UnspecializedNNModuleVariable are simply dropped, on the basis
    that a user wrapping their model in FSDP(model) is already opting into a
    requirement to not modify internal model state, which would already break FSDP without
    compilation.
    r   r<   r=   Nc                 j    |j                  d      }|J d       t        |   dd|i| || _        y )Nre   zMFSDPManagedNNModule depends on having an accurate source to control guarding.r   rD   )rl   r   r   re   )r   r   r<   re   rm   s       rI   r   z$FSDPManagedNNModuleVariable.__init__a  sE    H%! 	
[	
! 	/u//r   r  c                     t        |t        t        f      s:t        j                  j
                  j                  rt        |      S t        |      S |S rC   )rN   r   r   rT   _dynamoconfigskip_fsdp_guardsr  s     rI   r  z(FSDPManagedNNModuleVariable._wrap_sourcej  sG    ,.IJ
 }}##44)+662;??r   )r   r  rw  r  rT   r   ry  r   r   r  rz  r{  s   @rI   r  r  U  s=    	ehhoo     r   r  )Wr  r|   r   r<  ri   rx   collections.abcr   r   
contextlibr   r   typingr   r   torch.nnrT   torch._guardsr	   rS  r   r   r   excr   r   r   r   r   r   r   guardsr   r   mutation_guardr   re   r   r   r   r   r   r   r   utilsr   r    r!   r"   r#   r$   r%   r&   r'   r(   r)   r*   r+   r,   r-   r   r/   r0   r1   	functionsr2   lazyr3   user_definedr4   torch._dynamo.symbolic_convertr5   r   r7   dictsr8   r   ry  rO   r  rc   rp   r   r   r   r  r  rD   r   rI   <module>r     s  .    	  . 2 %    8 8   1 .      " = < 3 % 3 D*)&&	& ?
#& o%&	&
 
&R ++#+)@+GLxx++ +( TM  % 	 F[6 [6|r$= rj+H "? r   