
    9jE                        d Z ddlZddlZddlmZ ddlmZmZ ddlZddl	m
Z
 ddlmZ ddlmZ ddlmZ d	d
lmZmZ d	dlmZmZmZmZmZmZ d	dlmZ ddlmZ ddlm Z  ddl!m"Z" ddl#m$Z$ ddl%m&Z& ddl'm(Z( ddl)m*Z* erddl+m,Z,  G d de-      Z. G d de-      Z/ ee0d      Z1dejd                  de3fdZ4 G d de*      Z5y) a
  
This module implements variable tracking for PyTorch optimizers during Dynamo tracing.

The OptimizerVariable class provides specialized handling for optimizer instances by:
- Optimizing the tracing of expensive optimizer initialization
- Managing optimizer state and parameter group tracking
- Handling tensor sources and guards for optimizer state tensors
- Supporting CUDA graph execution through static tensor address management
- Providing special handling for parameter gradients and optimizer state tensors

Key features include:
- Efficient initialization tracing via _init_group optimization
- Automatic marking of optimizer state tensors as static for CUDA graphs
- Proper source tracking for parameter groups, gradients, and state tensors
- Guard installation for optimizer state structure
- Support for both CPU and GPU tensor handling
- Cleanup of static tensor references via finalizers

The module integrates with Dynamo's broader tracing system while providing
optimizer-specific optimizations and safety guarantees.
    N)Iterable)AnyTYPE_CHECKING)TensorVariable)Source)getArtifactLogger)tree_map_only   )GuardBuilderinstall_guard)
AttrSourceConstDictKeySourceDictGetItemSourceGetItemSourceGlobalWeakRefSource
GradSource)GLOBAL_KEY_PREFIX   )VariableTracker)ConstantVariable)ConstDictVariable)HashableTracker)ListVariable)GetAttrVariable)UserDefinedObjectVariable)InstructionTranslatorc                       e Zd Zy)ArgMappingExceptionN__name__
__module____qualname__     a/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/_dynamo/variables/optimizer.pyr   r   9       r$   r   c                       e Zd Zy)GuardInstallExceptionNr   r#   r$   r%   r(   r(   =   r&   r$   r(   
perf_hintsxreturnc                    ddl m} | j                  ry || j                  j                  d      }t
        j                  j                  j                  |       d u}|r-|j                  J |xs |j                  j                  |       S |S y)Nr   )get_managerFT)torch._inductor.cudagraph_treesr-   is_cudadeviceindextorch_dynamoutilsget_static_address_typecurrent_node_is_cuda_graph_recorded_tensor)r*   r-   manageris_static_addresss       r%   _is_static_for_cudagraphsr:   D   s    ;yyahhnne4!MM//GGJRVV''333! J''FFqI
 %$ r$   c                       e Zd Zdddhej                  Z	 	 	 ddej                  j                  dee	e
f   dz  dee   dz  deej                  ef   dz  de	ddf fdZd	d
dedee   deeef   ddf
 fdZd	d
dedef fdZddZddZde	de	deee	   eee	f   f   fdZddZddZd	d
dej                  defdZd	d
dee   de	dee	   de	ddfdZddZ xZS )OptimizerVariablegrad_to_sourcetensor_to_sourcestatic_tensor_namesNvaluekwargsr+   c                     t        |   |fi | || _        |xs i | _        |xs i | _        |xs
 t               | _        y N)super__init__r@   r=   r>   setr?   )selfr@   r=   r?   r>   rA   	__class__s         r%   rE   zOptimizerVariable.__init___   sG     	)&),1
,2 0 6B#6#?#% r$   txr   nameargsr   c                 `   |dk(  rt        | j                  d      st        
|   ||||      S 	 | j	                  |       | j                           | j                  |i |\  }} | j                  j                  |i |}| j                  |       | j                  |||||       dt        | j                         }|j                  || j                         | j                  |       t        j                  |      S t        
|   ||||      S # t        t         f$ r
}	Y d}	~	&d}	~	ww xY w)zVThis is an optimization to avoid tracing the very slow initialization of the optimizer_init_group__optimizer_N)hasattrr@   rD   call_methodgraph_break_if_pending_mutationmove_step_if_cpuget_python_argsrM   map_sources_and_install_guardsupdate_list_argsidstore_global_weakref_by_idcreate_finalizerr   creater   r(   )rG   rI   rJ   rK   rA   py_args	py_kwargsret_valmangled_name_rH   s             r%   rP   zOptimizerVariable.call_methodn   s*    = 4::}5w*2tT6BB44R8%%'%9T%9%94%J6%J"0$**00'GYG33B7%%b$K ".bn-=>--lDJJG%%b) (..w77
 w"2tT6::	 ()>? s   CD D-(D-c           
      ^   |dv rN| j                   sJ t        | |t        t        | j                  |            t        | j                   |            S |dk(  rFddlm} | j                  j                  D ]  }|d   D ]  } ||d         | j                  |       t        | -  ||      S )	NrM   )py_typesourceparam_groupsr
   mark_static_addressparamsTguard)ra   r   typegetattrr@   r   
decoratorsrd   rb   _set_capturablerD   var_getattr)rG   rI   rJ   rd   groupprH   s         r%   rl   zOptimizerVariable.var_getattr   s     M";;;"WTZZ67!$++t4	  >!800 7x 7A'677   $w"2t,,r$   c           	         | j                   j                  D ]t  }|d   D ]j  }|j                  j                  }|j                  j                  t        |      d       }|sA|j                  |      sSddlm	}  |dd| d| dg        l v y )	Nre   r
   )unimplementedz(optimizer: pending mutation on parameterz
variable: z, parameter: zSPending mutations on a parameter (e.g. due to using closure) require a graph break.)gb_typecontextexplanationhints)
r@   rb   outputside_effectsid_to_variablegetrV   has_pending_mutationexcrp   )rG   rI   grn   rv   variablerp   s          r%   rQ   z1OptimizerVariable.graph_break_if_pending_mutation   s    
 (( 	Ax[ !yy55'66::2a5$G A A( K3! J",XJmA3 G$y 		r$   c                      ddl m} dt        t        t        f   dt
        f fd} j                  j                  D ]  } ||      sd|d<     j                  xr t         j                  d      }|j                  t        j                  | j                  j                  |            }|j                  D ]B  }t        t        j                   d            }t        j                   d      |j                  |<   D y )	Nr   LazyVariableTrackerrm   r+   c                     d}d}| j                  dg       D ]:  }||j                  xs |j                  z  }||j                  j                  vz  }< d| v xr |xr |S )NTre   
capturable)rx   r/   is_xpur@   state)rm   all_uninitializedall_gpurn   rG   s       r%   safe_to_set_capturablezAOptimizerVariable._set_capturable.<locals>.safe_to_set_capturable   sq     $GYYx, ?19900!Qdjj.>.>%>>!?  5(J->J7Jr$   Tr   rb   ) r   dictstrr   boolr@   rb   ra   r   realize_allr   builditemsr   r   rY   )	rG   rI   r   r   rm   ra   param_groups_vtparam_group_vtkeys	   `        r%   rk   z!OptimizerVariable._set_capturable   s    )	K$sCx. 	KT 	K ZZ,, 	+E%e,&*l#	+ HDKK!H-99!!"djj&=&=vF
 .33 	FN!"2"9"9,"GHC(8(?(?(EN  %	Fr$   c                      dt         dt         f fd}|D cg c]
  } ||       }}|j                         D ci c]  \  }}| ||       }}}||fS c c}w c c}}w )z9Get python values equivalent to the variable tracker argsargr+   c                    t        | t              r | j                         r| j                         S t        | t              r| j
                  sg S t        | t              rt        | j                  t              rtt        | j                  j                  t              rP| j                  j                  j                  dk(  r-j                  j                  | j                  j                     S t        )Nrb   )
isinstancer   is_python_constantas_python_constantr   r   r   ra   r   baser   memberr@   rb   r1   r   )r   rG   s    r%   map_argz2OptimizerVariable.get_python_args.<locals>.map_arg   s    #/C4J4J4L--//C.syy	3 12szz=9szz
;JJOO**n<zz..szz/?/?@@%%r$   )r   r   )	rG   rK   rA   r   r   new_argskv
new_kwargss	   `        r%   rS   z!OptimizerVariable.get_python_args   sg    
	& 	& 	&  -11SGCL1106?1am?
?## 2?s
   AAc                     | j                   j                  j                         D ];  \  }}d|v s|d   j                  s|d   j	                  |j
                        |d<   = y )Nstep)r@   r   r   is_cputor0   )rG   rn   r   s      r%   rR   z"OptimizerVariable.move_step_if_cpu   sW    

((..0 	;HAu5=#7#7 %f 0 0 :f	;r$   c                    ddl m ddlm} i | _        i | _        dt        dd ffd}t        t        j                  || j                  j                         | j                  xr t        | j                  d      }|j                  t        j                   || j                  j"                  |            }| j                  xr t        | j                  d	      }t        j                   || j                  j                  |      }|j%                          |J |j&                  j(                  j+                  |       t-        | j                  j"                  |j.                        D ]  \  }}	t1        |d
         dkD  r|d
   D ]  }
|
j2                  d }t5        | j                  j                  j7                               D ]  \  }}||
u s|} n |sR|j                  t        j                   || j                  j                  |
   t9        |t;        ||                          n |	j=                  |t?        j@                  d
            }d}g }t-        |d
   |jC                  |            D ]  \  }}|j                  }|| j
                  |<   tE        |d      }|j2                  C|| j                  |j2                  <   tG        |j2                        rhd}|jI                  |       |tK        |jM                  tN        jP                                |rtR        jU                  tV        jX                        s|D cg c]  }|jZ                   }}tR        j]                  d|       
 t5        | j                  j                  j_                               D ]  \  }}t9        |t;        ||            }|j&                  j(                  j+                  |       t5        |j_                               D ]a  \  }}ta        |t        j                        s!|| j                  vs0|| j
                  vs?t9        |t;        ||            | j
                  |<   c  y c c}w )Nr
   rc   r   r~   r*   r+   c                      | d       y )NTrf   r#   )r*   rd   s    r%   mark_staticzEOptimizerVariable.map_sources_and_install_guards.<locals>.mark_static  s    .r$   rb   r   re   r   TgradF)zGrad tensors %s will be copied during cudagraphs execution.If using cudagraphs and the grad tensor addresses will be the same across runs, use torch._dynamo.decorators.mark_static_address to elide this copy.)1rj   rd   lazyr   r=   r>   r   r	   r2   Tensorr@   r   ra   r   r   r   r   rb   realizeru   guard_on_key_orderaddzipr   lenr   	enumeratekeysr   r   getitem_constr   rY   unpack_var_sequencer   r:   appendr   
make_guardr   CONSTANT_MATCHperf_hint_logisEnabledForloggingDEBUGrJ   warningvaluesr   )rG   rI   r   r   params_groups_sourcer   state_sourcestate_vtrm   group_vtparam	key_indexir   	params_vt
all_staticnon_static_gradsrn   p_vtparam_sourcegrad_sourcesrcnon_static_grad_namesidxr@   p_state_source	inner_idxr   rd   s                               @r%   rT   z0OptimizerVariable.map_sources_and_install_guards   s   4-  "	/3 	/4 	/ 	ellK1A1AB  ${{Vz$++~/V-99!!"djj&=&=?ST
 {{Gz$++w'G"((TZZ-=-=|L 	'''
		$$((6  #4::#:#:O<Q<QR 6	OE8 5?#a'"8_ "Ezz-$(	$-djj.>.>.C.C.E$F &DAq Ez,-	 %& %/;; / 5 5$&$(JJ$4$4U$;$5(4(:<(S%&!"	 "%"( !..r3C3J3J83TUIJ!uX	0M0Mb0QR W4#{{+7%%a(( 
 66%2=D''/4QVV<%*
(//<!+"8"89T9T"UVW$ -"<"<W]]"K=M(Nc(N%(N%%
 *_6	t $DJJ$4$4$;$;$=> 	JC.0sCN II((,,^< )%,,. 9 	1q%,,/!4!44!6!66/@&(:>9(U0D))!,	 )Os   Q"tensor_valuec                    ddl m} || j                  v rX ||d       | j                  |   }| j                  j	                  |j
                  j                  |j                               n|| j                  v r| j                  |   }ni ||d       |j                  t        |      }t        |      }| j                  j	                  |j
                  j                  |j                               t        j                  |||      S )z%Wrap state tensor in a TensorVariabler
   rc   Trf   )rj   rd   r>   r?   r   ru   module_key_namerJ   r=   rW   r   r   r   r   )rG   rI   r   rd   ra   global_names         r%   wrap_tensorzOptimizerVariable.wrap_tensorf  s     	5 4000D9**<8F$$(()B)B6;;)OPT000((6F  D9778I<XK(5F$$(()B)B6;;)OP$$Rv>>r$   rZ   r[   c           	      $   t        ||      D ]   \  }}t        |t              st        |t              sJ d       t	        |      D ]  \  }}	|j
                  j                  j                  |       t        |	t        j                        r,|j                  j                  | j                  ||	             q|j                  xr t        |j                  |      }
|j                  j                  t        j                   ||	|
               y)z7Update the args and kwargs to the traced optimizer callz-py_arg should be a list in optimizer variableN)r   r   r   listr   ru   rv   mutationr2   r   r   r   r   ra   r   r   r   )rG   rI   rK   rA   rZ   r[   r   py_argr   valra   s              r%   rU   z"OptimizerVariable.update_list_args  s     tW- 	QKC#|,!&$/ C/ (/ QFAsII**33C8!#u||4		(()9)9"c)BC!$!Lcjj!0L		(()>)>r3)OPQ	Qr$   c                     | j                   | j                  |j                  j                  dt        j
                  j                  dd ffd}|j                  j                  |       y )Ngmr+   c                 D     d fd}t        j                  |       y )Nc                     D ]  } j                   j                  | d        j                  j                  | d        j                  rj                  j	                          j
                  snj
                  j	                           y rC   )_bufferspop_parametersparams_flatclearparams_flat_unwrap_subclasses)rJ   r   names_to_deletetcs    r%   clear_static_tensor_refsz\OptimizerVariable.create_finalizer.<locals>.init_finalizer.<locals>.clear_static_tensor_refs  sl    + ADKKOOD$/NN&&tT2~~,,.7788>>@Ar$   r+   N)weakreffinalize)r   r   r   r   r@   s   ` r%   init_finalizerz:OptimizerVariable.create_finalizer.<locals>.init_finalizer  s    A U$<=r$   )r?   r@   ru   tracing_contextr2   fxGraphModuleadd_graph_finalizer)rG   rI   r   r   r   r@   s      @@@r%   rX   z"OptimizerVariable.create_finalizer  sW    22

YY&&
	>uxx33 
	> 
	> 			%%n5r$   )NNN)rI   r   r+   Nr   ) r    r!   r"   r   _nonvar_fieldsr2   optim	Optimizerr   r   r   rF   r   r   r   rE   r   r   rP   rl   rQ   rk   tuplerS   rR   rT   r   r   r   rU   rX   __classcell__)rH   s   @r%   r<   r<   W   s    
#	1	1	N 8</3>B@{{$$@ S*_-4@ !X_	@
 u||V34t;@ @ 
@";#"; "; ?#	";
 S/)*"; 
";H-5 -S -_ -0&F<$$$'$	tCy$sCx.(	)$>;
eN?)?9>?	?8Q#Q 'Q 	Q
 #Q Q 
Q,6r$   r<   )6__doc__r   r   collections.abcr   typingr   r   r2   torch._dynamo.variables.tensorr   torch._guardsr   torch._loggingr   torch.utils._pytreer	   guardsr   r   ra   r   r   r   r   r   r   r4   r   r   r   constantr   dictsr   hashabler   listsr   miscr   user_definedr   torch._dynamo.symbolic_convertr   	Exceptionr   r(   r    r   r   r   r:   r<   r#   r$   r%   <module>r      s   ,   $ %  9   , - 0  & ! & $ %  ! 3 D	) 		I 	 "(L9 $ &R61 R6r$   