
    9jWI              	          U d dl Z d dlZd dlmZmZ d dlmZ d dlmZm	Z	 d dl
Z
d dlmZ d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d
dlmZmZmZmZmZmZmZ d
dl m!Z!m"Z"m#Z#m$Z$m%Z%m&Z& d
dl'm(Z(m)Z) dgZ*e+e,   e-d<   de#de#de.fdZ/de+e#   de.fdZ0de+e#   de+e#   de+e#   fdZ1de
jd                  defdZ3dedefdZ4dedede$fdZ5de,dede%fdZ6de,dedede%fd Z7de,de
jd                  de%fd!Z8de,d"efd#Z9d$ Z:d% Z;de,d&ed'e+e   de+e"   fd(Z<d)ede#fd*Z=de,d+ede+e%   fd,Z>dedefd-Z?de
jd                  de+e   fd.Z@de,d/ed0ede+e"   fd1ZAd)eBe,ef   defd2ZCd3ed4ed5ed6efd7ZDy)8    N)bisect_rightinsort)Callable)Anycast)_get_device_module)ShardMetadata)ShardedTensor)DTensor)%compute_local_shape_and_global_offset   )BytesStorageMetadataChunkStorageMetadataMetadataIndexSTATE_DICT_TYPESTORAGE_TYPESTensorPropertiesTensorStorageMetadata)LoadItemTypeReadItemSavePlanTensorWriteData	WriteItemWriteItemType)"_check_shard_metadata_pair_overlap+_shards_get_overlap_region_wrt_saved_tensor create_read_items_for_chunk_list__all__plan
other_planreturnc                 "   | j                   |j                   k7  ryt        | j                        t        |j                        k7  ryt        | j                  |j                        D ]%  \  }}|j                  |j                  k7  r y|j
                  }|j
                  }|j                  |j                  k7  s2|j                  |j                  k7  s|j
                  |j
                  k7  r y|j                  }|j                  }|r|r|s|r y|s|s|j                  |j                  k7  r y|j                  }|j                  }	|r|	r|s|	r y|s|	s|j                  |	j                  k7  s|j                  |	j                  k7  s& y y)a  
    Compare the two Save plans and return True if they are equal.

    Args:
        plan (SavePlan): First SavePlan to compare.
        other_plan (SavePlan): Second SavePlan to compare.

    Returns:
       True if the two plans are equal, False otherwise.
    FT)usablelenitemsziptypeindexfqnoffsettensor_datasizechunkoffsetssizes)
r   r    	plan_itemother_plan_itemplan_metadata_indexother_plan_metadata_indexr+   other_tensor_datar-   other_chunks
             l/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/distributed/checkpoint/planner_helpers.py_compare_save_plansr7   *   sn    {{j''' 4::#j..// '*$**j6F6F&G )!"	?>>_111'oo$3$9$9!  ##'@'D'DD"))-F-M-MM"((,E,K,KK  +++77 1 1,#4#9#99  %%E+11Kk5[ MM[%8%88{{k&7&77 S)!V     delta_plansc                 &    t        d | D              S )z
    Check if any delta plan is usable, indicating the plan has changed.

    Args:
        delta_plans (List[SavePlan]): A list of delta plans to check.
    Returns:
        True if any delta plan is usable, False otherwise.
    c              3   <   K   | ]  }|xr |j                     y wN)r#   ).0
delta_plans     r6   	<genexpr>z(_contains_usable_plan.<locals>.<genexpr>t   s     NJz/j///Ns   )any)r9   s    r6   _contains_usable_planrA   k   s     N+NNNr8   cached_plansc                     g }t        | |      D ]6  \  }}|r|j                  s|j                  |       &|j                  |       8 |S )ac  
    Merge a list of delta plans into a single plan.

    Args:
        cached_plans (List[SavePlan]): A list of cached plans.
        delta_plans (List[SavePlan]): A list of delta plans to merge. It can contain empty plans

    Returns:
        A single merged plan. If a delta plan is not usable, use the cached plan. Otherwise, use the delta plan.
    )r&   r#   append)rB   r9   merged_planscached_planr>   s        r6   _merge_delta_local_plansrG   w   sS     L#&|[#A ,Zj//,
+	, r8   tensorc           	          t        t        j                  dgt        | j	                               z        | j	                               S )Nr   r.   r/   )r   torchSizer$   r,   )rH   s    r6   _create_chunk_from_tensorrM      s5    

A3V[[]!334FKKM r8   shard_mdc                     t        t        j                  | j                        t        j                  | j                              S NrJ   )r   rK   rL   shard_offsetsshard_sizes)rN   s    r6   _chunk_for_shardrS      s3    

8112jj--. r8   sharded_tensorc                    | j                         j                  }t        |j                  |j                  |j
                  |j                  |j                        }t        t        |      || j                         j                        S )N)dtypelayoutrequires_gradmemory_format
pin_memoryr-   
propertiesr,   )metadatatensor_propertiesr   rV   rW   rX   rY   rZ   r   rS   r,   )rT   rN   shard_propertiesr\   s       r6   _sharded_tensor_metadatar`      s|     &..0BB!$$&&&44&44#..J x($$&++ r8   r)   c                    t        |j                  |j                  |j                        \  }}t	        j
                  |      t	        j
                  |      }}t        t        | |      t        j                  t        t        ||      t        j                  |j                               |j                                     S )NrJ   r[   r(   r'   r+   )r   shapedevice_mesh
placementsrK   rL   r   r   r   SHARDr   r   r   create_from_tensorto_localr,   )r)   rH   r/   r.   s       r6   _create_write_items_for_dtensorri      s    :f((&*;*;NE7 ZZ&

7(;7EC)  #& (::6??;LM
 r8   c                     t        j                  |j                        }t        t	        | |      t
        j                  t        ||            S )Nrb   )rK   rL   rQ   r   r   r   rf   r`   )r)   rT   rN   r.   s       r6   _create_write_item_for_shardrk      sB     jj//0GC)  ,^XF r8   c                 :   t        j                  dgt        |j                               z        }t	        t        | |      t        j                  t        t        ||j                               t        j                  |      |j                                     S )Nr   rJ   r[   rb   )rK   rL   r$   r,   r   r   r   TENSORr   r   r   rg   )r)   rH   r.   s      r6   _create_write_item_for_tensorrn      sq    jj!s6;;=112GC)!!#&wfkkmL'::6B
 r8   bytesc                 J    t        t        |       t        j                        S )N)r(   r'   )r   r   r   BYTE_IO)r)   ro   s     r6   _create_write_item_for_bytesiorr      s     C "" r8   c           
          t        t        j                  | t        j                  |f      |t        j                  |f      t        j                  |f            S N)r'   
dest_indexdest_offsetsstorage_indexstorage_offsetslengths)r   r   rq   rK   rL   ru   dest_offsetrw   storage_offsetlengths        r6   _create_read_item_for_byteior~      sK     !!ZZ/#

N#45

F9% r8   c           
          t        t        j                  | t        j                  |      |t        j                  |      t        j                  |            S rt   )r   r   rm   rK   rL   ru   rv   rw   rx   ry   s        r6   _create_read_item_for_tensorr      sD       ZZ-#

?3

7# r8   checkpoint_mdlocal_chunksc                    !" g }|j                   }|r|s|S t        |d   j                        }d}|dkD  rDd}t        |      D ]4   t	         fdt        j                  ||      D              }||kD  s1|} }6 |dk(  rdgt        |      z  "dgt        |      z  !nx|D 	cg c]0  }	|	j                  |   |	j                  |   |	j                  |   z   f2 c}	"|D 	cg c]0  }	|	j                  |   |	j                  |   |	j                  |   z   f2 c}	!t        t        t        |            "fd      }
t        t        t        |            !fd      }g }d}t        |
      }|D ]  }||   }!|   \  }}t        ||df      }|r|d	|= ||k  r1|
|   }||   }"|   \  }}||k\  rnt        |||f       |dz  }||k  r1|D ]  \  }}||   }t        ||      sg }g }g }t        ||
      D ]:  \  }}}}|j                  |       |j                  |       |j                  |       < |j                  t        t        | |j                  |      |t        | |j                  |      ||               |S c c}	w c c}	w )aW  
    Create a list of ``ReadItem`` based on the checkpoint and local chunks.

    This applies the resharding algorithm and computes the reads needed
    to satisfy ``local_chunks`` with a checkpoint described by ``checkpoint_md``.

    Args:
        fqn (str) : The state_dict FQN to pass to ``ReadItem``.
        checkpoint_md (TensorStorageMetadata): metadata for a given tensor
            from a checkpoint.
        local_chunks (List[ChunkStorageMetadata]): Local chunks that needs to be
            loaded.

    Returns:
        A list of ``ReadItem`` that will satisfy all input chunks.
    r   r   c              3   \   K   | ]#  }|j                      |j                     z    % y wr<   rJ   )r=   r-   dims     r6   r?   z3create_read_items_for_chunk_list.<locals>.<genexpr>   s0       c"U[[%55s   ),)r   r   c                     |    d   S Nr    )idxsaved_boundss    r6   <lambda>z2create_read_items_for_chunk_list.<locals>.<lambda>9      S)!, r8   )keyc                     |    d   S r   r   )r   local_boundss    r6   r   z2create_read_items_for_chunk_list.<locals>.<lambda>=  r   r8   N)saved_shardcurrent_shardr   )chunksr$   r.   rangemax	itertoolschainr/   sortedr   r   r   r   rD   r   r   )#r)   r   r   
read_itemssaved_chunksnum_dims	sweep_dimmax_sizedim_sizecsaved_sorted_indiceslocal_sorted_indicesactive_saved	saved_ptr	num_saved	local_idxlocal_chunklocal_start	local_endcutoffstorage_idxstorage_chunksaved_start	saved_end_rx   rv   ry   _dimoffset_for_saved_tensoroffset_for_current_tensorr}   r   r   r   s#                                   @@@r6   r   r      s   * "$J ''L|<?**+H I!|? 	 C &__\<H H ("#		  1}x#l"33x#l"33 "
 YYy!199Y#7!''):L#LM
 "
 YYy!199Y#7!''):L#LM

 "c, , "c, ,
 +-LI()I) 1	"9-!-i!8Yl[",=>WfW%)#.y9K(5M%1+%>"Ki'<)[!9:NI )# + 	NA{(5M5k=Q OLG =)
' ')  &&'>?##$=>v&
' ,,S+2E2EyQ!-"/]22K# %4#
)	'1d Q

s   #5J5J
state_dictc                    g }| j                         D ]  \  t        t              r|j                  t	                     2t        t
              r4|j                  fdj                         j                  D               vt        t        j                        r|j                  t                     |j                  t                      t        |      S )Nc              3   8   K   | ]  }t        |        y wr<   )rk   )r=   rN   r)   objs     r6   r?   z5_create_default_metadata_only_plan.<locals>.<genexpr>  s"       -S#x@s   )r%   
isinstancer   rD   ri   r
   extendr]   shards_metadatarK   Tensorrn   rr   r   )r   requestsr)   r   s     @@r6   "_create_default_metadata_only_planr   y  s    H$$& FSc7#OO;CEF]+OO  # > >  U\\*OO9#sCDOO:3DEF Hr8   objectc                 6   t        |d      r|j                  | |      S t        |t              r3|j	                         D cg c]  }t        | ||j                         c}S t        |t        j                        rt        | |      gS t        | |      gS c c}w )N__create_write_items__)hasattrr   r   r
   local_shardsrk   r]   rK   r   rn   rr   )r)   r   shards      r6   _create_write_itemsr     s    v/0,,S&99	FM	*  ,,.
 )fennE
 	
 
FELL	)-c6:;;.sF;<<
s   Bc                     t        | j                  | j                  | j                        \  }}t	        j
                  |      t	        j
                  |      }}t        ||      S rP   )r   rc   rd   re   rK   rL   r   )rH   r/   r.   s      r6   _create_chunk_from_dtensorr     sW    :f((&*;*;NE7 ZZ&

7(;7E r8   c                 J   t        | d      r| j                         }|S t        | t              r2| j	                         D cg c]  }t        |j                         }}|S t        | t        j                        rt        |       g}|S t        dt        |              c c}w )N__create_chunk_list__zMUnsupported Type, expecting one of [Tensor, DTensor, ShardedTensor] ,but got )r   r   r   r
   r   rS   r]   rK   r   rM   
ValueErrorr'   )rH   r   r   s      r6   _create_chunk_listr     s    v./335  
FM	*:@:M:M:O
16U^^,
 
  
FELL	)1&9:  V~'
 	

s   B mdr   c                     t        |t              s	 t        |      }t        | ||      S t        t        |       dt        |       dd      gS # t        $ r$}t        d|  ddt	        |       z         |d }~ww xY w)Nz Invalid checkpoint metadata for z, z(expected BytesStorageMetadata but found r   rz   )r   r   r   r   r'   r   r~   r   )r)   r   r   r   exs        r6   _create_read_itemsr     s    b./	-c2L 0RFF )(-+C0 
 	
  	23%r:<T"XJGH 	s   A 	A:A55A:c                 p    dt         fd}dt        fd}dt        j                  fd}t	        | |||       y)zP
    Initializes meta tensor if the meta tensor is DTensor or torch.Tensor.
    valuec                    t        | dd       }|t        j                  d      k(  rt        j                  j                         j                  }t        t        j                  t        |      j                               }t        j                  | j                         |      }t        j                  || j                  | j                  | j!                         | j#                               }|S | S )Ndevicemetar   )rd   re   rc   stride)getattrrK   r   distdistributed_c10d_get_pg_default_devicer'   r   r   current_device
empty_likerh   r   
from_localrd   re   r,   r   )r   r   device_typenew_local_tensordtensors        r6   dtensor_funcz&_init_state_dict.<locals>.dtensor_func  s    $/U\\&))//FFHMMK0=LLNF  %//0@P (( !-- ++jjl||~G NLr8   c                     t        | dd       }|t        j                  d      k(  rt        dt	        |        d      | S )Nr   r   zFound unsupported type z for meta device loading.)r   rK   r   RuntimeErrorr'   )r   r   s     r6   sharded_tensor_funcz-_init_state_dict.<locals>.sharded_tensor_func  sF    $/U\\&)))$u+6OP  Lr8   c                 4   t        | dd       }|t        j                  d      k(  rrt        j                  j                         j                  }t        t        j                  t        |      j                               }t        j                  | |      }|S | S )Nr   r   r   )r   rK   r   r   r   r   r'   r   r   r   r   )r   r   r   rH   s       r6   tensor_funcz%_init_state_dict.<locals>.tensor_func  s{    $/U\\&))//FFHMMK0=LLNF %%eF;FMLr8   N)r   r   rK   r   _iterate_state_dict)r   r   r   r   s       r6   _init_state_dictr     s@    
G *3 
5<< 
 	r8   iter_objectr   r   r   c           	      *   t        | t              r ||       S t        | t              r ||       S t        | t        j                        r ||       S t        | t
        t        t        t        t        j                  f      s| | S t        | t              r+| j                         D ]  \  }}t        ||||      | |<    | S t        | t        t        f      r8| D cg c]  }t        ||||       }}t        | t              rt        |      }|S yc c}w )a$  
    Iterate through the state dict, applying the given functions to each tensor type
    and update the state dict in place.

    Args:
        iter_object (Any): the target state_dict.
        sharded_tensor_func (Callable): the function to apply to ShardedTensor
        dtensor_func (Callable): the function to apply to DTensor
        tensor_func (Callable): the function to apply to Tensor

    # TODO: let state_dict_util._iterate_state_dict() to support in place option
    so we don't need to have two versions of _iterate_state_dict.
    N)r   r   r
   rK   r   intfloatstrro   ioBytesIOdictr%   r   listtuple)r   r   r   r   r   r   vrets           r6   r   r     s   ( +w'K((	K	/";//	K	.;'';eS% DE	K	&%++- 	JC2|%8+ K	 	K$	/ !
  <1DkR
 
 k5)*C
 
0
s   D)Er   r   bisectr   r   collections.abcr   typingr   r   rK   torch.distributeddistributedr   torch._utilsr   !torch.distributed._shard.metadatar	   'torch.distributed._shard.sharded_tensorr
   torch.distributed.tensorr   torch.distributed.tensor._utilsr   r]   r   r   r   r   r   r   r   plannerr   r   r   r   r   r   
reshardingr   r   r   r   r   __annotations__boolr7   rA   rG   r   rM   rS   r`   ri   rk   rn   rr   r~   r   r   r   r   r   r   r   r   r   r   r   r8   r6   <module>r     s   	  ' $     + ; A , Q    99c 9>h >H > >B	OtH~ 	O$ 	Ox.h 
(^2ell 7K } 1E !-:( g ) (	+7D
s 
ELL 
Y 
 C 

x	x(x +,x 
(^	xv? x "=S =# =$y/ =w 3G u|| 5I0J &
C 
] 
 
h 
.4c3h 4C 4n,,, ", 	,r8   