
    9jC'                     j   U d dl mZ d dlZd dlmZmZmZmZ d dlm	Z	 ej                  j                  Zej                  j                  Zej                  j                  Zej                  j                  Zej                  j                   ej"                  j                   ej$                  j                   ej&                  j                   ej(                  j                   ej*                  j                   ej,                  j                   ej.                  j                   ej0                  j                   ej2                  j                   ej4                  j                   ej6                  j                   ej8                  j                   ej:                  j                   ej<                  j                   ej>                  j                   hZ e!ejD                  jF                     e$d<   ejJ                  j                   ejL                  j                   ejN                  j                   ejP                  j                   ejR                  j                   ejT                  j                   ejT                  j                   ejV                  j                   ejX                  j                   ejZ                  j                   ej\                  j                   ej^                  j                   ej`                  j                   ejN                  j                   ejb                  j                   ejP                  j                   ej                  j                   ejd                  j                   ejf                  j                   ejh                  j                   ejj                  j                   hZ6e!ejD                  jF                     e$d<   ej<                  j                   ej>                  j                   ejV                  j                   hZ7e!ejD                  jF                     e$d<   e!jq                  e6e       Z9 G d d	      Z:y)
    )AnyN)_resolve_process_groupFakeWorkProcessGroupWork)tree_map_onlynon_functional_collectivesfunctional_collectivessync_opsc            
       >   e Zd Zej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  hZej                  j
                  ej                  j
                  ej                  j
                  ej                   j
                  ej"                  j
                  ej$                  j
                  ej&                  j
                  ej(                  j
                  hZej.                  j
                  ej                  j
                  ej0                  j
                  ej2                  j
                  ej4                  j
                  ej6                  j
                  ej8                  j
                  ej:                  j
                  ej8                  j
                  ej>                  j
                  h
Z ejB                  j
                  ejD                  j
                  ejB                  j
                  ejF                  j
                  ejF                  j
                  ejH                  j
                  ejJ                  j
                  e&jN                  j
                  hZ(ejR                  j
                  hZ*ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                   j
                  ej                  j
                  ej$                  j
                  ej&                  j
                  ejH                  j
                  ejJ                  j
                  h
Z+ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej"                  j
                  ej(                  j
                  ej                  j
                  hZ,ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej                  j
                  ej"                  j
                  ej                  j
                  ej                  j
                  ej4                  j
                  ej6                  j
                  h
Z-ej                  j
                  ej$                  j
                  hZ.ej8                  j
                  ej8                  j
                  hZ/ej                  j
                  ej.                  j
                  ej                  j
                  ej0                  j
                  ej2                  j
                  ejB                  j
                  ejB                  j
                  hZ0ejF                  j
                  ejF                  j
                  e&jN                  j
                  hZ1ej>                  j
                  ejD                  j
                  hZ2e3de4de5fd       Z6e3de7fd       Z8e3de5fd       Z9e3de:fd       Z;y)CollectiveOpargreturnc                 r    ddt         j                  ddffd}t        t         j                  ||        S )z?Calculate total memory consumed by the tensors in the argument.r   tr   Nc                 H    | j                         j                         z  y )N)untyped_storagenbytes)r   total_memorys    i/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/distributed/_tools/fake_collectives.py	sum_bytesz+CollectiveOp.sum_tensors.<locals>.sum_bytes   s    A--/6688L    )torchTensorr   )r   r   r   s     @r   sum_tensorszCollectiveOp.sum_tensors   s9     	9 	9$ 	9 	ellIs3r   c                    | t         j                  v rt        j                  |d         S | t         j                  v rt        j                  |d         S | t         j
                  v rt        |d         S | t         j                  v rt        |d         S | t         j                  v rt        |d         S t        d|  dt               )zKRetrieve the process group for collective operations, except `wait_tensor`.            Func  not found in )r   PG_ARG_1r   unboxPG_ARG_2PG_ARG_3r   PG_ARG_4PG_ARG_5	TypeErrorcollective_ops)funcargss     r   get_process_groupzCollectiveOp.get_process_group   s     <(((%%d1g..<(((%%d1g..<((()$q'22<((()$q'22<((()$q'22%v^N3CDEEr   c                 ^   | t         j                  v rt         j                  |d         S | t         j                  v rt         j                  |d         S | t         j                  v r|j                         j                         S | t         j                  v r!|d   j                         j                         S | t        j                  j                  u r!|d   j                         j                         S | t        j                  j                  u r8t        t         j                  |d         t         j                  |d               S | t        j                  j                  u rJt        |d   j                         j                         |d   j                         j                               S | t        j                  j                  k(  r!|d   j                         j                         S | t         j                   v rt         j                  |      S | t         j"                  v r@|d   j                         j                         |j                         j                         z   S | t        j$                  j                  u rt         j                  |d         S t'        d|  dt(               )zdCompute the communication tensor size, except for `wait_tensor`, `barrier`, and `monitored_barrier`.r   r   r   zUnknown function: z in )r   COMM_TENSOR_ARG_0r   COMM_TENSOR_ARG_1COMM_TENSOR_ARG_RESr   r   "COMM_TENSOR_SINGLE_UNTYPED_STORAGEc10d_reduce_scatter_base_default	alltoall_maxalltoall_base__c10d_functionalall_gather_into_tensor_outCOMM_TENSOR_RES_SUMCOMM_TENSOR_ARG_0_AND_RESbatch_p2p_opsr)   r*   )r+   resr,   kwargss       r   get_comm_tensor_sizez!CollectiveOp.get_comm_tensor_size   s8    <111++DG44<111++DG44<333&&(//11<BBB7**,33554--5557**,33554>>)))((a1<3K3KDQRG3T  4&&...Q'')002DG4K4K4M4T4T4V  #>>FFF8++-4466<333++C00<9997**,3358K8K8M8T8T8VVV#11999++DG44,TF$~6FGHHr   c                     | t         j                  v rt        j                  |      S | t         j                  v rt        j                  |d         S t        d|  dt               )Nr   r!   r"   )r   WKr   r$   WK_ARG_1r)   r*   )r+   r?   s     r   get_workzCollectiveOp.get_work  sW    <??">>#&&\***>>#a&))%v^N3CDEEr   N)<__name__
__module____qualname__r4   
broadcast_r6   
allreduce_reduce_sendrecv_recv_any_source_barrierr#   
allgather__allgather_base_reduce_scatter_r5   gather_scatter_r7   r9   r%   r:   	broadcast
all_reduceall_reduce_all_reduce_coalescedall_reduce_coalesced_all_gather_into_tensorr;   _c10d_functional_autograd all_gather_into_tensor_coalescedr&   reduce_scatter_tensorreduce_scatter_tensor_coalescedall_to_all_singleisendirecv_dtensorshard_dim_alltoallr'   r>   r(   rD   rC   r0   r1   r2   r3   r=   r<   staticmethodr   intr   r   r-   rA   r   rE    r   r   r   r   M   sE    			

%%	H 	%%$$""**##H 	""**##++##++$$,,--55..66//7733;;!88@@99AAH 	..6688@@!77??**22!33;;&&&&##++	H 	&&..H
 	$$""**%%&&&&H 			

%%##
B 			

%%--55..66" 	$$ 	//77!88@@ 	%%""**##++##++$$,,..66!77??*& 	**22!33;;##++! 	99AA88@@
 	 	 	 	 F F F I I IB Ft F Fr   r   );typingr   r   torch._C._distributed_c10dr   r   r   r   torch.utils._pytreer   opsr4   r:   r[   rb   rI   r6   rJ   rK   rL   rM   rN   rP   rR   r5   rQ   rS   rT   r7   r9   rO   monitored_barrier_r	   set_ops
OpOverload__annotations__rU   rV   rZ   r]   reduce_scatter_tensor_outr_   wait_tensorrW   rX   rY   r;   r\   r^   r`   ra   r>   rc   r
   r   unionr*   r   rf   r   r   <module>rs      s      . yy~~99-- !II?? 99 	OOOOLLIIJJ!!OO  &&!!LLMMNNLL##!: C

 5 56 & &&''++33**22..66&&..//77  ((  (())11**22//7755==44<<44<<33;;''""""""**''+6 EJJ112 2 	LL##  (((#ejj##
$  13MNAF AFr   