9jHqddlZddlZddlmcmZddlmZddlm Z ddl m Z m Z ddl mZejeZde j$de j&d efd Zde j$de j&d d d dfd Zde j$d e j.fdZe eeefZde j.d dfdZde j.d efdZde j$ded d fdZde j$fdZ dZ!dZ"y)N) is_symbolic) OrderedSet)configir)Vxcomm_buffer_typereturnct|}t|tjryt|tjr|j sy|j }t|tjryt|tjryt|tjrt|jsyy)ze Check if an input can be realized as a comm buffer of the specified `comm_buffer_type`. TF) _get_data isinstancerLoopsBuffershould_allocateget_output_specCommBufferLayout FixedLayoutFlexibleLayoutr get_numel)r r datalayouts ]/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/_inductor/comm_lowering.pycan_realize_as_comm_bufferr7s Qfd}|j@fd}|jBfd}|jDfd}y#t$rt j dYywxYw)z4 Register lowerings for the comm subsystem. zRInductor support for distributed collectives depends on building torch.distributedNr)add_layout_constraintcloneconstrain_to_fx_stridescopy_register_loweringc&||Sr3)fnrQrSrUs rregister_comm_loweringz7register_comm_lowerings..register_comm_loweringsb"9: $$rr5r6rrr ct|||r t|||S|}tjrG|j t j jj|jtjj|}tjjjj ||||Sr3)rCrOr reorder_for_compute_comm_overlapr rr.no_fuse_buffer_namesr,r/r ExternKernelrequire_contiguous_AllReduce_Kernelcreate_inplace all_reduce_rN)r5r6rc10drRs r _all_reducez,register_comm_lowerings.._all_reduces 0Y K'Y C CCj  2 2 KKM GG ( ( , ,S\\^ <oo005 ++    $ $      rct|||r!|t|||}t||Stjj |}tj jjj||||Sr3) rCrOr1rr]r^r_r`rarN)r5r6rretrbrTs r _all_reduce_z-register_comm_lowerings.._all_reduce_s 0Y K$S)Z@C c "Joo005 ++    $ $      rc|Dcgc] }| }}tjjjj||||Scc}wr3r_CollectiveKernelr`all_reduce_coalesced_rN)inputsr6rr5rbrRs r_all_reduce_coalescedz6register_comm_lowerings.._all_reduce_coalesced sR(./%*// ++  & & . .      0sActtjjjj||||Sr3rh)rkr6rrbs r_all_reduce_coalesced_z7register_comm_lowerings.._all_reduce_coalesced_s6 ++  & & . .      rctjj||g|}t|tjsJtj j |Sr3)rricreate_out_of_placerIRNoderGrH)kernelrkargsr)s r_create_out_of_placez5register_comm_lowerings.._create_out_of_place"sH##77NN$ ***||""4((rcBjj|||Sr3)all_gather_into_tensorrN)r5 group_sizerrtrbs r_all_gather_into_tensorz8register_comm_lowerings.._all_gather_into_tensor's(#  ' ' / /      rc tjtjjtj j jj|||Sr3) rErFrrGrHrirp all_gather_into_tensor_coalescedrN)rkrwrrbs r!_all_gather_into_tensor_coalescedzBregister_comm_lowerings.._all_gather_into_tensor_coalesced0sM LL    4 455==    rcxtjjjj|||||SN)out)rrir`all_gather_into_tensor_outrN)r5rwrr~rbs r_all_gather_into_tensor_outz._all_gather_into_tensor_out<s> ++  + + 3 3    ,  rcDjj||||Sr3)reduce_scatter_tensorrN)r5r6rwrrtrbs r_reduce_scatter_tensorz7register_comm_lowerings.._reduce_scatter_tensorGs+#  & & . .       rcztjjjj||||||Sr})rrir`reduce_scatter_tensor_outrN)r5r6rwrr~rbs r_reduce_scatter_tensor_outz;register_comm_lowerings.._reduce_scatter_tensor_outQsA ++  * * 2 2     ,  rc tjtjjtj j jj||||Sr3) rErFrrGrHrirpreduce_scatter_tensor_coalescedrN)rkr6rwrrbs r _reduce_scatter_tensor_coalescedzAregister_comm_lowerings.._reduce_scatter_tensor_coalesced]sP LL    4 444<<    rcDjj||||Sr3)all_to_all_singlerN)r5output_split_sizesinput_split_sizesrrtrbs r_all_to_all_singlez3register_comm_lowerings.._all_to_all_singlejs+#  " " * *       rc|}tjjjj||||Sr3rrir` broadcast_rN)r5srcrrbrRs r _broadcastz+register_comm_lowerings.._broadcastts:Cj ++ OO # #S#z  rcttjjjj||||Sr3r)r5rrrbs r _broadcast_z,register_comm_lowerings.._broadcast_|s1 ++ OO # #S#z  rcttjjjj||||Sr3)rJrK_dtensorshard_dim_alltoallrN)r5 gather_dim shard_dimrrts r_shard_dim_alltoallz4register_comm_lowerings.._shard_dim_alltoalls5# II   1 1 9 9       rct|r|Stjjjj ||Sr3)r4r _WaitKernel create_wait wait_tensorrN)r5rbs r _wait_tensorz-register_comm_lowerings.._wait_tensors5 C J ""4#3#3#;#;SA rctjj|}jj||||Sr3)rr]r^isendrN)r5dsttagrrtrbs r_isendz'register_comm_lowerings.._isends4oo005#DJJ$6$6S#zRRrctjj|}tjj j j |||||Sr3)rr]r^rir`irecvrN)r5rrrrbs r_irecvz'register_comm_lowerings.._irecvsGoo005 ++ JJ  S#z  rc |Dcgc]!}tjj|#}}jj}t j j5tjj||||||\}}} } } ddd r J|d| t|D]H\} } | j| dk(st j j| jJ|dj}tjtj|||  }g}t!t||D]\}\} }}| dk(r|j"j%tj&tj(||||j*j%|j|j%|tj,tjj/||t0|fg}|j%tj2j5||Scc}w#1swYxYw)N rr)device)rr]r^ batch_p2p_opsrNrr. fake_moderiprocess_kernelzipr mark_buffer_mutatedr/ get_deviceMultiOutputLayout enumeratemutation_outputsappendMutationOutput NoneLayout alias_names MultiOutputtensor_to_layoutlistrGrH)op_list peer_listtag_listtensorsrtrrexample_output tensor_argsnon_tensor_argsunflatten_argsunbacked_bindingsop tensor_argrpackedresultsiex_out placeholderrbs r_batch_p2p_opsz/register_comm_lowerings.._batch_p2p_opss#BIJQ2??55a8JJ##++ WW   $$33   !  %E2C1D&EE$!';7 CNB    W}++J,?,?,AB C Q**,%%   /      "+C.,Q"R A AAvW}''..%%bmm6&BAvN""))!**,7q!!nn((99&AAYK r||22;?@ AaK  s&I +II)#rJrK_c10d_functional all_reduceAttributeErrorloginfoloweringrQrRrSrTrUrrGstrraall_reduce_coalescedrjrqrvrzrrrrr broadcastrrrrrrr)rYrcrfrlrnrxr{rrrrrrrrrrrrrtrQrbrRrSrTrUs @@@@@@@rregister_comm_loweringsrs ""--% 99 % %DDOO, \\C  -<D,,- \\C  .0D5567D6678)ryy) D778 9 DAAB  C  D;;<=D667 8 D::; < D@@A  B  D223 4 DNN+,DOO,- EII..AAB C D,,-.DJJ'S(SDJJ'(D../101G      s$J77KKc   tjjjddlm}|jjjj|jjjjddl m}dtj dtj"d d d tj fd dtj d t$d tj ffd |jdtj dt$d t$ffd }|jdtj dt$d t$dtj ffd }|jdtj dtj dt$d t$ffd }|jdtj dtj dt$d t$dtj f fd }|j&dtj dt$d t$ffd }|j(dtj dt$d t$dtj ffd }|j*dtj dt$d t$ffd }|j,dtj dt$d t$ffd } |j.dtj dt$d t$dtj ffd } |j0dtj dt$dt2d t$dtj f fd } |j4dtj d t$dtj ffd } |j6dtj d t$dt8dtj ffd } |j:dtj dtj d!tj d"tj d t$f fd# }|j< d+dtj dtj d!tj d"tj d t$f fd$ }|j>dtj dtj d%tj d"tj d t$f fd& }|j@ d,d'tj d(tj dt2d t$dt$f fd) }|jB d,d(tj d t$dt$ffd* }y#t$rt j dYywxYw)-zH Register lowerings for symmetric memory (symm_mem) operations. z7symm_mem ops not available, skipping symm_mem loweringsNr)register_out_variantr)rUr5r rrr c|jtjj|j |j |j |j}t||||S)z Fallback: insert a Pointwise identity copy allocated in P2P via CommBufferLayout. Used when we don't control the input's allocation. )rdtypeinner_fnranges) r r PointwiserHrr; make_loaderget_sizer#)r5r rcopys r_copy_input_to_comm_bufferz?register_symm_mem_lowerings.._copy_input_to_comm_buffers` ||"">>#--/__&<<> #  t%5zB rct|tjjr't |tjj||S|tjj|S)a Ensure inp is in P2P memory for a symm_mem collective. If inductor controls the buffer's allocation (ComputedBuffer, or any buffer with FlexibleLayout/FixedLayout), switch its layout to CommBufferLayout in-place, zero-copy. If inductor does not control allocation (e.g. InputBuffer), insert a Pointwise identity copy into a new CommBufferLayout buffer. This adds an extra Triton kernel. Returns the possibly new TensorBox. TODO(tianrengao): eliminate the extra kernel for static-shape InputBuffers by pre-allocating P2P memory in the wrapper and DMA .copy_() )rrr?r@r#)r5rrs r_maybe_realize_symm_memz._maybe_realize_symm_memsZ$ &c2+<+<+E+E F "3(9(9(B(BJ OJ-!!** rr6c ||}tjtjjtj j j j|||Sr3)rErFrrGrHrIrMrNr5r6rrrLs r_symm_mem_one_shot_all_reducezBregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce#s[ &c:6 LL      $ $,,44    rr~c ||}tjtjjtj j j j||||Sr3)rErFrrGrHrIone_shot_all_reduce_outrNr5r6rr~rrLs r!_symm_mem_one_shot_all_reduce_outzFregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce_out4s^&c:6 LL      $ $0088    r symm_buffer local_inputc ||}tjtjjtj j j j||||Sr3)rErFrrGrHrIone_shot_all_reduce_copyrN)rrr6rrrLs r"_symm_mem_one_shot_all_reduce_copyzGregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce_copyGs^.k:F  LL      $ $1199    rc ||}tjtjjtj j j j|||||Sr3)rErFrrGrHrIone_shot_all_reduce_copy_outrN)rrr6rr~rrLs r&_symm_mem_one_shot_all_reduce_copy_outzKregister_symm_mem_lowerings.._symm_mem_one_shot_all_reduce_copy_outZsa.k:F  LL      $ $55==    rc||}tjjjj||||Sr3)rrIrHtwo_shot_all_reduce_rNrs r_symm_mem_two_shot_all_reduce_zCregister_symm_mem_lowerings.._symm_mem_two_shot_all_reduce_oD &c:6    ) ) 1 1      routputc ||}tjtjjtj j j j||||Sr3)rErFrrGrHrItwo_shot_all_reduce_outrN)r5r6rrrrLs r!_symm_mem_two_shot_all_reduce_outzFregister_symm_mem_lowerings.._symm_mem_two_shot_all_reduce_out~s^&c:6 LL      $ $0088    rc||}tjjjj||||Sr3)rrIrHmultimem_all_reduce_rNrs r_symm_mem_multimem_all_reduce_zCregister_symm_mem_lowerings.._symm_mem_multimem_all_reduce_rrc ||}tjtjjtj j j j|||Sr3)rErFrrGrHrImultimem_one_shot_all_reducerNrs r&_symm_mem_multimem_one_shot_all_reducezKregister_symm_mem_lowerings.._symm_mem_multimem_one_shot_all_reduces[ &c:6 LL      $ $55==    rc ||}tjtjjtj j j j||||Sr3)rErFrrGrHrI multimem_one_shot_all_reduce_outrNrs r*_symm_mem_multimem_one_shot_all_reduce_outzOregister_symm_mem_lowerings.._symm_mem_multimem_one_shot_all_reduce_outs^&c:6 LL      $ $99AA    rrootc ||}tjtjjtj j j j|||||Sr3)rErFrrGrHrImultimem_one_shot_reduce_outrN)r5r6rrr~rrLs r&_symm_mem_multimem_one_shot_reduce_outzKregister_symm_mem_lowerings.._symm_mem_multimem_one_shot_reduce_outsa&c:6 LL      $ $55==    rc ||}tjtjjtj j j j|||Sr3)rErFrrGrHrImultimem_all_gather_outrN)r5rr~rrLs r!_symm_mem_multimem_all_gather_outzFregister_symm_mem_lowerings.._symm_mem_multimem_all_gather_outs[ &c:6 LL      $ $0088    rsplit_last_dimc ||}tjtjjtj j j j||||Sr3)rErFrrGrHrIreduce_scatter_outrN)r5rr rrrLs r_symm_mem_reduce_scatter_outzAregister_symm_mem_lowerings.._symm_mem_reduce_scatter_outs^&c:6 LL      $ $++33    r in_splitsout_splits_offsetsc||}||}tjjjj|||||yr3)rrIrHall_to_all_vdevrN)r5r~rrrrrLs r_symm_mem_all_to_all_vdevz>register_symm_mem_lowerings.._symm_mem_all_to_all_vdevsT&c:6%c:6    $ $ , ,       rc ||}||}tjjjj||||||yr3)rrIrHall_to_all_vdev_2drN)r5r~rrr major_alignrrLs r_symm_mem_all_to_all_vdev_2dzAregister_symm_mem_lowerings.._symm_mem_all_to_all_vdev_2dsW&c:6%c:6    ' ' / /       rin_splits_offsetsc||}||}tjjjj|||||yr3)rrIrHall_to_all_vdev_2d_offsetrN)r5r~rrrrrLs r#_symm_mem_all_to_all_vdev_2d_offsetzHregister_symm_mem_lowerings.._symm_mem_all_to_all_vdev_2d_offset'sT&c:6%c:6    . . 6 6       rin_tileout_tilec||}||}tjjjj|||||yr3)rrIrH tile_reducerN)rrrrr6rrLs r_symm_mem_tile_reducez:register_symm_mem_lowerings.._symm_mem_tile_reduce;sT*':>*8Z@    ( (       rct|D]\}}||||<||}tjjjj |||||yr3)rrrIrHmulti_root_tile_reducerN) in_tilesrrootsrr6rrrrLs r _symm_mem_multi_root_tile_reducezEregister_symm_mem_lowerings.._symm_mem_multi_root_tile_reduceOsr$H- GJAw1':FHQK G*8Z@    + + 3 3       rr3)r9)"rJrKrLrMrrrtorch._library._out_variantrrNrrrrrUrrGr?rrrrrrrintr rboolrrrr r#)rrUrrrrrrrrrrr rrrrr!r&rrrLs @@@rregister_symm_mem_loweringsr*s 99%% $$ A$$,,((00))11--55 , \\++C  & \\ 8x334 \\   5 x778 \\   \\  9 $x889 \\ \\    : $x<<= \\ \\    \\  > (x445  \\   6 x778 \\     9 $x445  \\   6 x<<= \\   > x@@A \\   \\  B $x<<= \\     \\  > (x778 \\  \\ 9 x223 \\     4 $x//0 \\ \\<<LL   1&x223  \\ \\<<LL   4*x99: \\ \\<<LL   ;&x++,  ,,   -&x667  ,,   8]  JKs&WW?>W?)#loggingrJtorch.utils._pytreeutils_pytreerEtorch._inductor.utilsrtorch.utils._ordered_setrrr virtualizedr getLogger__name__rrGr?r)rr#rqr tupler(rr+r1r4rCrOrr*rWrrr6s0 $$-/g!T ||')'8'8 <% ||%''%?% %P    "))  1Zc3h0288t8= =d= ?" bll ZzKr