9jddlmZddlZddlZddlZddlZddlZddlZddlZ ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlZddlZddlmZmZmZddlmZmZmZddlmZmZmZmZddlm Z m!Z!m"Z"ddl#Z#ddl$Z#ddl%m&Z&ddl'm(Z(dd l)m*Z*dd l+m,Z,m-Z-m.Z.m/Z/m0Z0m1Z1dd l2m3Z3dd l4m5Z5m6Z6m7Z7m8Z8m9Z9dd l:m;Z;ddle j~jddZAe"rddlBmCZCddlDmEZEmFZFmGZGddlHmIZIddlJmKZKddlLmMZMdZNe;eOdZPGddeQZRGddZSGddZTe*je*jzZWejGdd ZYejGd!d"ZZGd#d$eZZ[Gd%d&Z\Gd'd(Z]Gd)d*eZZ^Gd+d,e\e^Z_Gd-d.e]e^Z`Gd/d0eZZaGd1d2e\eaZbGd3d4e]eaZcGd5d6eZZdGd7d8e\edZeGd9d:e]edZfGd;de]eZZhGd?d@e\eZZiejdLdAZk dMdBZlGdCdDZmdNdEZndOdFZo dPdGZpGdHdIZqGdJdKZry)Q) annotationsN)CallableIterableSequence)FutureProcessPoolExecutorThreadPoolExecutor)byrefc_size_tc_void_pCDLL)AnyIO TYPE_CHECKING)get_interface_for_device) rand_strided)ir) CppCodeCache CUDACodeCache DLLWrapperget_hash PyCodeCache XPUCodeCache)Timer)do_bench_using_profiling get_gpu_typeget_ld_library_pathis_gpupython_subprocess_env)getArtifactLogger) OrderedSet.TORCHINDUCTOR_AUTOTUNE_POOL_INACTIVITY_TIMEOUT600) ModuleType) ChoiceCaller PartialRenderTritonTemplateCaller)config) benchmarker)VCUDA_VISIBLE_DEVICES autotuningc eZdZy)!NonzeroWorkspaceNotSupportedErrorN__name__ __module__ __qualname__`/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/_inductor/autotune_process.pyr/r/Kr5r/ceZdZdZeddZe d ddZeddZddZdZ ddZ ddd Z ddd Z ddd Z dd Zdd ZddZddZy) TuningProcesszF Class to launch and interact with a benchmarking subprocess. ctjdtjtjj t fd} |y#t$rYywxYw)z4 Entry point for the child process. z3Started autotune subprocess %s. Visible devices: %sc tj\}}|y |rtjj ||}tj |[#t $r }|}Yd}~'d}~wwxYwN)r9recvosenvironupdate Exceptionsend)job extra_envresulte read_pipe write_pipes r6workloopz,TuningProcess.process_main..workloop_sq!.!3!3I!>Y;  )))4 UF""6:6!Fs(A A3'A..A3N)autotuning_logdebugr>getpidr?getr,EOFError)rGrHrIs`` r6 process_mainzTuningProcess.process_mainTsQ  A IIK JJNN/ 0 7  J   sA A('A(NcTtj||f||jyr<)pickledumpflush)objrHrDs r6rBzTuningProcess.sendss#  S)$j1r5c,tj|Sr<)rQload)rGs r6r=zTuningProcess.recvzs{{9%%r5c2||_|jyr<)devicestart)selfrXs r6__init__zTuningProcess.__init__~s  r5ctjjtjjtd}tj \}}tj \}}tj |d|_tj |d|_tj|_ |jj|jtjtj|dtj dt#|dt#|g}it%dt't(j*rdndd }|j,t#|j,|t.<t1j2||||f |_tj6|tj6|d |_y ) z4 Start the benchmarking subprocess. z__autotune_main__.pywbrbz --parent=z --read-fd=z --write-fd=01)TORCH_WARM_POOLLD_LIBRARY_PATH3TORCHINDUCTOR_PROFILE_WITH_DO_BENCH_USING_PROFILINGN)envpass_fdsT)r>pathjoindirname__file__pipefdopenrHrG selectorsDefaultSelectorselectorregister EVENT_READsys executablerLstrrrr)/profile_bandwidth_with_do_bench_using_profilingrXr, subprocessPopenprocesscloserunning)rZentrysubproc_read_fdwrite_fdread_fdsubproc_write_fdcmdrds r6rYzTuningProcess.startsw RWW__X68NO$&GGI!$&GGI!!))Hd37D1!113  t~~y/C/CD NN  } %_-. /#./0 1    #%   #24EEDG   ;; "(+DKK(8C$ %!'' %'78  ! !" r5cV|jxr|jjduS)z: True if the subprocess is still running. N)ryrwpollrZs r6alivezTuningProcess.alives%||; 1 1 3t ;;r5c|js|jtj||j|y)z8 Push a work item to the child process. rDN)rrYr9rBrH)rZreqrDs r6putzTuningProcess.puts/zz| JJL39Er5c |jj|s"td|jjt j |j\}}t|tr||S#t$r|jt$r|jt$r<tjd|jj|jwxYw)z Get a response from the child process. Raises TimeoutError on timeout; raises EOFError if the subprocess crashes. zTimeout in autotune subprocess z.Unexpected exception in autotune subprocess %s)rnselect TimeoutErrorrwpidr9r=rGkillrNrxrArJ exception isinstance)rZtimeoutrE_s r6rMzTuningProcess.gets  ==''0"%DT\\EUEUDV#WXX%**4>>:IFA fi (L !  IIK   JJL    $ $@$,,BRBR  IIK   s AA55A7C,c|jr tjd|j|r|j yy)zC Signal the child process to shut down gracefully. N)rr9rBrHwaitrZrs r6shutdownzTuningProcess.shutdowns2 ::<   tT__ 5  IIK r5cx|jr|jj|jy)z5 Wait for the child process to exit. N)rrwrrxrs r6rzTuningProcess.waits& ::< LL    r5c|jj|jj|jjd|_y)z" Close resources. FN)rnrxrGrHryrs r6rxzTuningProcess.closes;    r5c|jrDtjd|jj|jj |j y)z6 Send a SIGKILL to the child process. z)Sending SIGKILL to autotune subprocess %dN)rrJerrorrwrrrxrs r6rzTuningProcess.killsF ::<  ;     LL    r5cH|jd|jy)z8 Gracefully restarts the child process. TrN)rrYrs r6restartzTuningProcess.restarts 4   r5)rG IO[bytes]rHrreturnNoner<)rTrrHrrDdict[str, str] | Nonerr)rGrrr)rX int | Nonerbool)rrrDrrr)g^@)rfloatrr)T)rrrrrr)r1r2r3__doc__ staticmethodrOrBr=r[rYrrrMrrrxrrr4r5r6r9r9Os<LP '4I  &&+Z< F6 r5r9cJeZdZdZddZed dZddZd dZ d dZ y) TuningProcessPoolz Maintains a pool of TuningProcesses to benchmark kernels in parallel across devices. By default, we create one TuningProcess per device and set the sub-process environment to make only that device visible. cV|j}tjd||Dcgc]}t|c}|_t j |_|jD]}|jj|tt||_ ycc}w)z, Start the child processes. z$Sub-process autotune device list: %srX max_workersN) get_device_listrJrKr9 processesqueueQueue process_queuerr lenexecutor)rZdevicesrXps r6r[zTuningProcessPool.__init__ s&&(CWMFMM6-v6M9> &A    " "1 % & +s7|D NsB&cltjsdgSt}t|}|j }t t jvrNt jt jdDcgc] }t|}}t||ksJ|Stt|Scc}w)zD Gather the list of devices to be used in the pool. N,) r)autotune_multi_devicerr device_countr,r>r?splitintrlistrange)gpu_typedevice_interfacecountdrs r6rz!TuningProcessPool.get_device_list s ++6M>3H= --/ 2:: -')zz2F'G'M'Mc'RS!s1vSGSw<5( ((NE%L!! Ts7B1c|jj|jD]}|jd|jD]}|jy)z5 Signal all child processes to exit. FrN)rrrr)rZrs r6rzTuningProcessPool.shutdown5sQ   #A JJEJ " # A FFH r5cR|jJddg}|Dcic])}|tjvs|tj|+}}|jj }|j |jj | |j tj|jj |Scc}w#t$rBtjd|dtdcY|jj |St$rl}tjd|dt|}d |vsd |vr|jtdcYd}~|jj |Sd}~wwxYw#|jj |wxYw) z Entry point for the thread-pool helper threads: Wait for an open TuningProcess, remove it from the queue, execute the benchmark in that subprocess, and return the TuningProcess to the queue. NTORCHINDUCTOR_CACHE_DIRTRITON_CACHE_DIRrzTimed out benchmarking choice 'z['. It will be ignored. Please debug the root cause in case the choice can bring perf gains.infzFailed to benchmark choice 'cudaErrorLaunchFailurecudaErrorIllegalAddress)bmreqr>r?rrMr benchmarkr)+max_autotune_subproc_result_timeout_secondsrwarningswarnrrArsr)rZchoiceenv_varsvrDrwprocess_exception error_msgs r6targetzTuningProcessPool.target@s ||'''-/AB/7K!1 ?Q 1 %K K$$((* FLL**i @ ,;;BB4    " "7 +=L MM1&:WW  < "    " "7 +! MM.vh7WW  -.I(I5, 9!<     " "7 +!    " "7 +sGCC C-F5F FAF F!F FF F&c xtt||jj|j|}|S)z> Benchmark each choice in a separate process. )dictziprmapr)rZchoicesresultss r6rzTuningProcessPool.benchmarkis/s7DMM$5$5dkk7$KLMr5Nr)rzSequence[int | None])rr'rrrzlist[TritonTemplateCaller]rz!dict[TritonTemplateCaller, float]) r1r2r3rr[rrrrrr4r5r6rrsC E&""( ',R +  + r5rcpeZdZUded<ded<ded<ded<d ed <d Zd ed <e ddZddZy ) TensorMetaz torch.devicerXz torch.dtypedtypeztorch._prims_common.ShapeTypesizesztorch._prims_common.StrideTypestridesroffsetNz str | Nonenamec ddlm}t|tr4|Dcgc]}|j |}}t d|DsJ|S|}t|t jrt jd|}|j}|J|j}|Jt||tjjj|j!tjjj||tjjj#|j%j&|j)Scc}w)Nr)#get_strides_with_layout_constraintsc3<K|]}t|tywr<)rr.0xs r6 z*TensorMeta.from_irnodes..sAQz!Z0Afake)rlayout)rXrrrrr) torch._inductor.select_algorithmrrr from_irnodesallrLayoutBuffer get_dtype get_devicerr+graphsizevarsoptimization_hintsget_sizeoptimization_hint get_layoutrget_name)clsirnodesrrrEnoderrXs r6rzTensorMeta.from_irnodess# Y gx (>E F!1!1!!4 FF FA&AA AAM dBII &99&6D    "!!!''""55dmmoFGG$$773D977##55doo6G6N6NO  !GsE&ct|j|j|j|j|j S)N)rXr extra_size)rrrrXrrrs r6 to_tensorzTensorMeta.to_tensors2 JJ LL;;**{{   r5)rz)LayoutOrBuffer | Sequence[LayoutOrBuffer]rTensorMeta | list[TensorMeta])r torch.Tensor)r1r2r3__annotations__r classmethodrrr4r5r6rr{sP   (( ++ KD* ? &  < r5rcxeZdZdZ d dZ d dZd dZdd d dZdd d dZy) BenchmarkRequesta1 Only handle triton template benchmark for now. The extern kernel benchmark can be done inside the same process since they usually don't cause crash. Important: Instances of this class and subclasses have to be serializable across process boundaries. Do not put CUDA Tensors in here! c ||_t|tr |g|_n||_rEttt fr/t dkDrtfdDsJd|_n|_||_ d|_ y)Nr(c3dK|]'}dD] }td|t||k(")yw))rXrrrrrN)getattr)rrattroutput_tensor_metas r6rz,BenchmarkRequest.__init__..sG Q.q148GAt.s!PA!++-!Prrz0Skipping op due to nonzero workspace requirementrz6InChildProcess %s: load %f, create tensor %f, bench %f)rJ isEnabledForloggingDEBUGtimer r rr rrr/inforrr*benchmark_gpu_with_cuda_graphrrKr) rZrrrKstart_tscreate_tensor_elapser load_elapseres bench_elapses r6rzBenchmarkRequest.benchmarks ++GMM: yy{H ;))d.E.E \ E}%* **!!P9O9O!PPM))335C #'99;#9 yy{H !!!=:c:B ))+0Kyy{H  ) );;B?C$--8]8C8C 99;1L  H$    11    R S<  sF )F54F5) r rsr rr rr Iterable[Any]rrrrrrrzCallable[[], None]rrrrtorch.Tensor | Nonerr) r1r2r3rr[rrrrr4r5r6rrs//9/: / " /  />"*"1=" "  $( "%"! "  "$(/$/!/  /r5rcNeZdZdZ d ddZdd ddZy) _TestBenchmarkRequestz Supports unit testing. Defined in this file instead of the test file so the TuningProcess sub-process can unpickle these objects. NcJ||_||_||_||_||_yr<)rErXsleepexccrash)rZrErXr2r3r4s r6r[z_TestBenchmarkRequest.__init__s'    r5rcr|jr?rMr,rsr2r#r3r4rqexitrErs r6rz_TestBenchmarkRequest.benchmark,sx ;; "::>>"6=T[[AQQ QQ :: JJtzz " 88((N :: HHQK{{r5)NNNF) rErrXrr2z float | Noner3zException | Noner4rr-)r1r2r3rr[rr4r5r6r0r0sq!" $        HL * 1D   r5r0c$eZdZdd ddZy)GPUDeviceBenchmarkMixinNrctdg||D}t|dks Jd|td|Dd}t|}t|dk(rtt |}n|j }|j |5tj||}|jddd|S#1swYSxYw)Nc3K|]i}t|tjrMt|jj r.|jj |jj kywr<)rtorchTensorrrXtypeindexrtensors r6rz3GPUDeviceBenchmarkMixin.do_bench..AsR$ &%,,/v}}))* ##/ MM  $ sA/A1r(zCan not mix devices c3K|]9}t|jjr|jj;ywr<)rrXr>r@s r6rz3GPUDeviceBenchmarkMixin.do_bench..Js4 &--,,- "" s?Acudar) r!rnextritercurrent_devicerXr*r synchronize) rZrrrdevice_idx_set device_typer device_idxr)s r6rz GPUDeviceBenchmarkMixin.do_bench;s $$ /M/3/$  >"a'P+??O)PP' +    4K@ ~ ! #d>23J)88:J  $ $Z 0 +'';?C  ( ( * +  + s (CC r-r1r2r3rr4r5r6r9r9:s* $( %!   r5r9c$eZdZdd ddZy)CPUDeviceBenchmarkMixinNrc,tj|Sr<)r* benchmark_cpurs r6rz CPUDeviceBenchmarkMixin.do_bench^s ((,,r5r-rKr4r5r6rMrM]s* $( -%-! -  -r5rMceZdZdZ d dfd Z ddZdZd dZxZS) TritonBenchmarkRequestz Represents a standalone benchmark request for a Triton Template. Important: Instances of this class have to be serializable across process boundaries. Do not put CUDA Tensors in here! ct|||||||_||_||_||_| |_| |_| |_| |_ | |_ ||_ ||_ yr<) superr[ module_pathmodule_cache_key num_stages num_warpsnum_consumer_groupsnum_buffers_warp_specmatrix_instr_nonkdim waves_per_eukpackworkspace_sizeworkspace_zero_fill)rZr r r rrTrUrVrWrXrYrZr[r\r]r^ __class__s r6r[zTritonBenchmarkRequest.__init__oso$ &79KZX& 0$"#6 %:"$8!( ,#6 r5c|tj|j|j}tj d|j|jt ||jj}t|j}|jsddl m }tj|jftj |j"}|j$r|j'|j)|}|||<d|j*_i} ddl} d| j1|j2vrd| d<|j"j4dk(rd} nP|j"j4} t7| } | j9|j:j"j(} t=t ||jtj>j@jBjDr!tGjH|g|||i| d| iStGjH|g|||i| | d d S) Nz"benchmark module key: %s, path: %sr)WORKSPACE_ARG_PLACEHOLDERrrXFwarmupcpustreamT)re benchmark_run)%rload_by_key_pathrUrTrJrKrr runrrr]rrar<emptyuint8rXr^zero_r?__self__with_bandwidth_infoinspect signature parametersr>rget_raw_streamr r _inductorruntimetriton_heuristicsDebugAutotuner functoolspartial)rZrrmod run_methodrraworkspace_tensorworkspace_index warmup_argrnrerIrs r6rz"TritonBenchmarkRequest.make_run_fns,**4+@+@$BRBRS 0  ! !    S$"2"2377 $//*    * R${{$$&kkzz   '' &&((../HIO*:J '27 /  w((4?? ?#(Jx ::??e #F**//K7 D %44''..44F  C)) * OO # # 5 5 D D $$      $$    " r5ctj|j|j}t ||j }|j |jdj|_yNr) rrgrUrTrr  precompile launchersn_regs)rZrxkernels r6rz!TritonBenchmarkRequest.precompilesV**4+@+@$BRBRSd../&&q)00 r5cTd|jd|jd|jS)Nself.kernel_name=z, self.module_path=z, self.module_cache_key=)r rTrUrs r6__str__zTritonBenchmarkRequest.__str__s2#$""$$8t'7'7&99RDE*E1=E EN1Ur5rQc eZdZy)TritonGPUBenchmarkRequestNr0r4r5r6rrr7r5rc eZdZy)TritonCPUBenchmarkRequestNr0r4r5r6rrr7r5rceZdZdZ d d fd Z d dZddd fdZddZdZdd Z xZ S)ExternKernelBenchmarkRequesta* A class to handle extern kernel benchmark requests. This allows extern kernels (like aten::mm) to be benchmarked in a subprocess, similar to Triton kernels. Important: Instances of this class have to be serializable across process boundaries. Do not put CUDA Tensors in here! Nc\t|||||||_|xsi|_||_yr<)rSr[ callable_pathkwargshas_out_variant) rZr r r rrrrr_s r6r[z%ExternKernelBenchmarkRequest.__init__s5 &79KZX*l .r5c|j}|jrtj|g|d|iStj|g|S)Nr) to_callablerrvrw)rZrrrs r6rz(ExternKernelBenchmarkRequest.make_run_fnsN      $$RA-ASA A$$R8-8 8r5rcR||jdk(ry|jstdk(rt|d|iS|j }|tt jjjj|t|jt|j|j||jrt!j"fdSt$j&rt)fdSt!jiS)Nrr7rcSr<r4algorsr6z8ExternKernelBenchmarkRequest.benchmark..s D-0r5cSr<r4rsr6rz8ExternKernelBenchmarkRequest.benchmark..s m8Lr5)numelrrrSrrr<_C_dynamoguardsassert_size_strider sizestridecopy_rr*r%r)rtr)rZrrout_newrr_s ` @r6rz&ExternKernelBenchmarkRequest.benchmark s ?syy{a/   3}#5#:7$m== =##%DM*G  ''::U388:.cjjl0C '"--"@@0EE/0LMM((}bA Ar5cyr<r4rs r6rz'ExternKernelBenchmarkRequest.precompile! r5cddlm}t||j}|jr t j |fi|jS|S)Nr)extern_kernels)rrrr rrvrw)rZrrs r6rz(ExternKernelBenchmarkRequest.to_callable%s@ D ^T%5%5 6 ;;$$R74;;7 7 r5c"d|jdS)NzExternKernelBenchmarkRequest())rrs r6rz$ExternKernelBenchmarkRequest.__str__1s.t/A/A.B!DDr5)NT)r rsr rr rrr+rrsrzdict[str, Any] | Nonerrrrr,)rrrr.rr) r1r2r3rr[rrrrrrrs@r6rrs)- $ / /9 /: / " /  /& / /  / 9* 91= 9  9RVB,  Er5rc eZdZy)ExternKernelGPUBenchmarkRequestNr0r4r5r6rr5 r5rc eZdZy)ExternKernelCPUBenchmarkRequestNr0r4r5r6rr;rr5rcheZdZdZ dfd Z ddZddZd dZxZS) SubgraphBenchmarkRequestz Benchmark request for subgraph choices. Pre-compiles the subgraph in the main process and stores the module path/cache key for loading in subprocess. cTt|||||||_||_||_yr<)rSr[rTrUsym_input_values) rZr r r rrTrUrr_s r6r[z!SubgraphBenchmarkRequest.__init__Is2 &79KZX& 0 0r5ctj|j|j|jfdS)Nc.jgSr<)call)rrxrsr6rz6SubgraphBenchmarkRequest.make_run_fn..^ssxx C"2 C] CDr5)rrgrUrTr)rZrrrxrs `@@r6rz$SubgraphBenchmarkRequest.make_run_fnXs5**4+@+@$BRBRS00DDr5cyr<r4rs r6rz#SubgraphBenchmarkRequest.precompile`rr5c<d|jd|jdS)NzSubgraphBenchmarkRequest(z, r)r rTrs r6rz SubgraphBenchmarkRequest.__str__ds&*4+;+;*PPQRRr5)r rsr rr rrr+rTrsrUrsrz list[int]rrr,rrrrs@r6rrAs 1 19 1: 1 " 1  1 1$ 1  1E*E1=E E Sr5rc eZdZy)SubgraphGPUBenchmarkRequestNr0r4r5r6rrhr7r5rc eZdZy)SubgraphCPUBenchmarkRequestNr0r4r5r6rrlr7r5rceZdZdZ d d fd ZdZ d dZddZdZddZ ddZ dd Z dd Z xZ S)CUTLASSBenchmarkRequestae A class to handle CUDA (CUTLASS) benchmark requests. This class is for managing the lifecycle of a CUDA kernel benchmark, including compiling the source code, managing workspace memory, and executing the kernel. Important: Instances of this class have to be serializable across process boundaries. Do not put CUDA Tensors in here! cLt|||||||_d|_d|_d|_d|_d|_d|_||_ |dk(rtnt|_ t||_|jj|jd\|_|_y)NrFxpuso)rSr[ source_coder] workspaceDLL_workspace_size_updatedhash_key source_filerIrr codecache_clsrrwrite)rZr r r rrrIr_s r6r[z CUTLASSBenchmarkRequest.__init__zs &79KZX&#$.2&*',$ "&-8E-A\} 8 E*.*<*<*B*B   d+ ' t'r5ctjd||jj|jdtjd|y)z Precompile the CUDA source code to populate the CUDACodeCache. This may happen in a separate thread pool. Precompiling %srDone precompiling %sN)rJrKrcompilerrs r6rz"CUTLASSBenchmarkRequest.precompilesB .5 ""4#3#3T:3T:r5c  |j|jt||gzDcgc]}t|j }}t j d|j|j|j|j||jt|jj|jj}t|j|j}td}|j dkDrht#j$|j dzdzt"j&|j(|_t|j*j }t-j.|g||jd||} ||Scc}w#t0$r,} t3|  fd} |j5| cYd} ~ Sd} ~ wwxYw)zg Create a function to run the CUDA/XPU kernel with the given input and output tensors. zqmake_run_fn: self.kernel_name=%s, self.source_file=%s, self.hash_key=%s, self.DLL=%s, args=%s, self.extra_args=%srrbNctr<) RuntimeError)err_msgsr6raise_runtime_errorz@CUTLASSBenchmarkRequest.make_run_fn..raise_runtime_errors"7++r5)ensure_dll_loadedupdate_workspace_sizerr data_ptrrJrKr rrrrrrqrFrr]r<zerosfloat64rXrrvrwrrsr) rZrrrAargs stream_ptrry workspace_ptrretrFrrs @r6rz#CUTLASSBenchmarkRequest.make_run_fns   ""$:>}:MQTPU:UV*+VV        MM HH  OO   ! ! 0 01F1F1U1U1W X TXXt'7'78      ""[[$$q(Q.mmzzDN %T^^%<%<%>?M   __         ' E [WH '!fG ,    !& & 's# F2)F77 G,!G'!G,'G,c |jry|jttj d|j D}t |dzDcgc] }td}}t|jj|jj}t|j|j}t}|g||jt!|d||jj#|j$|_t(j+d|j&|j|j,|j.|j||jd|_ycc}w)Nc34K|]}|jywr<)r)rmetas r6rz@CUTLASSBenchmarkRequest.update_workspace_size..sG$))Gsr(zupdate_workspace_size called: new workspace size=%d, self.kernel_name=%s, self.source_file=%s, self.hash_key=%s, self.DLL=%s, args=%s, self.extra_args=%sT)rrrrfromkeysr rr rrqrFrrr r rr rGvaluer]rJrKrr)rZunique_input_countrrrryc_workspace_sizes r6rz-CUTLASSBenchmarkRequest.update_workspace_sizesi  ' '    MMG0F0FG G )..@1.D(EF1FF  ! ! 0 01F1F1U1U1W X TXXt'7'78 #:  __          ))+.44 h          MM HH  OO (,$;GsFc|j9|jj|jd\|_|_|_|jj y)Nr)rrrVrrropenrs r6rz)CUTLASSBenchmarkRequest.ensure_dll_loadedsJ 88 8<8J8J8O8O  $9 5DHdmT%5  r5cl|j!|jjd|_d|_yr<)rrxrrs r6rz&CUTLASSBenchmarkRequest.cleanup_run_fns( 88  HHNN DHr5cTd|jd|jd|jS)Nrz, self.source_file=z, self.hash_key=)r rrrs r6rzCUTLASSBenchmarkRequest.__str__s0#$""$$8t'7'7&99JDMM;KLLr5cX|jj}d|d<d|d<d|d<|S)NrrFr)__dict__copyrZstates r6 __getstate__z$CUTLASSBenchmarkRequest.__getstate__ s7 ""$e !k+0'( r5c:|jj|yr<)rr@rs r6 __setstate__z$CUTLASSBenchmarkRequest.__setstate__s U#r5)rC)r rsr rr rrr+rrsrIrsrrr,rr)rdict[str, Any])rrrr)r1r2r3rr[rrrrrrrrrrs@r6rrps "  9 :  "       0;6*61=6 6p$,L M$r5rcZeZdZ dfd ZdZ ddZddZxZS)CppBenchmarkRequestcft|||||||_t||_d|_yr<)rSr[rrrr)rZr r r rrr_s r6r[zCppBenchmarkRequest.__init__s5 &79KZX& - -1r5ctjd|tj|jdtjd|y)NrrdrIr)rJrKrrVrrs r6rzCppBenchmarkRequest.precompile%s< .5$**>3T:r5c\tj|jd|_t ||gzDcgc]}|j }}t jd|j|j||jt|j|j}td|jDsJtjgt|tt |jzz|_t!j"|g||jScc}w)NrdrzJmake_run_fn: self.kernel_name=%s, self.DLL=%s, args=%s, self.extra_args=%sc3PK|]}t|tj ywr<)rctypes c_ulonglong)rargs r6rz2CppBenchmarkRequest.make_run_fn..;sR3:c6#5#56Rs$&)rrVrrrrrJrKr rrrrrrargtypesrvrw)rZrrrArrys r6rzCppBenchmarkRequest.make_run_fn,s $$T%5%55I04]0Cse0KLf!LL X    HH  OO  TXXt'7'78 R$//RRRR%112 ID12 2      __  !MsD)c d|jS)Nr)r rs r6rzCppBenchmarkRequest.__str__Gs#$""$%%r5) r rsr rr rrr+rrsrrr,r)r1r2r3r[rrrrrs@r6rrsi 2 29 2: 2 " 2  2  2; * 1=  6&r5rcPeZdZdZ dfd Z ddZxZS)CuteDSLBenchmarkRequestz;Benchmark request for CuteDSL (CUTLASS Python DSL) kernels.ct||||||j}tj|\|_|_yr<)rSr[ finalize_allrrrUrT)rZr r r rrfinalized_coder_s r6r[z CuteDSLBenchmarkRequest.__init__NsC &79KZX$1132=2C2CN2S/t/r5c T tj|j|j}ddlm}|j d|}t||s?t|Dcgc]}tt||s|}}td|d|t||  fd}|Scc}w)z Create a function to run the CuteDSL kernel with the given input and output tensors. Similar to TritonBenchmarkRequest.make_run_fn but for CuteDSL kernels. r() MAIN_SUFFIXrz-Could not find CuteDSL main kernel function 'z'. Available callables: c~td}|jjj}gd|iS)NrCre)rrqrXr?)rrer kernel_funcrs r6 run_kernelz7CuteDSLBenchmarkRequest.make_run_fn..run_kernelqs@7? %44SZZ5E5EFFB BsB6B Br5) rrgrUrTcodegen.cutedsl.cutedsl_kernelrr hasattrdircallablerr) rZrrrxrmain_func_namer availablerrs `` @r6rz#CuteDSLBenchmarkRequest.make_run_fn[s**4+@+@$BRBRS @ ,,-Q{m<sN+*-c(S$hwsD?Q6RSIS??OOghqgrs c>2  C Ts B%9B%) r rsr rr rrztuple[Any, ...]rr&rrr,)r1r2r3rr[rrrs@r6rrKsiE T T9 T: T $ T # T  T*1= r5rcXt}tj|j|Sr<)ratexitror)pools r6get_tuning_process_poolrys  D OODMM" Kr5c4tj|S)zO Do benchmarking in a subprocess and return the perf number (latency). )rr)rs r6benchmark_in_sub_processrs # $ . .w 77r5ceZdZUdZdZded<ejZded<dZ ded <d Z e d Z e d Zdd ZddZddZdZddZddZddZdZe dZy)AutotuneProcessPoolzr Singleton pool manager for running autotuning (precompilation + benchmarking) in a separate process. NzAutotuneProcessPool | None _instancezthreading.Lock_lockFr_shutdown_for_inactivityct|j|_d|_d|_|j |_yr<) _init_pool_pool_warmup_future_warmup_start_time _init_timer_timerrs r6r[zAutotuneProcessPool.__init__s0151B 2604$($4$4$6 r5c|j9|j5|j ||_ddd|jS|jS#1swY|jSxYw)z*Get or create the singleton pool instance.N)rrrs r6 get_instancez AutotuneProcessPool.get_instancesY ==  *==($'ECM *}}s}}  *}}s AA&ctjsJd|j*|j|_|j |_|jS)zGet the process pool.zFTo use AutotuneProcessPool, pipeline_max_autotune_gemm must be enabled)r)pipeline_max_autotune_gemmrrrr rs r6rzAutotuneProcessPool.poolsQ00 T 0 :: *DJ**,DKzzr5cJtdkDrtt|jSyr~) AUTOTUNE_POOL_INACTIVITY_TIMEOUTr_on_inactivity_timeoutrs r6rzAutotuneProcessPool._init_timers! +a /94;V;VW Wr5cR|j|jjyyr<)r  record_callrs r6_record_activityz$AutotuneProcessPool._record_activitys! ;; " KK # # % #r5ctjdt|j5|j#|jj dd|_d|_dt_dddy#1swYyxYw)NzAAutotuneProcessPool shutting down due to inactivity (timeout=%ds)FrT) rJr$r'rrrr rrrs r6r(z*AutotuneProcessPool._on_inactivity_timeoutsq O , ZZ @zz% ###/! DK <@  8 @ @ @s AA22A;ctjd}td|}tj|j t jd|S)a Get or create the process pool. Uses ProcessPoolExecutor with 'spawn' context for CUDA safety. ProcessPoolExecutor is lazily initialized - workers are not spawned until the first submit() call, making this property non-blocking. spawnr()r mp_contextz2AutotuneProcessPool created (workers spawn lazily))mp get_contextrrro _shutdownrJr$)rZctxrs r6rzAutotuneProcessPool._init_poolsHnnW%"  'PQ r5c|j|j5|jtj|_|j j ttjjjj|_|jj|jtj!dddd|jS|jS#1swY|jSxYw)z Submit a warmup job to eagerly spawn workers and initialize CUDA. This is optional - call it early to hide spawn latency. Returns the warmup future which can be ignored or awaited. N)fp32_precisionzWarmup job submitted)rrr# perf_counterrrsubmit_init_autotune_subprocessr<backendsrCmatmulr5add_done_callback_on_warmup_completerJr$rs r6warm_upzAutotuneProcessPool.warm_ups    & @&&..2.?.?.AD+*.))*:*:1',~~':':'A'A'P'P+;+D'''99$:R:RS"''(>? @"""t""" @"""s B2C++C?c$d}|j!tj|jz } |j}tj d|||j y#t$r}tjd||d}~wwxYw)z/Callback invoked when the warmup job completes.NzEAutotuneProcessPool warmup completed successfully in %.4f seconds: %sz4AutotuneProcessPool warmup failed after %.4f seconds) rr#r6rErJr$r+rAr)rZfuturewarmup_elapsed_timerErFs r6r<z'AutotuneProcessPool._on_warmup_completes"  " " ."&"3"3"58O8O"O  ]]_F   W#   ! ! #   F# G  s7A)) B2B  Bcjj|g|i|}j|jfd|S)z-Submit a job to the pool and return a Future.c$jSr<)r+)rrZs r6rz,AutotuneProcessPool.submit.. st/D/D/Fr5)rr7r r;)rZrrrr?s` r6r7zAutotuneProcessPool.submitsA!!!"6t6v6 ;; "  $ $%F G r5c|j!|jjd|_|j$|jjdd|_yy)zShutdown the pool on exit.NFr)r quitrrrs r6r2zAutotuneProcessPool._shutdown sN ;; " KK   DK :: ! JJ  U  +DJ "r5c|jC|j5|j!|jjd|_dddyy#1swYyxYw)z+Explicitly shutdown the singleton instance.Nrrr2r"s r6shutdown_instancez%AutotuneProcessPool.shutdown_instancesU == $ )==,MM++-$(CM ) ) % ) )s .AA)rz Timer | Noner)r Future[Any])r?rHrr)r1r2r3rrr threadingLockrrr[rr#propertyrrr+r(rr=r<r7r2rGr4r5r6rrs -1I)0*INN,E>,%*d*7  &@"(#(*))r5rcHtjxrtj Sr<)r)r%rrr4r5r6use_pipelined_autotuningrMs!)) =#<< <r5cddl}|jjr|jdd||jjj _y)z9 Warmup function run in the autotune subprocess. rNr(rCrT)r<rC is_availablerr9r:r5)r5r<s r6r8r8&sD zz  Af%0>ENN- r5c |j}|S#t$r&tjd|dt dcYSwxYw)a* Run autotuning benchmarks in a subprocess. This function is submitted to AutotuneProcessPool and runs in isolation to prevent GPU contention with the main compilation process. Args: picklable_choices: List of picklable choice information Returns: timing zFailed to benchmark choice %sT)exc_infor)rrArJwarningr)benchmark_requesttimings r6run_autotune_in_subprocessrU5sQ ",,.  +   U|s,AAceZdZUdZdZded<ejZd d dZ e d dZ dZ d ddZ e dd Zy)PrecompileThreadPoolz Thread pool for running precompilation asynchronously. This allows the main compilation process to continue while precompilation happens in background threads. NzPrecompileThreadPool | Nonerc&t||_y)Nr)r _executor)rZrs r6r[zPrecompileThreadPool.__init__`s+ Dr5cddlm}|j?|j5|j|||_ddd|jS|jS#1swY|jSxYw)Nr)get_num_workers)rr[rr)rr[s r6r#z!PrecompileThreadPool.get_instancecs`D ==  ;==($'(9$:CM ;}}s}} ;}}s AA2ctj}tj|j|}|j j |g|i|Sr<) contextvars copy_contextrvrwrhrYr7)rZrrrr3s r6r7zPrecompileThreadPool.submitmsG&&(   sww +$t~~$$R9$9&99r5c:|jj|S)Nr)rYrrs r6r2zPrecompileThreadPool._shutdownss~~&&D&11r5c|jE|j5|j#|jjdd|_dddyy#1swYyxYw)NFrrFr"s r6rGz&PrecompileThreadPool.shutdown_instancevsX == $ )==,MM+++7$(CM ) ) % ) )s 0AA))rr)rrW)F)rrr)r1r2r3rrrrIrJrr[rr#r7r2rGr4r5r6rWrWUsZ.2I*1 INN EE: 2))r5rWcVeZdZdZiZeddZeddZe ddZ y) AsyncAutotunera Handles asynchronous autotuning of kernel choices in a separate process. This class manages the lifecycle of autotuning: 1. Accepts precompiled choices from the main process 2. Submits benchmarking work to AutotuneProcessPool 3. Returns results via a Future Usage: autotuner = AsyncAutotuner(choices) autotuner.start() # Kicks off async benchmarking timings = autotuner.get_results() # Blocks until complete c(|j|zSr<)r)r inputs_keys r6get_choice_hashzAsyncAutotuner.get_choice_hashs :--r5c|D]}tj||}|tjvr,t|ddJdtj j t|j}|tj|<y)z Start asynchronous autotuning in a subprocess. This method: 1. Extracts picklable benchmark requests from choices 2. Submits benchmarking work to AutotuneProcessPool 3. Returns immediately (non-blocking) rNzbmreq is None for choice) rcrfchoice_hash_to_futurerrr#r7rUr)rrrer choice_hashautotune_futures r6rYzAsyncAutotuner.starts PF(88LKnBBB67D1= * =2>>@GG* O APN 0 0 = Pr5ci}|D]<}tj||}tj|j||<>|S)z Get autotuning results, blocking until complete. Args: timeout: Maximum time to wait in seconds. None means wait forever. Returns: Dict mapping ChoiceCaller to benchmark timing )rcrfrhrE)rrretimingsrris r6 get_resultszAsyncAutotuner.get_resultssP YF(88LK,BB;OVVXGFO Yr5N)rr%rersrrs)rlist[ChoiceCaller]rers)rrnrersrzdict[ChoiceCaller, float]) r1r2r3rrhrrfrrYrmr4r5r6rcrcsc ..PP6(69 "r5rc)rrrr)r5rsrr)rSrrr)s __future__rrr]r dataclassesrvr!multiprocessingr0r>rQrrlrurqrIr#rcollections.abcrrrconcurrent.futuresrrr r r r r typingrrrr<torch._inductor.async_compiletorch._dynamo.device_interfacertorch._dynamo.testingrtorch._inductorrtorch._inductor.codecacherrrrrr$torch._inductor.compile_worker.timerrtorch._inductor.utilsrrrrrtorch._loggingr torch.utils._ordered_setr!rr?rMr'typesr$rr%r&r'rr)runtime.benchmarkingr* virtualizedr+r,r1rJrAr/r9rrrLayoutOrBuffer dataclassrrr0r9rMrQrrrrrrrrrrrcacherrrrMr8rUrWrcr4r5r6rs"     88NN22)) $C.7-/ $'JJNNCUK$   -."8\:  ttnoodRYY& . . . b gggT,D  F--vU-vUr 79O  79O JE#3JEZ 9  9 $S/$SN "9;S  "9;S a$57Ga$H4&13C4&n+57G+\  8 '8&8S)S)l ' @')')TCCr5