Failed to fetch: status={{ resp.status_code }}
{{ resp.text }}
{% else %}
{{ resp.text }}
{% endif %}
{% endfor %}
{% endblock %}
a
{% extends "base.html" %}
{% block header %}
{{ fetch_summary }}{% endif %}
Failed to fetch: status={{ resp.status_code }}
{{ resp.text }}
{% else %}
{% endif %}
{% endfor %}
{% endblock %}
a
{% extends "base.html" %}
{% block header %}
{% for k, v in zip(keys, values) -%}
{{ k }}: {{ v | truncate(100) }}
{% endfor %}
{% endblock %}
c , e Zd ZddZddZddZd dZy)
IndexHandlerc 0 t d| j gS )N/r _handleselfs g/media/conek/DATA/Code/OCR/venv/lib/python3.12/site-packages/torch/distributed/debug/_debug_handlers.pyrouteszIndexHandler.routes s c4<<()) c t dd gS )Nr Homer
r s r nav_linkszIndexHandler.nav_links s V$%%r! c dt iS Nz
index.html)INDEX_TEMPLATEr s r templateszIndexHandler.templates s
n--r! c 8 |j j d S r' )frontendrender_template)r reqs r r zIndexHandler._handle s ||++L99r! Nreturnz list[Route]r/ z
list[NavLink]r/ zdict[str, str]r- r r/ bytes__name__
__module__ __qualname__r r% r) r r! r r r s *&.:r! r c 4 e Zd ZddZddZd dZd
dZd dZy)
StacksHandlerc 0 t d| j gS )N/stacksr r s r r zStacksHandler.routes s i .//r! c t dd gS )Nr< zPython Stack Tracesr$ r s r r% zStacksHandler.nav_links s #89::r! c t t d| j \ }}|j j dd|| S )Ndump_tracebacktimeoutz
raw_resp.htmlStackstitleaddrsrespsr
fetch_timeoutr+ r, r r- rE rF s r r zStacksHandler._handle s> !14;M;MN u||++ 85 ,
r! c t d| j \ }}g }t || }|r"|j | |j d t t || D ]Y \ }\ }}|j d| d| d |j |j dk( r |j nd|j [ d j | S )
Nr? r@ === Rank : === Error:
r rH r append enumeratezip status_codetextjoinr rE rF partssummaryiaddrresps r dumpzStacksHandler.dump s !14;M;MN u &ue4 LL ! LL (UE): ; OA|d LL9QCr$t4 5 LL ! - - 4 GDDTDTCU:V
yy r! c y)Nstacksr8 r s r
dump_filenamezStacksHandler.dump_filename s r! Nr. r0 r2 r/ z
str | Noner/ strr5 r6 r7 r r% r r_ rb r8 r! r r: r: s 0;
r! r: c < e Zd ZddZd dZd
dZd dZd dZd
dZy) PySpyHandlerc 0 t d| j gS )N /pyspy_dumpr r s r r zPySpyHandler.routes s mT\\233r! c t dd gS )Nrj z
py-spy Stacksr$ r s r r% zPySpyHandler.nav_links s
788r! c dt iS )Npyspy_dump.html)PYSPY_DUMP_TEMPLATEr s r r) zPySpyHandler.templates s !#677r! c |j }d|vr |rd| nd}t d|| j \ }}|j j d|| S )N nonblockingznonblocking=1&
nonblocking=1
pyspy_dumpr@ rm rE rF )
get_raw_queryr rH r+ r, )r r- queryrE rF s r r zPySpyHandler._handle se !!# %05nUG,?E ud>P>PQ u||++ ,
r! c t dd| j \ }}g }t || }|r"|j | |j d t t || D ]Y \ }\ }}|j d| d| d |j |j dk( r |j nd |j [ d
j | S ) Nrr rq r@ rK rL rM rN rO rP rQ rR rY s r r_ zPySpyHandler.dump s /43E3E
u &ue4 LL ! LL (UE): ; OA|d LL9QCr$t4 5 LL ! - - 4 GDDTDTCU:V
yy r! c y)Nrr r8 r s r rb zPySpyHandler.dump_filename s r! Nr. r0 r1 r2 rc rd r5 r6 r7 r r% r) r r_ rb r8 r! r rh rh s 498
r! rh c ~ e Zd Zd
dZddZddZedd Z ddZddZ ddZ
ddZ dd Z dd
Z
dd Zy )FlightRecorderHandlerc t d| j t d| j t d| j t d| j gS )N /fr_trace/fr_trace_json/fr_trace_nccl/fr_trace_nccl_json)r _handle_fr_trace_handle_fr_trace_json_handle_fr_trace_nccl_handle_fr_trace_nccl_jsonr s r r z FlightRecorderHandler.routes sO +t 4 4 5 "D$>$> ? "D$>$> ? ')H)H I
r! c ^ t dd t dd t dd t dd gS )Nr| zFlightRecorder CPUr} (JSON)r~ FlightRecorder NCCLr r$ r s r r% z FlightRecorderHandler.nav_links s9 K!5 6 $h / $&; < )8 4
r! c dt iS N
fr_trace.htmlFR_TRACE_TEMPLATEr s r r) z FlightRecorderHandler.templates !233r! c r t }|j g }d|_ d|_ i }t | D ]? \ }}|j
dk7 r|| | d|j
}d|vrg |d< ||d| d< A |s t d t t |j d }t ||| S )
N)argsTrO )rank host_nameentriesr z.jsonz All workers failed to respondversion) r
parse_argsallow_incomplete_ranksverboserT rV json RuntimeErrornextitervaluesr ) rE rF configr detailsr r^ r_ r s r _build_dbz FlightRecorderHandler._build_db s b )&*# #E*
/JD$ 3& "4[ ))+D
$"$Y*.Gd4&& '
/ >? ?tGNN ,-.y9w//r! c
| j || }|j ddt || t |j t
j d t |j t j d t |j t j d t |j t j d S )Nr FlightRecorderhtmlheaderstablefmtrD
fetch_summarygroups memberships collectives ncclcalls)
r r, r r r r _fieldsr r r r r r r serverrE rF dbs r _render_tablesz$FlightRecorderHandler._render_tables4 s ^^E5
)%% ".ue<BIIu}}vN
(:(:V !
(:(:V r||X5E5EPVW &
r! c t d| j \ }}| j |j |t | S )N
fr_trace_jsonr@ r rH r r+ listrI s r r z&FlightRecorderHandler._handle_fr_traceF s5 $:L:LM u""3<<U DDr! c t t d| j \ }}|j j dd|| S )Nr r@ json_resp.htmlr rC rG rI s r r z+FlightRecorderHandler._handle_fr_trace_jsonJ s@ $:L:LM u||++ " ,
r! c t dd| j \ }}| j |j |t | S )Ndump_nccl_trace_jsononlyactive=truer@ r rI s r r z+FlightRecorderHandler._handle_fr_trace_ncclS s= "$5t?Q?Q
u ""3<<U DDr! c v t dd| j \ }}|j j dd|| S )Nr r r@ r r rC rG rI s r r z0FlightRecorderHandler._handle_fr_trace_nccl_jsonY sH "$5t?Q?Q
u ||++ ' ,
r! c & g }t d| j \ }}t || }|r"|j | |j d | j || }|j ddt
|j t j d dt
|j t j d d t
|j t j d d
t
|j t j d g t d d | j \ }}| j || }|j dd
dt
|j t j d dt
|j t j d d t
|j t j d d
t
|j t j d g
dj# | S # t $ r |j d Y -w xY w)Nr r@ rK z=== FR Trace ===--- Groups ---plainr --- Memberships ------ Collectives ------ NCCL Calls ---r r z=== FR Trace NCCL ===z1
=== FR Trace NCCL ===
Failed to fetch NCCL tracerQ )r rH r rS r extendr r r r r r r r r r ExceptionrX ) r rZ rE rF r[ r
nccl_addrs
nccl_respsnccl_dbs r r_ zFlightRecorderHandler.dumpd s $:L:LM u&ue4 LL ! LL
^^E5
)
" EMMGL% 1C1CgV% 1C1CgV$ x/?/?'R
P%.&(94CUCU& "J
nnZ