+
    jHàjP  ã            
      óp  € R t ^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIt^ RIt^ RI	t	^ RI
t
^ RIt^ RIHtHtHt ^ RIHt ^ RIHtHtHtHt ^ RIHt ^ RIHtHt ^ RIHt ]P:                  ! ]4      tR	t R
t!Rt"R
t#Rt$^t%REt&RFt'RGR R llt( ! R R4      t) ! R R4      t*]"3R R llt+R]#R]$R]%R]/R R llt,R R lt-R R lt.R R  lt/R! R" lt0RHR# R$ llt1RIR% R& llt2RHR' R( llt3R)RR*RR+RR,R/R- R. llt4R/] /R0 R1 llt5R2R3/R4 R5 llt6R6RR7RR^R8RR9R:/R; R< llt7R= R> lt8R? R@ lt9RA]!R*RR+RRBR/RC RD llt:R# )Jag  Out-of-loop shutdown and event-loop liveness backstops.

A frozen asyncio loop takes every asyncio-based recovery path down with it, and launchd/systemd
KeepAlive only restarts a *dead* process. Hence: (1) an OS-thread shutdown watchdog that dumps
stacks and ``os._exit``s past ``restart_drain_timeout + grace``; (2) a heartbeat file at
``<HERMES_HOME>/state/gateway.heartbeat`` so supervisors can tell "process alive" from "loop
frozen"; (3) a lifetime thread watchdog that hard-exits when the loop is too frozen to run its
own callbacks; (4) a self-rescheduling floor timer that keeps the selector timeout finite.)ÚannotationsN)ÚdatetimeÚtimezoneÚUTC)ÚPath)ÚAnyÚCallableÚDictÚOptional)Ú!GATEWAY_SERVICE_RESTART_EXIT_CODE)Úget_hermes_homeÚget_process_hermes_home)Úatomic_json_writeg      N@g      >@g      @g      $@c               ó(   € V ^8„  d   QhRRRRRRRR/# )é   Úvaluer   ÚdefaultÚfloatÚfloorÚreturn© )Úformats   "Ú>/home/ubuntu/.hermes/hermes-agent/gateway/shutdown_watchdog.pyÚ__annotate__r   0   s(   € ÷ ñ ˜ð  uð °Uð ÀUñ ó    c                ó`   €  \        \        V 4      V4      #   \        \        3 d    Tu # i ; i)z@``max(float(value), floor)``, or ``default`` when not coercible.)Úmaxr   Ú	TypeErrorÚ
ValueError)r   r   r   s   &&&r   Ú_coerce_floatr   0   s1   € ðÜ”5˜“< Ó'Ð'øÜ”zÐ"ô ØŠðús   ‚ —-¬-c                  ó>   € ] tR t^8tRtR R ltR R ltR R ltRtR	# )
Ú_LoopFloorTimerHandlezBCancelable owner for the currently scheduled selector floor timer.c               ó    € V ^8„  d   QhRRRR/# )r   Úloopúasyncio.AbstractEventLoopÚintervalr   r   )r   s   "r   r   Ú"_LoopFloorTimerHandle.__annotate__:   s   € ÷ ñ Ð6ð À%ñ r   c                	ó^   € WR uV n         V n        V n        RV n        V P	                  4        R# )FN)Ú_loopÚ	_intervalÚ
_cancelledÚ_timerÚ_tick)Úselfr#   r%   s   &&&r   Ú__init__Ú_LoopFloorTimerHandle.__init__:   s&   € Ø6:ÀeÐ3ˆŒ
�D”N D¤OØ59ˆŒØ�
‰
Žr   c               ó   € V ^8„  d   QhRR/# ©r   r   ÚNoner   )r   s   "r   r   r&   ?   s   € ÷ Lñ L�tñ Lr   c                	ó˜   € V P                   '       g8   V P                  P                  V P                  V P                  4      V n        R # R # ©N)r*   r(   Ú
call_laterr)   r,   r+   ©r-   s   &r   r,   Ú_LoopFloorTimerHandle._tick?   s0   € Ø��ˆØŸ*™*×/Ñ/°·±ÀÇ
Á
ÓKˆDŽKñ r   c               ó   € V ^8„  d   QhRR/# r1   r   )r   s   "r   r   r&   C   s   € ÷ !ñ !˜ñ !r   c                	óh   € R V n         V P                  e   V P                  P                  4        R# R# )TN)r*   r+   Úcancelr6   s   &r   r:   Ú_LoopFloorTimerHandle.cancelC   s(   € ØˆŒØ�;‰;Ò"Ø�K‰K×ÑÖ ñ #r   )r*   r)   r(   r+   N)	Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__r.   r,   r:   Ú__static_attributes__r   r   r   r!   r!   8   s   † ÙLõõ
L÷!ñ !r   r!   c                  ó"   € ] tR t^ItR R ltRtR# )Ú_LoopLivenessWatchdogHandlec               ó    € V ^8„  d   QhRRRR/# )r   Ú
stop_eventúthreading.EventÚthreadzthreading.Threadr   )r   s   "r   r   Ú(_LoopLivenessWatchdogHandle.__annotate__J   s   € ÷ [ñ [ ?ð [Ð<Lñ [r   c                	óz   € Wn         VP                  VP                  VP                  uV n        V n        V n        R # r4   )Ú_stop_eventÚsetÚjoinÚis_aliveÚstop)r-   rE   rG   s   &&&r   r.   Ú$_LoopLivenessWatchdogHandle.__init__J   s*   € Ø%ÔØ.8¯n©n¸f¿k¹kÈ6Ï?É?Ð+ˆŒ	�4”9˜džmr   )rJ   rM   rL   rN   N)r<   r=   r>   r?   r.   rA   r   r   r   rC   rC   I   s   † ÷[ñ [r   rC   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r#   r$   r%   r   r   r!   r   )r   s   "r   r   r   O   s)   € ÷ ^ñ ^Ø
#ð^Ø/4ð^àñ^r   c                óR   € \        VR4      p\        Y^ 8”  d   V4      # \        4      # )zAKeep at least one timer pending so selector waits remain bounded.ç        )r   r!   Ú#DEFAULT_LOOP_FLOOR_TIMER_INTERVAL_S)r#   r%   Úivs   && r   Ú_arm_loop_floor_timerrU   O   s*   € ô 
�x Ó	%€BÜ  °!¬V rÓ]Ð]Ô9\Ó]Ð]r   Úprobe_intervalÚprobe_timeoutÚmax_strikesÚ	exit_codec               ó0   € V ^8„  d   QhRRRRRRRRRRR	R
/# )r   r#   r$   rV   r   rW   rX   ÚintrY   r   z%Optional[_LoopLivenessWatchdogHandle]r   )r   s   "r   r   r   W   sB   € ÷ <;ñ <;Ø
#ð<;Ø8=ð<;àð<;ð ð<;ð ð	<;ð
 +ñ<;r   c               ó  a aaaaa€ \         P                  ! 4       oR VV VVVV3R llp\         P                  ! VRRR7      p VP                  4        \        ST4      #   \         d    \
        P                  RRR7        R# i ; i)	z´Start an out-of-loop watchdog that hard-exits after missed probes. The caller
(``GatewayRunner._start_loop_liveness_guards``) enforces the ``gateway.loop_watchdog: false``
opt-out.c               ó   € V ^8„  d   QhRR/# r1   r   )r   s   "r   r   Ú2start_loop_liveness_watchdog.<locals>.__annotate__b   s   € ÷ *"ñ *"�tñ *"r   c                 ó  <€ ^ p S	P                  SR7      '       Eg¡   \        P                  ! 4       p SP                  VP                  4       \        P                  ! 4       S,           pS	P                  4       '       gI   T\        P                  ! 4       ,
          pT^ 8:  g%   TP                  \        TR4      R7      '       g   K]  MR# TP                  4       '       d   ^ p Kà  S	P                  4       '       d   R# T ^,          p T S8  d   EK  S	P                  4       '       d   R# \        P                  ! \        4      ;_uu_ 4        \        P                  RT S4       RRR4       \!        \#        4       ST ,          RT R	S/R
R7       S	P                  4       '       d   R# \%        SR4       \'        S4       EK¹  R#   \
         d     R# \         d    \        P                  RRR7        R# i ; i  + '       g   i     Lœ; i)é    ©ÚtimeoutNz.Failed to schedule gateway loop liveness probeT©Úexc_infogš™™™™™©?z–Gateway event loop missed %d consecutive liveness probes; dumping all thread stacks and exiting with code %d so the service supervisor can restart it.ÚstrikesÚprobe_interval_sÚloop_liveness_watchdog_fired)Údelay_sÚsnapshotÚeventÚloop_liveness_watchdog)ÚwaitÚ	threadingÚEventÚcall_soon_threadsaferK   ÚRuntimeErrorÚ	ExceptionÚloggerÚdebugÚtimeÚ	monotonicÚis_setÚminÚ
contextlibÚsuppressÚcriticalÚ_write_watchdog_dumpÚget_shutdown_watchdog_dump_pathÚ_mark_exited_quietlyÚ
_hard_exit)
re   Úprobe_eventÚdeadlineÚ	remainingrY   r#   rX   rV   rW   rE   s
       €€€€€€r   Ú	_watchdogÚ/start_loop_liveness_watchdog.<locals>._watchdogb   s   ø€ ØˆØ—/‘/¨.�/×9Ó9Ü#Ÿ/š/Ó+ˆKðØ×)Ñ)¨+¯/©/Ô:ô —~’~Ó'¨-Õ7ˆHØ ×'Ñ'×)Ò)Ø$¤t§~¢~Ó'7Õ7�	Ø ”> [×%5Ñ%5¼cÀ)ÈTÓ>RÐ%5×%SÔ%SØáØ×!Ñ!×#Ò#Ø�ÙØ× Ñ ×"Ò"ÙØ�q�LˆGØ˜Ô$ÚØ× Ñ ×"Ò"ÙÜ×$Ò$¤Y×/Õ/Ü—‘ð`à˜Yô(÷ 0ô !Ü/Ó1¸=È7Õ;RØ# WÐ.@À.ÐQØ4õ6ð × Ñ ×"Ò"ÙÜ  Ð,DÔEÜ�y×!ñQ :øô  ô ÚÜô Ü—‘ÐMÐX\�Ô]Úðú÷( 0×/ús)   ³F? Å	G5Æ?G2ÇG2ÇG2Ç1G2Ç5H	Tzgateway-loop-liveness-watchdog©ÚtargetÚdaemonÚnamez.Failed to start gateway loop liveness watchdogrc   N)rm   rn   ÚThreadÚstartrq   rr   rs   rC   )r#   rV   rW   rX   rY   r‚   rG   rE   s   fdddd  @r   Ústart_loop_liveness_watchdogrŠ   W   sv   ý€ ô —’Ó"€J÷*"õ *"ôV ×Ò Y°tÐBbÔc€FðØ�‰Œô ' z°6Ó:Ð:øô ô Ü�‰ÐEÐPTˆÔUÚðús   ÁA  Á "BÂBc               ó    € V ^8„  d   QhRRRR/# )r   rY   r[   r   r2   r   )r   s   "r   r   r   –   s   € ÷ ñ ˜#ð  $ñ r   c                óÌ   € \         P                  ! \        4      ;_uu_ 4        ^ RIHp V! RR7       RRR4       \
        P                  ! V 4       R#   + '       g   i     L(; i)z¥``os._exit`` skips every cleanup: SIGKILL in-flight foreground commands first, they run in their
own process group and would outlive the gateway, reparented to init.)Úkill_live_foreground_processesT)ÚnowN)rx   ry   rq   Útools.environments.baser�   ÚosÚ_exit)rY   r�   s   & r   r~   r~   –   s<   € ô 
×	Ò	œY×	'Õ	'ÝJÙ&¨4Õ0÷ 
(ô ‡H‚HˆYÖ÷ 
(×	'ús   £AÁA#	c               ó$   € V ^8„  d   QhRRRRRR/# )r   rY   r[   ÚreasonÚstrr   r2   r   )r   s   "r   r   r   Ÿ   s&   € ÷ Xñ X Cð X°ð X¸ñ Xr   c                ó   € \         P                  ! \        4      ;_uu_ 4        ^ RIHp V! WR7       RRR4       \         P                  ! \        4      ;_uu_ 4        ^ RIHp V! R4       RRR4       \         P                  ! \        4      ;_uu_ 4        ^ RIHp V \        8X  d   RR/M/ pV! RR	R
RVRR/VB  RRR4       R#   + '       g   i     L›; i  + '       g   i     Lu; i  + '       g   i     R# ; i)uÅ  Best-effort terminal stamp on BOTH lifecycle records before ``os._exit`` skips teardown:
the lifecycle ledger (so the next boot names the watchdog, not SIGKILL/OOM) and
``gateway_state.json`` (so ``hermes gateway status`` and every other reader of that file stop
seeing ``running`` for a process the watchdog killed â€” #113372). The runtime-status write goes
LAST: it is the record housekeeping refreshes, so nothing may overwrite it after we stamp it.)Úmark_exited)r“   N)Ú
stamp_exitÚwatchdog)Úwrite_runtime_statusÚrestart_requestedTÚgateway_stateÚdegradedÚexit_reasonÚwait_timeoutg      Ð?r   )
rx   ry   rq   Úgateway.lifecycle_ledgerr–   Ú/hermes_cli.observability.shared_metrics_processr—   Úgateway.statusr™   r   )rY   r“   r–   r—   r™   Úrestarts   &&    r   r}   r}   Ÿ   sÉ   € ô 
×	Ò	œY×	'Õ	'Ý8Ù�IÕ-÷ 
(ô 
×	Ò	œY×	'Õ	'ÝNÙ�:Ô÷ 
(ô 
×	Ò	œY×	'Õ	'Ý7ð 2;Ô>_Ô1_Ð&¨Ñ-ÐegˆÙñ 	XØ$ð	XØ28ð	XØGKð	XØOVò	X÷ 
(Ñ	'÷ 
(×	'ú÷ 
(×	'ú÷ 
(×	'Ð	'ús#   £CÁCÂ&C,ÃC	ÃC)	Ã,C=	c               ó   € V ^8„  d   QhRR/# )r   r   r   r   )r   s   "r   r   r   ´   s   € ÷ iñ i˜dñ ir   c                 ó’   € \         P                  P                  RR4      P                  4       '       d   \	        4       # \        4       # )zHHERMES_HOME for process-level identity files (ignore profile overrides).ÚHERMES_HOMEÚ )r�   ÚenvironÚgetÚstripr   r   r   r   r   Ú_process_hermes_homerª   ´   s2   € ä(*¯
©
¯©°}ÀbÓ(I×(OÑ(O×(QÒ(QÔ"Ó$ÐhÔWfÓWhÐhr   c               ó    € V ^8„  d   QhRRRR/# ©r   ÚhomeúOptional[Path]r   r   r   )r   s   "r   r   r   ¹   s   € ÷ @ñ @�ð @ 4ñ @r   c                ó"   € V e   V # \        4       # r4   )rª   ©r­   s   &r   Ú_homer±   ¹   s   € ØÒ#ˆ4Ð?Ô)=Ó)?Ð?r   c               ó    € V ^8„  d   QhRRRR/# r¬   r   )r   s   "r   r   r   ½   s   € ÷ 6ñ 6 .ð 6¸Dñ 6r   c                ó:   € \        V 4      P                  ! \         # r4   )r±   ÚjoinpathÚ_HEARTBEAT_RELATIVEr°   s   &r   Úget_loop_heartbeat_pathr¶   ½   s   € Ü�‹;×ÒÔ!4Ð5Ð5r   c               ó$   € V ^8„  d   QhRRRRRR/# )r   r­   r®   ÚpidúOptional[int]r   r   r   )r   s   "r   r   r   Á   s'   € ÷ 
Cñ 
C Nð 
CÀð 
CÐY]ñ 
Cr   c                ó„   € \        Ve   TM\        P                  ! 4       4      p\        V 4      R,          RV R2,          # )uæ  ``<HERMES_HOME>/state/gateway.loop-tick.<pid>.sock`` â€” PID-suffixed so a stale node from a
dead process is never mistaken for this gateway's witness. Served by the loop itself
(``_tick_socket_handler``), so an answer proves the loop dispatches; the heartbeat cannot.

Served by the gateway loop itself (see ``_tick_socket_handler``): an answer is direct proof that the
loop is dispatching, which is exactly the property the heartbeat file lost when its write moved off-loop
(#90502).
Ústatezgateway.loop-tick.z.sock)r[   r�   Úgetpidr±   )r­   r¸   s   &&r   Úget_loop_tick_socket_pathr½   Á   s8   € ô �S’_‰c¬"¯)ª)«+Ó
6€CÜ�‹;˜Õ Ð%7¸°u¸EÐ#BÕBÐBr   c               ó    € V ^8„  d   QhRRRR/# r¬   r   )r   s   "r   r   r   Î   s   € ÷ :ñ :¨.ð :ÀDñ :r   c                ó:   € \        V 4      P                  ! \         # r4   )r±   r´   Ú_WATCHDOG_DUMP_RELATIVEr°   s   &r   r|   r|   Î   s   € Ü�‹;×ÒÔ!8Ð9Ð9r   r¸   Ú
start_timer­   Úextrac          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )r   r¸   r¹   rÁ   úOptional[float]r­   r®   rÂ   úOptional[dict[str, Any]]r   r   r   )r   s   "r   r   r   Ò   s4   € ÷ ñ ØðØ.=ðà
ðà(@ðàLPñr   c                óT  € \        V4      pR\        V e   T M\        P                  ! 4       4      R\        P
                  ! \        4      P                  4       R\        P                  ! 4       /pVe   \        V4      VR&   \        P                  ! \        4      ;_uu_ 4        ^ RIHp V! 4       ;p'       d   WuR&   RRR4       V'       d   VP!                  V4        \#        WERR7       V#   + '       g   i     L9; i  \         d    \$        P'                  R	R
R7        T# i ; i)z“Atomically rewrite the loop-liveness heartbeat file; never raises.
``start_time`` (process start, epoch seconds) lets supervisors detect PID reuse.r¸   NÚ
updated_atru   rÁ   )Úsample_memoryÚmem)Úindentz&Failed to write gateway loop heartbeatTrc   )r¶   r[   r�   r¼   r   rŽ   r   Ú	isoformatrt   ru   r   rx   ry   rq   rŸ   rÈ   Úupdater   rr   rs   )r¸   rÁ   r­   rÂ   ÚpathÚpayloadrÈ   rÉ   s   $$$$    r   Úwrite_loop_heartbeatrÏ   Ò   së   € ô
 # 4Ó(€DØ$¤c°²©#ÄbÇiÂiÃkÓ&RØ+¬X¯\ª\¼#Ó->×-HÑ-HÓ-JØ*¬D¯NªNÓ,<ð>€Gð ÒÜ % jÓ 1ˆ�ÑÜ	×	Ò	œY×	'Õ	'Ý:Ù“/Ð!ˆ3Ö!Ø �E‰N÷ 
(÷ Ø�‰�uÔðNÜ˜$°Õ5ð €K÷ 
(×	'ûô ô NÜ�‰Ð=ÈˆÕMØ€KðNús   Â#C.ÃD Ã.C>	Ä"D'Ä&D'Úgrace_sc               ó$   € V ^8„  d   QhRRRRRR/# )r   Údrain_timeoutr   rÐ   r   r   )r   s   "r   r   r   ê   s$   € ÷ 5ñ 5Øð5Ø&+ð5ØTYñ5r   c               óH   € \        V\        4      p\        V R4      V,           # )z=Return the wall-clock leash for the shutdown watchdog thread.rR   )r   Ú!DEFAULT_SHUTDOWN_WATCHDOG_GRACE_S)rÒ   rÐ   Úgraces   &$ r   Úresolve_shutdown_watchdog_delayrÖ   ê   s#   € ô ˜'Ô#DÓE€EÜ˜¨Ó,¨uÕ4Ð4r   rj   Úshutdown_watchdog_firedc          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )r   Ú	dump_pathr   rh   r   ri   rÅ   rj   r”   r   r2   r   )r   s   "r   r   r   ñ   s2   € ÷ 6ñ 6 Dð 6°eð 6ÐG_ð 6Ø #ð6ØDHñ6r   c               óþ  € RTR\         P                  ! 4       RTR\        P                  ! \        4      P                  4       RT;'       g    / /p\        P                  ! \        4      ;_uu_ 4        V P                  P                  RRR7       RRR4       \        P                  ! \        4      ;_uu_ 4        \        V R	R
R7      ;_uu_ 4       pVP                  \        P                  ! V\         R7      R,           4       VP#                  4         \$        P&                  ! VRR7       VP                  R4       VP#                  4        RRR4       RRR4       \        P                  ! \        4      ;_uu_ 4        \(        P*                  P                  RVP-                  RR4       RVR R\         P                  ! 4        R24       \(        P*                  P#                  4        \$        P&                  ! RR7       RRR4       R#   + '       g   i     EL†; i  \         d    TP                  R4        ELi ; i  + '       g   i     Lþ; i  + '       g   i     EL
; i  + '       g   i     R# ; i)zNBest-effort faulthandler + metadata dump before hard-exit (file, then stderr).rj   r¸   rh   Úfired_atri   T©ÚparentsÚexist_okNÚazutf-8)Úencoding)r   z)
--- faulthandler dump (all threads) ---
)ÚfileÚall_threadsz%(faulthandler.dump_traceback failed)
z--- end dump ---
zGateway Ú_Ú z after z.0fzs (pid=z); dumping all thread stacks.
)râ   )r�   r¼   r   rŽ   r   rË   rx   ry   ÚOSErrorÚparentÚmkdirrq   ÚopenÚwriteÚjsonÚdumpsr”   ÚflushÚfaulthandlerÚdump_tracebackÚsysÚstderrÚreplace)rÙ   rh   ri   rj   ÚheaderÚfhs   &$$$  r   r{   r{   ñ   s·  € ð �u˜e¤R§Y¢Y£[°)¸WØœ(Ÿ,š,¤sÓ+×5Ñ5Ó7¸ÀXÇ^À^ÐQSðU€Fä	×	Ò	œW×	%Õ	%Ø×Ñ×Ñ t°dÐÔ;÷ 
&ä	×	Ò	œY×	'Õ	'¬¨i¸Àw×)OÕ)OÐSUØ
�‰”—’˜F¬CÔ0Ð3`Õ`ÔaØ
�‰Œ
ð	?Ü×'Ò'¨R¸TÕBð 	�‰Ð%Ô&Ø
�‰Œ
÷ *P×	'ô 
×	Ò	œY×	'Õ	'Ü�
‰
×Ñ˜8 E§M¡M°#°sÓ$;Ð#<¸GÀGÈCÀ=ð Q!Ü!#§¢£ Ð-LðNô 	Oä�
‰
×ÑÔÜ×#Ò#°Õ5÷	 
(Ñ	'÷ 
&×	%Ð	%ûô ô 	?Ø�H‰HÐ=×>ð	?ú÷ *P×)Oú×	'×	'Ð	'ú÷ 
(×	'Ð	'úsb   Á2HÂ:IÃAI	ÄH!Ä-!I	ÅIÆ BI+ÈH	È!IÈ=I	É IÉI	ÉIÉIÉI(	É+I<	Ú
done_eventÚsnapshot_fnrÙ   r‡   zgateway-shutdown-watchdogc               ó4   € V ^8„  d   QhRRRRRRRRR	R
RRRR/# )r   rh   r   rô   zOptional[threading.Event]rõ   z&Optional[Callable[[], dict[str, Any]]]rY   r[   rÙ   r®   r‡   r”   r   rF   r   )r   s   "r   r   r     sL   € ÷ 1ñ 1Øð1Ø#<ð1à7ð1àKNð1ð ð1ð -0ð1ð Sbñ1r   c               ó4  aaaaa€ Ve   TM\         P                  ! 4       o\        V \        4      oS^ 8:  d   S# R VVVVV3R llp \         P                  ! VRVR7      P                  4        S#   \         d    \        P                  RRR7        S# i ; i)zåArm a daemon-thread hard-exit backstop for a wedged shutdown path: exits quietly if
``done_event`` is set within ``delay_s``, else dumps diagnostics and ``os._exit(exit_code)``.
Never raises; returns ``done_event`` for disarming.c               ó   € V ^8„  d   QhRR/# r1   r   )r   s   "r   r   Ú+arm_shutdown_watchdog.<locals>.__annotate__  s   € ÷  ñ  �tñ  r   c                 óÒ  <€ \         P                  ! 4       S,           p \         P                  ! 4       V 8  dA   S	P                  \        V \         P                  ! 4       ,
          R 4      R7      '       g   KX  R# S	P	                  4       '       d   R#  Se   S! 4       MRpS
e   S
M	\        4       p\        VSVR7       \        P                  ! \
        4      ;_uu_ 4        \        P                  RSV4       RRR4       \        P                  \        P                  3 F>  p\        P                  ! \
        4      ;_uu_ 4        VP!                  4        RRR4       K@  	  \        P                  ! \
        4      ;_uu_ 4        ^ RIHpHp V! 4        V! 4        RRR4       \        P                  ! \
        4      ;_uu_ 4        ^ RIHp V! R R7       RRR4       \-        SR4       \/        S4       R#   \
         d   pR\        T4      /p Rp?ELtRp?ii ; i  + '       g   i     EL-; i  + '       g   i     EK"  ; i  + '       g   i     L²; i  + '       g   i     L‹; i)	ç      ð?ra   NÚsnapshot_error)rh   ri   uh   Shutdown watchdog fired after %.0fs â€” forcing process exit (asyncio drain path appears wedged; see %s))Úremove_pid_fileÚrelease_gateway_runtime_lock)Údrain_log_queueÚshutdown_watchdog)rt   ru   rl   rw   rv   rq   Úreprr|   r{   rx   ry   rr   rz   rï   Ústdoutrð   rì   r¡   rý   rþ   Úhermes_loggingrÿ   r}   r~   )r€   ri   Úexcr…   Ústreamrý   rþ   rÿ   ÚdelayÚdonerÙ   rY   rõ   s           €€€€€r   r‚   Ú(arm_shutdown_watchdog.<locals>._watchdog  sŸ  ø€ Ü—>’>Ó# eÕ+ˆÜ�nŠnÓ Ô)Ø�y‰y¤ X´·²Ó0@Õ%@À#Ó!Fˆy×GÔGÙØ�;‰;�=Š=Ùð	5Ø(3Ò(?‘{”}ÀTˆHð (Ò3‘Ô9XÓ9ZˆÜ˜V¨U¸XÕFÜ× Ò ¤×+Õ+Ü�O‰Oð JØKPÐRXôZ÷ ,ô —z‘z¤3§:¡:Ó.ˆFÜ×$Ò$¤Y×/Õ/Ø—‘”÷ 0Ñ/ñ /ô
 × Ò ¤×+Õ+÷ UÙÔÙ(Ô*÷ ,ô × Ò ¤×+Õ+Ý6Ù CÕ(÷ ,ô 	˜YÐ(;Ô<Ü�9Öøô/ ô 	5Ø(¬$¨s«)Ð4ŽHûð	5ú÷ ,×+Ð+ú÷ 0×/Ñ/ú÷ ,×+ú÷ ,×+úsH   ÂG4 ÃHÅH.ÆIÇIÇ4HÇ?HÈHÈH+	È.I ÉI	ÉI&	Tr„   zFailed to arm shutdown watchdogrc   )	rm   rn   r   rÔ   rˆ   r‰   rq   rr   rs   )	rh   rô   rõ   rY   rÙ   r‡   r‚   r  r  s	   &$ddd$ @@r   Úarm_shutdown_watchdogr	    s�   ü€ ð $Ò/‰:´Y·_²_Ó5F€DÜ˜'Ô#DÓE€EØ�„zØˆ÷ ô  ðBGÜ×Ò 	°$¸TÔB×HÑHÔJð €Køô ô GÜ�‰Ð6ÀˆÕFØ€KðGús   Á'A1 Á1"BÂBc               ó$   € V ^8„  d   QhRRRRRR/# )r   Úreaderzasyncio.StreamReaderÚwriterzasyncio.StreamWriterr   r2   r   )r   s   "r   r   r   <  s$   € ÷ ñ Ð';ð ÐEYð Ð^bñ r   c           	   ƒ  óÌ  "  €  VP                  R4       VP                  4       G Rj  x€L
  \        P                  ! \        4      ;_uu_ 4        VP                  4        RRR4       R#  LA  \         d     LLi ; i  + '       g   i     R# ; i  \        P                  ! \        4      ;_uu_ 4        TP                  4        RRR4       i   + '       g   i     i ; i; i5i)zÒAnswer a liveness ping with one byte; never raises. Runs on the gateway loop, so a reply
witnesses loop schedulability; the write is a socket-buffer copy (no fsync), immune to the
stalls that age the heartbeat.ó   1N)ré   Údrainrq   rx   ry   Úclose)r  r  s   &&r   Ú_tick_socket_handlerr  <  s“   é € ðØ�‰�TÔØ�l‰l‹n×Ðô × Ò ¤×+Õ+Ø�L‰LŒN÷ ,Ñ+ñ	 	øÜô Ùðú÷ ,×+Ð+ûŒZ× Ò ¤×+Õ+Ø�L‰LŒN÷ ,×+×+Ð+ÿss   ‚C$„$A, ¨A*©A, ­"C$ÁA=Á 
C$Á*A, Á,A:Á7B Á9A:Á:B Á=B	Â	C$Â#C!Â4C	Ã	C!ÃCÃC!Ã!C$c               ó    € V ^8„  d   QhRRRR/# )r   Úown_pathr   r   r2   r   )r   s   "r   r   r   J  s   € ÷ Kñ K¨ð K°ñ Kr   c                ó”  a €  V 3R lS P                   P                  R4       4        FD  p \        P                  ! \	        VP
                  P                  R4      R	,          4      ^ 4       KF  	  R#   \        \        \        3 d    TP                  RR7        Ku  i ; i  \         d    \        P                  RRR7        R# i ; i)
a/  Unlink loop-tick socket nodes left by dead PIDs (POSIX only; never raises).
create_unix_server removes a leftover node at OUR path (os._exit / SIGKILL skip the
finally-unlink) but not SIBLING nodes from other dead PIDs. os.kill(pid, 0) is a liveness
probe only on POSIX (Windows would TerminateProcess).c              3  ó8   <"  € T F  qS8w  g   K  Vx € K  	  R # 5ir4   r   )Ú.0Úpr  s   & €r   Ú	<genexpr>Ú,_sweep_stale_tick_sockets.<locals>.<genexpr>P  s   øé € ÐcÑ!Q˜AÐZbÑUb—a’aÓ!Qùs   ƒ�
zgateway.loop-tick.*.sockÚ.T©Ú
missing_okz#stale loop-tick socket sweep failedrc   Néþÿÿÿ)ræ   Úglobr�   Úkillr[   r‡   Úsplitr   Ú
IndexErrorrå   Úunlinkrq   rr   rs   )r  Ústales   f r   Ú_sweep_stale_tick_socketsr$  J  sœ   ø€ ð
KÜc §¡×!5Ñ!5Ð6PÔ!QÖcˆEð.Ü—’œ˜EŸJ™J×,Ñ,¨SÓ1°"Õ5Ó6¸Ö:ó døô ¤
¬GÐ4ô .Ø—‘¨�×-ð.ûäô KÜ�‰Ð:ÀTˆ×JðKús5   ƒ(B! ¬A A2Á,B! Á2(BÂB! ÂBÂB! Â!"CÃCÚ
interval_sÚshould_continuec          
     ó,   € V ^8„  d   QhRRRRRRRRR	R
/# )r   r%  r   rÁ   rÄ   r­   r®   r&  zOptional[Callable[[], bool]]r   r2   r   )r   s   "r   r   r   Y  s4   € ÷ :=ñ :=Øð:=ØETð:=à
ð:=à2Nð:=àZ^ñ:=r   c           	   ƒ  óh	  "  € \        V \        RR7      p^ RIHp R;p;rx \        P
                  R8X  d   \        V4      MRp	V	eg   V! V	4      '       dY   T	pVP                  P                  RRR7       \        V4       \        P                  ! \        \        V4      R7      G Rj  x€L
 pMÁ\        P                  ! \        R	^ R
7      G Rj  x€L
 pVP                  ;'       g    .  F�  p
\         P"                  ! \$        4      ;_uu_ 4        \'        V
P)                  4       ;p\*        4      '       d.   \-        V4      ^8¼  d   \/        V^,          4      p RRR4        MRRR4       Kƒ  	   RVRJRV/p   \        P4                  ! \6        WVR7      G Rj  x€L
  Ve­   V! 4       '       g     Ve™   VP;                  4        \         P"                  ! \$        4      ;_uu_ 4        VP=                  4       G Rj  x€L
  RRR4       Ve@   \         P"                  ! \$        4      ;_uu_ 4        VP?                  RR7       RRR4       R# R# R# \        P@                  ! V4      G Rj  x€L
  Vf   Kû  V! 4       '       d   EK   Ve™   VP;                  4        \         P"                  ! \$        4      ;_uu_ 4        VP=                  4       G Rj  x€L
  RRR4       Ve@   \         P"                  ! \$        4      ;_uu_ 4        VP?                  RR7       RRR4       R# R# R#  ELz ELW  + '       g   i     EKQ  ; i  \$         d    R;rh\0        P3                  RRR7        ELöi ; i ELÏ  \$         d    \0        P9                  RRR7        ELòi ; i ELš  + '       g   i     EL¡; i  + '       g   i     R# ; i ELX Lô  + '       g   i     Lú; i  + '       g   i     R# ; i  Te¿   TP;                  4        \         P"                  ! \$        4      ;_uu_ 4        TP=                  4       G Rj  x€L 
  RRR4       M  + '       g   i     M; iTeR   \         P"                  ! \$        4      ;_uu_ 4        TP?                  RR7       RRR4       i   + '       g   i     i ; ii i ; i5i)a  Rewrite the loop heartbeat file on a cadence until cancelled / gated off. Runs on the
gateway loop so a frozen loop lets the file age for monitors. The fsync write goes to a thread
(inline, a stalled filesystem blocked the loop inside its own heartbeat and the liveness
watchdog killed it: WSL2 fsync stalls p99 31s / max 112s vs a ~90-120s budget) but is *awaited*,
never fire-and-forget: an unawaited task would keep the file fresh while the loop was wedged,
and one in-flight write at a time keeps a long stall from queuing a thread per interval.
Off-loop, file freshness no longer proves loop schedulability, so this task also arms a
loop-scheduling witness (``_tick_socket_handler``, flagged as ``loop_tick_socket``); probes must
require the witness to agree before classifying WEDGED.rû   )r   )Ú_fits_sun_pathNÚposixTrÜ   )rÍ   z	127.0.0.1)ÚhostÚportu€   Loop tick socket unavailable â€” liveness probes will have no loop-scheduling witness and will not escalate on a stale heartbeatrc   Úloop_tick_socketÚloop_tick_tcp_port)rÁ   r­   rÂ   z$Loop heartbeat write failed off-loopr  )!r   ÚDEFAULT_HEARTBEAT_INTERVAL_SÚgateway.control_socketr)  r�   r‡   r½   ræ   rç   r$  ÚasyncioÚstart_unix_serverr  r”   Ústart_serverÚsocketsrx   ry   rq   Ú
isinstanceÚgetsocknameÚtupleÚlenr[   rr   ÚwarningÚ	to_threadrÏ   rs   r  Úwait_closedr"  Úsleep)r%  rÁ   r­   r&  r%   r)  Útick_serverÚtick_socket_pathÚtick_tcp_portÚ	unix_pathÚ_sÚ_snamerÂ   s   $$$$         r   Úloop_heartbeat_foreverrC  Y  sÂ  é € ô ˜ZÔ)EÈSÔQ€Hõ 6Ø59Ð9€KÐ9Ð"ð&Ü79·w±wÀ'Ô7IÔ-¨dÔ3Ètˆ	ØÒ ¡^°I×%>Ò%>Ø(ÐØ×#Ñ#×)Ñ)°$ÀÐ)ÔFÜ%Ð&6Ô7Ü '× 9Ò 9Ô:NÜ?BÐCSÓ?Tô!V÷ V‰Kô !(× 4Ò 4Ô5IÐP[ÐbcÔ d×dˆKØ!×)Ñ)×/Ð/¨RÒ/�Ü×(Ò(¬×3Õ3Ü!¨B¯N©NÓ,<Ð"< &¼e×DÒDÌÈVËÐXYÔIYÜ(+¨F°1­I«˜Ø÷ 4×3Ó3ò 0ð   °DÐ!8Ð:NÐP]Ð^€Eð=ØðTÜ×'Ò'Ô(<ÈØ.3ô5÷ 5ð 5ð Ò*±?×3DÒ3DØð
 Ò"Ø×ÑÔÜ×$Ò$¤Y×/Õ/Ø!×-Ñ-Ó/×/Ð/÷ 0àÒ+Ü×(Ò(¬×3Õ3Ø$×+Ñ+°tÐ+Ô<÷ 4Ñ3ñ ,ñ	 #ô	 —-’- Ó)×)Ð)ØÔ*±?×3DÕ3DØàÒ"Ø×ÑÔÜ×$Ò$¤Y×/Õ/Ø!×-Ñ-Ó/×/Ð/÷ 0àÒ+Ü×(Ò(¬×3Õ3Ø$×+Ñ+°tÐ+Ô<÷ 4Ñ3ñ ,ñ	 #ò9Vò eç3×3Ñ3ûô ô &Ø&*Ð*ˆÜ�‰ð \à $ð 	÷ 	&ð&úò5øäô TÜ—‘ÐCÈd�×SðTúò 0÷ 0×/Ð/ú÷ 4×3Ð3úò *ñ 0÷ 0×/ú÷ 4×3Ð3ûð Ò"Ø×ÑÔÜ×$Ò$¤Y×/Õ/Ø!×-Ñ-Ó/×/Ñ/÷ 0×/×/Ð/úàÒ+Ü×(Ò(¬×3Õ3Ø$×+Ñ+°tÐ+Ô<÷ 4×3×3Ð3úð ,ð	 #üs¼  ‚R2¡3L< ÁAL< Â'L!Â(L< Â,R2Â- L< ÃL$ÃL< Ã%(L< ÄA
L'Å	L< Å R2Å!L< Å-	R2Å7O+ Å9 M+ ÆM(ÆM+ ÆO+ Æ06R2Ç&NÇ:NÇ;NÇ?.R2È-N+É R2ÉO+ É'N?É(O+ É2
O+ Ê6R2Ê9OËOËOË.R2Ì OÌR2Ì!L< Ì$L< Ì'L9Ì2
L< Ì<%M%Í!R2Í$M%Í%R2Í(M+ Í+"NÎO+ ÎNÎO+ ÎNÎN(	Î"	R2Î+N<	Î6	R2Î?O+ ÏOÏO	ÏR2ÏO(	Ï"	R2Ï+7R/Ð"Q	Ð6P9Ð7Q	Ð<	R/ÑQÑ.R/Ñ>R	Ò	R/ÒR*Ò%
R/Ò/R2)r»   zgateway.heartbeat)Úlogszgateway-shutdown-watchdog.log)rR   r4   )NN);r@   Ú
__future__r   r1  rx   rí   rê   Úloggingr�   rï   rm   rt   r   r   r   Úpathlibr   Útypingr   r   r	   r
   Úgateway.restartr   Úhermes_constantsr   r   Úutilsr   Ú	getLoggerr<   rr   rÔ   r/  rS   Ú DEFAULT_LOOP_WATCHDOG_INTERVAL_SÚDEFAULT_LOOP_WATCHDOG_TIMEOUT_SÚ!DEFAULT_LOOP_WATCHDOG_MAX_STRIKESrµ   rÀ   r   r!   rC   rU   rŠ   r~   r}   rª   r±   r¶   r½   r|   rÏ   rÖ   r{   r	  r  r$  rC  r   r   r   Ú<module>rP     s¦  ðñ^õ #ã Û Û Û Û Û 	Û 
Û Û ß ,Ñ ,Ý ß 0Ó 0å =ß EÝ #à	×	Ò	˜8Ó	$€ð %)Ð !Ø#Ð Ø&)Ð #Ø#'Ð  Ø"&Ð ð %&Ð !Ø4Ð ØCÐ ÷÷!ñ !÷"[ñ [ð 8[÷^ð<;Ø@`ð<;à:ð<;ð 9ð<;ð 7÷	<;õ~õXõ*iõ
@÷6÷
C÷:ðØ ðØ@DðàðàCG÷ð05Ø.O÷5ð6Ø&?÷6ð.1Ø?Cð1à:>ð1àQRð1ð !%ð1ð 3N÷1õhõKð:=Ø7ð:=ØW[ð:=àð:=àQU÷:=ñ :=r   