+
    iHàjT,  ã                  óì   € R t ^ RIHt ^ RIt^ RIt^ RIt^ RIt^ RIHtH	t	H
t
HtHtHt ^ RIHt ]P                   ! ]4      tRtRtRtRtR	t ! R
 R]4      tR R ltR R ltRR R llt ! R R4      tR# )u:  Turn liveness watchdog: force-abort turns that stall silently.

A turn can wedge mid-flight with no error and its durable lease still renewing, so nothing
ever frees the session. This module owns config resolution (``agent.turn_liveness``,
validated â€” a typo, NaN or Inf warns and falls back), the sampling state machine, and the
watcher thread. Race safety: the abort decision is bound to the observed
``(generation, timestamp)``; the commit callback revalidates it under the same lock
``_touch_activity`` stamps with, so a turn that resumed is never hard-cancelled.
)ÚannotationsN)ÚAnyÚCallableÚDictÚ
NamedTupleÚOptionalÚTuple)ÚAwakeIdleMeterg     À‚@g      .@g{®Gáz„?zagent.turn_liveness.timeout_szagent.turn_liveness.poll_sc                  ó:   € ] tR t^t$ RtR]R&   R]R&   R]R&   RtR	# )
ÚActivitySnapshotzOne activity-clock observation; ``(generation, activity_ts)`` must be
revalidated by the commit callback under the shared lock.ÚintÚ
generationzOptional[float]Úactivity_tsÚfloatÚidle_seconds© N)Ú__name__Ú
__module__Ú__qualname__Ú__firstlineno__Ú__doc__Ú__annotations__Ú__static_attributes__r   ó    Ú8/home/ubuntu/.hermes/hermes-agent/agent/turn_liveness.pyr   r      s   ‡ ñAð ƒOØ Ó Ø×r   r   c               ó(   € V ^8„  d   QhRRRRRRRR/# )	é   ÚkeyÚstrÚrawr   Údefaultr   ÚreturnÚNoner   )Úformats   "r   Ú__annotate__r$   (   s.   € ÷ iñ i˜Sð i sð i°Uð i¸tñ ir   c                ó4   € \         P                  R WV4       R# )u?   Invalid %s in config.yaml: %r â€” falling back to default %.1f.N)ÚloggerÚwarning)r   r   r    s   &&&r   Ú_warn_invalid_valuer(   (   s   € Ü
‡N�NÐTÐVYÐ`gÖhr   c               ó(   € V ^8„  d   QhRRRRRRRR/# )r   r   r   r    r   r   r   r!   r   )r#   s   "r   r$   r$   ,   s(   € ÷ ñ  ð °%ð ¸cð Àeñ r   c               óÀ   €  \        V 4      p\        P
                  ! V4      '       g   \        W V4       V# V#   \        \        3 d    \        P                  p LOi ; i)zÀCoerce one duration knob, rejecting typos, NaN and Inf (never raises).

NaN would silently disable the timeout via the ``> 0`` comparison and
Inf would freeze the poll loop in ``Event.wait``.
)r   Ú	TypeErrorÚ
ValueErrorÚmathÚnanÚisfiniter(   )r   r    r   Úvalues   &$$ r   Ú_resolve_finite_secondsr1   ,   sT   € ðÜ�c“
ˆô �=Š=˜×ÒÜ˜C gÔ.ØˆØ€Løô ”zÐ"ô Ü—‘Šðús   ‚9 ¹!AÁAc               ó    € V ^8„  d   QhRRRR/# )r   ÚconfigzOptional[dict[str, Any]]r!   ztuple[Optional[float], float]r   )r#   s   "r   r$   r$   <   s   € ÷ ñ Ø$ðà"ñr   c                ó4  € \        V \        4      '       d   V P                  R4      MRp\        V\        4      '       d   VP                  R4      MRp\        V\        4      '       d   TM/ pVe(   \        V\        4      '       g   \        RV\        4       \        VP                  R\        4      \        \        R7      p\        VP                  R\        4      \        \        R7      pV^ 8:  d   \        \        V\        4       \        pV^ 8:  d   RpWE3# )z®Resolve ``(timeout_s, poll_s)``; ``timeout_s <= 0`` opts out (``None``).

Invalid values (typo, NaN, Inf, non-positive poll) warn and fall back to
the default. Never raises.
ÚagentNÚturn_livenesszagent.turn_livenessÚ	timeout_s)r    r   Úpoll_s)	Ú
isinstanceÚdictÚgetr(   ÚDEFAULT_TURN_LIVENESS_TIMEOUT_Sr1   Ú_CONFIG_TIMEOUT_KEYÚDEFAULT_TURN_LIVENESS_POLL_SÚ_CONFIG_POLL_KEY)r3   Ú	agent_cfgÚraw_sectionÚsectionr7   r8   s   &     r   Úresolve_turn_liveness_settingsrC   <   sâ   € ô (2°&¼$×'?Ò'?�—
‘
˜7Ô#ÀT€IÜ4>¸yÌ$×4OÒ4O�)—-‘- Ô0ÐUY€KÜ-7¸ÄT×-JÒ-J™kÐPR€GØÒ¤z°+¼t×'DÒ'DÜÐ1°;Ô@_Ô`ä'Ø�‰�KÔ!@ÓAÜ/Ô5Hô€Iô %Ø�‰�HÔ:Ó;ÔEaÜô€Fð �„{ÜÔ,¨fÔ6RÔSÜ-ˆØ�A„~Øˆ	ØÐÐr   c                  ón   € ] tR t^ZtRtR R ltR tR tR R ltR R	 lt	R
 R lt
R R ltR R ltRtR# )ÚTurnLivenessWatchdoga  Sampled-idle watchdog bound to one conversation turn (via the shared
periodic scheduler; timer thread orders, body runs on its own worker).

``activity_lock`` must be the SAME lock ``AIAgent._touch_activity`` stamps
the activity clock with; run_agent owns the lease state and callbacks.
c               ó@   € V ^8„  d   QhRRRRRRRRRR	R
RRRRRRRRR/
# )r   r5   r   Ú
session_idr   r7   r   r8   Ú
stop_eventzthreading.EventÚactivity_lockzthreading.LockÚis_turn_activezCallable[[], bool]Úcommit_abortz'Callable[[ActivitySnapshot, str], bool]Údeactivate_turnzCallable[[], None]r!   r"   r   )r#   s   "r   r$   Ú!TurnLivenessWatchdog.__annotate__b   se   € ÷ ,ñ ,Øð,Ø),ð,Ø9>ð,ØHMð,à#ð,à4Bð,ð +ð,ð ;bð,ð ,ð	,ð
 
ñ,r   c               	óÔ   € Wn         W n        \        V4      V n        \	        \
        \        V4      4      V n        WPn        W`n        Wpn	        W€n
        W�n        \        4       V n        R # ©N)Ú_agentÚ_session_idr   Ú
_timeout_sÚmaxÚMIN_TURN_LIVENESS_POLL_SÚ_poll_sÚ_stop_eventÚ_activity_lockÚ_is_turn_activeÚ_commit_abortÚ_deactivate_turnr	   Ú_awake_idle)
Úselfr5   rG   r7   r8   rH   rI   rJ   rK   rL   s
   &&$$$$$$$$r   Ú__init__ÚTurnLivenessWatchdog.__init__b   sW   € ð ŒØ%ÔÜ 	Ó*ˆŒÜÔ3´U¸6³]ÓCˆŒØ%ÔØ+ÔØ-ÔØ)ÔØ /Ôä)Ó+ˆÖr   c                óH   € ^ RI Hp V! V P                  V P                  4      # )zŸStart polling via the shared periodic scheduler; returns the cancel handle.
Scheduled at turn entry, after the turn-active flag and activity clock are stamped.)Úschedule)Úagent.periodic_schedulerr`   Ú_tickrU   )r\   r`   s   & r   r`   ÚTurnLivenessWatchdog.schedulet   s   € õ 	6á˜Ÿ
™
 D§L¡LÓ1Ð1r   c                óÚ  € V P                   P                  4       '       d   R# V P                  4       pVf   R# V P                  P	                  VP
                  4      V P                  8  d   R# V P                  V4       R\        VP
                  4       R2pV P                  W4      '       g   R# T P                  4        V P                  V4       ^ RIHp V! V P                  4       R# )z6One poll. Returns False when the watchdog is finished.FNzTurn made no progress for z#s; aborting to release the session.)Úrecord_watchdog_turn_abort)rV   Úis_setÚ_sampler[   Úmeasurer   rR   Ú_surface_stallr   rY   rZ   Ú_surface_committed_abortÚ/hermes_cli.observability.shared_metrics_processre   rP   )r\   ÚsnapshotÚmessagere   s   &   r   rb   ÚTurnLivenessWatchdog._tick{   s¿   € à×Ñ×"Ñ"×$Ò$ÙØ—<‘<“>ˆØÒÙØ×Ñ×#Ñ# H×$9Ñ$9Ó:¸T¿_¹_ÔLÙð 	×Ñ˜HÔ%Ø.¬s°8×3HÑ3HÓ/IÐ.JÐJmÐnˆØ×!Ñ! (×4Ò4Ùà×ÑÔØ×%Ñ% hÔ/Ý^Ù" 4§;¡;Ô/Ùr   c               ó   € V ^8„  d   QhRR/# )r   r!   zOptional[ActivitySnapshot]r   )r#   s   "r   r$   rM   –   s   € ÷ Gñ GÐ3ñ Gr   c                	óp  € V P                   ;_uu_ 4        V P                  4       '       g    R R R 4       R # \        V P                  R^ 4      p\        V P                  RR 4      pR R R 4       Xf   RM%\	        R\
        P
                  ! 4       V,
          4      p\        XW#4      #   + '       g   i     LH; i)NÚ"_turn_liveness_activity_generationÚ_last_activity_tsg        )rW   rX   ÚgetattrrP   rS   Útimer   )r\   r   r   r   s   &   r   rg   ÚTurnLivenessWatchdog._sample–   sŒ   € Ø× × Ó Ø×'Ñ'×)Ò)Ø÷ !Ñ ô ! §¡Ð.RÐTUÓVˆJÜ! $§+¡+Ð/BÀDÓIˆK÷	 !ð
 *Ò1‘s´s¸3ÄÇ	Â	ÃÈkÕ@YÓ7ZˆÜ 
¨KÓFÐF÷ !× ús   •B%·.B%Â%B5	c               ó   € V ^8„  d   QhRR/# )r   r!   r"   r   )r#   s   "r   r$   rM   Ÿ   s   € ÷ #Fñ #F¨ñ #Fr   c                ó  €  ^ RI p^ RIpVP                  4       pR.p\        P                  ! 4        Uu/ uF#  qUP
                  '       g   K  VP
                  VbK%  	  pp\        V P                  RR4      p^ pVP                  4        F©  w  ršV^(8¼  d(   VP                  R\        V4      V,
           R24        MzV^,          pVP                  V	4      pVe   VP                  MRpW—8X  d   RMRpVP                  R	V R
V	 V 24       VP                  R VP                  V
4       4       4       K«  	  \        P!                  RP#                  V4      4       R# u upi   \$         d    \        P'                  RRR7        R# i ; i)u*  Dump the turn thread's stack to the error log when the watchdog fires (#131740).

A stall this deep cannot be localized from the activity description alone â€” the
reported wedge sat between the ``Turn ended`` log line and ``run_conversation``
returning, where no activity stamps exist. The frame at the moment of the stall
is the only direct evidence; the turn thread's frame is marked so the log names
the wedged call site (in-tree precedent for thread-stack diagnostics:
``tools/delegate_tool_child_run.py``). Best-effort: never raises into the tick.
Nz2## All thread stacks at turn stall (watchdog dump)Ú_execution_thread_idz  <z more threads omitted>Ú?z <- turn threadÚ z### thread z tid=c              3  óx   "  € T F0  pVP                  4       P                  R 4       F
  pRV 2x € K  	  K2  	  R# 5i)Ú
z  N)ÚrstripÚsplit)Ú.0Ú
frame_lineÚsubs   &  r   Ú	<genexpr>Ú?TurnLivenessWatchdog._dump_turn_thread_stack.<locals>.<genexpr>»   s>   é € ð á&C˜
Ø)×0Ñ0Ó2×8Ñ8¸Ö>˜ð ˜˜•Já>ñ Û&Cùs   ‚8:r|   zwatchdog stack dump failedT©Úexc_info)ÚsysÚ	tracebackÚ_current_framesÚ	threadingÚ	enumerateÚidentrs   rP   ÚitemsÚappendÚlenr;   ÚnameÚextendÚformat_stackr&   ÚerrorÚjoinÚ	ExceptionÚdebug)r\   r†   r‡   ÚframesÚlinesÚtÚby_identÚ
turn_identÚdumpedr‹   ÚframeÚthreadr�   Úmarkers   &             r   Ú_dump_turn_thread_stackÚ,TurnLivenessWatchdog._dump_turn_thread_stackŸ   s[  € ð	FÛÛà×(Ñ(Ó*ˆFØIÐJˆEÜ,5×,?Ò,?Ô,AÓMÑ,A qÇWÅWœ
˜Ÿ™ š
Ñ,AˆHÐMÜ  §¡Ð.DÀdÓKˆJØˆFØ &§¡¦‘�Ø˜R”<Ø—L‘L 3¤s¨6£{°VÕ';Ð&<Ð<RÐ!SÔTÙØ˜!•�Ø!Ÿ™ eÓ,�Ø&,Ò&8�v—{’{¸c�Ø.3Ô.AÑ*Àr�Ø—‘˜{¨4¨&°°e°W¸V¸HÐEÔFØ—‘ñ à&/×&<Ñ&<¸UÔ&Cóö ñ !/ô �L‰L˜Ÿ™ 5Ó)Ö*ùò# Nøô$ ô 	FÜ�L‰LÐ5ÀˆL×Eð	Fús)   ‚2E  ´EÁEÁC<E  ÅE  Å "FÆFc               ó$   € V ^8„  d   QhRRRRRR/# )r   Útextr   Ú	debug_msgr!   r"   r   )r#   s   "r   r$   rM   Ä   s!   € ÷ 3ñ 3 #ð 3°#ð 3¸$ñ 3r   c                	ó¾   € \        V P                  R R4      p\        V4      '       g   R#  V! V4       R#   \         d    \        P                  TRR7        R# i ; i)Ú_emit_warningNTr„   )rs   rP   Úcallabler”   r&   r•   )r\   r¢   r£   Úemit_warnings   &&& r   r¥   Ú"TurnLivenessWatchdog._emit_warningÄ   sN   € Ü˜tŸ{™{¨O¸TÓBˆÜ˜×%Ò%Ùð	3Ù˜ÖøÜô 	3Ü�L‰L˜¨TˆL×2ð	3ús   ¬6 ¶"AÁAc               ó    € V ^8„  d   QhRRRR/# ©r   rl   r   r!   r"   r   )r#   s   "r   r$   rM   Í   s   € ÷ 
ñ 
Ð'7ð 
¸Dñ 
r   c           
     ó„  € VP                   p\        V RR4      V8X  d   R# W n        V P                  4        \        P                  R\        V P                  RR4      ;'       g    V P                  VP                  \        V P                  RR4      4       V P                  R\        VP                  4       R2R4       R# )	zžLog + UI-warn that recovery is beginning (not that it committed). Rate-limited per
activity generation so repeatedly declined aborts do not re-log every poll.Ú_last_surfaced_generationNzÁTurn liveness watchdog fired for session %s: no progress for %.1fs (last activity: %r). Attempting recovery: force-interrupting the turn and stopping lease renewal if it cannot resume (#95548).rG   Ú_last_activity_descu*   âš ï¸� This turn stopped making progress (zEs without activity); attempting recovery so the session can continue.z$Failed to emit turn liveness warning)r   rs   r¬   rŸ   r&   r’   rP   rQ   r   r¥   r   )r\   rl   r   s   && r   ri   Ú#TurnLivenessWatchdog._surface_stallÍ   s·   € ð ×(Ñ(ˆ
Ü�4Ð4°dÓ;¸zÔIÙØ)3Ô&Ø×$Ñ$Ô&Ü�‰ðCô �D—K‘K ¨tÓ4×HÐH¸×8HÑ8HØ×!Ñ!Ü�D—K‘KÐ!6¸Ó=ô	
ð 	×ÑðÜ�H×)Ñ)Ó*Ð+ð ,?ð?ð 3ö		
r   c               ó    € V ^8„  d   QhRRRR/# rª   r   )r#   s   "r   r$   rM   å   s   € ÷ 
ñ 
Ð1Að 
Àdñ 
r   c                óì   € \         P                  R\        V P                  RR4      ;'       g    V P                  VP
                  4       V P                  R\        VP
                  4       R2R4       R# )ap  Publish the definitive settlement once the abort has authority.

Runs only once ``_commit_abort`` succeeded (the interrupt was published) and the turn lease was
deactivated: the turn IS force-aborted and lease renewal IS stopped, so stating that is now true.
Separated from the pre-commit surface so a declined abort never reports a committed outcome (#95663
review).
zTurn liveness watchdog aborted turn for session %s: no progress for %.1fs; turn interrupted and lease renewal stopped (#95548).rG   Nu.   âš ï¸� Turn aborted by the liveness watchdog (zgs without activity); lease renewal stopped so the session can be reclaimed. You can retry your message.z&Failed to emit committed-abort warning)r&   r’   rs   rP   rQ   r   r¥   r   )r\   rl   s   &&r   rj   Ú-TurnLivenessWatchdog._surface_committed_abortå   ss   € ô 	�‰ð ô �D—K‘K ¨tÓ4×HÐH¸×8HÑ8HØ×!Ñ!ô	
ð 	×ÑðÜ�H×)Ñ)Ó*Ð+ð ,*ð*ð 5ö	
r   )rW   rP   r[   rY   rZ   rX   r¬   rU   rQ   rV   rR   N)r   r   r   r   r   r]   r`   rb   rg   rŸ   r¥   ri   rj   r   r   r   r   rE   rE   Z   s7   † ñõ,ò$2òõ6Gõ#FõJ3õ
÷0
ñ 
r   rE   rO   )r   Ú
__future__r   Úloggingr-   r‰   rt   Útypingr   r   r   r   r   r   Úagent.session_activityr	   Ú	getLoggerr   r&   r<   r>   rT   r=   r?   r   r(   r1   rC   rE   r   r   r   Ú<module>r·      sz   ðñõ #ã Û Û Û ß C× Cå 1à	×	Ò	˜8Ó	$€à"'Ð Ø#Ð ØÐ à5Ð Ø/Ð ô�zô õiõ÷ ÷<`
ó `
r   