߲jVN5UdZddlZddlZddlmZddlmZmZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlZddlZddlZ ddlZ ddlZddlmZmZmZddlmZddlmZddlmZmZmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'm(Z(m)Z)m*Z*e jVe,Z-dd l.m/Z0dd l.m1Z1dd l2m3Z3m4Z4da5da6e"e7e8d<de7fdZ9ejuZ;e;dz ZeZgd?e#eFe%fde#eFe%ffd@ZhdpdAe%dBeFdeFfdCZiedhdDZjdEZkdFZld?e#eFe%fde7fdGZmd?e#eFe%fdeFfdHZnd?e#eFe%fde#eFe%ffdIZod?e#eFe%fde7fdJZpd?e#eFe%fde7fdKZqd?e#eFe%fdeFfdLZrd?e#eFe%fde7fdMZsd?e#eFe%fde7fdNZtdOefdPZudOefdQZvdOedRe"ejddfdSZxdOede7fdTZydUeddfdVZzdWZ{dXe%de"e|fdYZ}dZdd[d\Z~d]eFde|fd^Zid_d`dad`dbdcdddcdedfdgdfdhdfdidjdkdjdldjdmdndodndpdndqdndrdsdtdsdudvdwdviZdxdxdxdydydzdzd{Zd|eFde"efd}Zd~eFde"eFfdZdeFdeFdeFdeFde#eFe%ff dZde|de#eFe%ffdZdeFde#eFe%ffdZdedefdZdAe%de"efdZdedede7fdZdedede7fdZddde#eFe%fdede"eFde"eFfdZdZdZdede|fdZdZdeZde|deFfdZdae|e8d<dZgZee8d<d?e#eFe%fde#eFe%fdede7fdZiZe#eFe"eZfe8d<de#eFe%fde"eZfdZdeFde#eFe%fdeddfdZd?e#eFe%fdeFddfdZde#eFe%ffdZde#eFe%fdede7fdZdZdZdZde#eFe%fdededeFfdZdae|e8d<gZee8d<d?e#eFe%fdededdfdZde#eFe%ffdZdqde#eFe%fde"eFde"eFfdZdeFd|eFdeFddfdZdrde7ddfdZdeFde"eZfdZde"eZfdZde"eZfdZde|fdZdndZdeFddfdZdndZde"eFfdZdede(e%e7ffdÄZde$e#eFe%ffdĄZde"e$e#eFe%ffdńZdede"e(e|e|e|ffdƄZde"e(e|e|e|fddfdȄZde$e#eFe%fde"e*eFde$e#eFe%ffd˄Zdd̜de$e#eFe%fde"e*eFde$e#eFe%ffd̈́ZdOe"eFddfd΄Zdede$e#eFe%fdeFfdτZdZdddќde$e#eFe%fde"e*eFde7fdӄZdddќde$e#eFe%fde"e*eFde7fdԄZeZ dqd%e%d6e&e$e#eFe%fe|e#eFe%fge%fde%de%fdքZd?e#eFe%fddfdׄZd?e#eFe%fddfd؄Zde"eFde"eFfdڄZde(e"eFe"eFffdۄZddܜdAe%de7de"eFfdބZdAe%de"eFfd߄Zde%de"e$eFfdZdAe%de"e$eFfdZdAe%de"eFfdZdAe%de"eFfdZeeeeeedee7eed Ze#eFe&e%ge%ffe8d<deeedZe#eFe&e%ge%ffe8d<de"eFde"eFde7de"eFddf dZde%defdZde#eFe%fdeFde%deFde"eFf dZ dsde"eFdeFde"eFdXe"e|de"eFde"e#eFe%fde|d?e7d@e7dAe7de)e7e#eFe%fff dBZd%eFd8eFde7fdCZdDZdEeFdFe%dGe&e%ge%fde%fdHZdeZfdIZdd̜dJe$e#eFe%fdede"e'eFde7fdKZde$e#eFe%ffdLZeGdMdNZdJe$e#eFe%fde7fdOZd?e#eFe%fdPeddfdQZd?e#eFe%fdPede"eFfdRZeGdSdTZdUede7fdVZdUedefdWZdUede7fdXZdUedYe|de7fdZZdUede7fd[Zd?e#eFe%fdPede7fd\Zd?e#eFe%fdPede"eFfd]Zd?e#eFe%fdPed^eZde7fd_Z de$e#eFe%ffd`Z daZ de|fdbZ dcedde|de|fdeZ d%eFdeFfdfZdge%deFfdhZde'eFfdiZ dodje"e#eFeFfdke"e$eFde#eFe%ffdlZd%eFddfdmZy#e$rdZY !wxYw#e$rdZY *wxYw(uzcCron job storage: ~/.hermes/cron/jobs.json; output in ~/.hermes/cron/output/{job_id}/{timestamp}.mdN) ContextVar) dataclassfield)datetime timedeltatimezone)Path)get_hermes_home)CLAIM_TTL_INACTIVITY_HEADROOMFIRE_CLAIM_SKEW_SECONDSFIRE_CLAIM_TTL_SECONDS)cron_env_setting) OptionalDictListAnyCallableSetTupleUnion Collection)now) get_timezone)atomic_replaceatomic_write_text HAS_CRONITERreturnct ddlm}|adattSttS#t$rdaYttSwxYw)zDImport croniter on first use; honor a pre-set HAS_CRONITER override.r)croniterTF)rr ImportErrorbool) _croniters ./home/ubuntu/.hermes/hermes-agent/cron/jobs.py_ensure_croniterr$/sT ! 6 HL  4  ! L   !s 0 A  A cron jobs.jsonticker_heartbeatticker_last_success<_fire_fence_locksg>@outputxT)frozencFeZdZUeed<eed<eed<ededdfdZy)_CronStorePathscron_dir jobs_file output_dirrc"|||dz |dz S)Nr&r+)clsr0s r#for_dirz_CronStorePaths.for_dirls8X 3X5HIIN)__name__ __module__ __qualname__r __annotations__ classmethodr6r4r7r#r/r/fs7NOJtJ(9JJr7r/cron_store_override)default_cron_store_overrideceZdZdZdefdZy)_SelfRemovalDeliveryzKMutable run-local marker shared with the agent's copied ContextVar context.job_idc ||_d|_yNF)rBremovedselfrBs r#__init__z_SelfRemovalDelivery.__init__xs  r7N)r8r9r:__doc__strrHr4r7r#rArAusUsr7rAself_removal_delivery_self_removal_deliveryrBc#Kt|}tj|} |tj|y#tj|wxYww)zEPermit this run's final delivery after it removes its own job record.N)rArLsetreset)rBmarkertokens r#self_removal_delivery_scoperRsI"& )F " & &v .E, $$U+$$U+s!A>AAActj}||jk7s |jsyt fdt DS)zWhether the active run deleted exactly its own job record and no record has since taken its id (a replacement record belongs to another owner, so that stays fail-closed).Fc3FK|]}|jdk7ywidNget.0itemrBs r# z0self_removal_delivery_allowed..s@Dtxx~'@s!)rLrXrBrEall load_jobs)rBrPs` r#self_removal_delivery_allowedr_s?$ ' ' )F ~&0 @IK@ @@r7ctj}||Stttt }|t k7r|Stj}|tk(r|Stj|dz S)a^Paths pinned to this execution context's profile. Precedence: (1) active use_cron_store() override; (2) deliberately re-pointed module constants; (3) the ACTIVE profile home via get_hermes_home(), so re-pointing HERMES_HOME after import uses ITS OWN store rather than the user's real jobs.json frozen at import; (4) import-time constants.r%) r?rXr/CRON_DIR JOBS_FILE OUTPUT_DIR _IMPORT_STOREr resolve HERMES_DIRr6)overridelive_constantshomes r#_current_cron_storerjsm $'')H$Xy*EN&   $ $ &D z  " "4&= 11r7ric#Ktjtjt |j j dz } dtj|y#tj|wxYww)z@Route cron storage to ``home`` without mutating process globals.r%N)r?rNr/r6r expanduserrerO)rirQs r#use_cron_storermsg ! $ $T 5 5 7 ? ? AF JK ME* ""5)""5)sABA.B.BBc*tjS)z>Return the output directory for the active cron store context.)rjr2r4r7r#get_cron_output_dirros  + ++r7ig@ctdj} |r t|nt}|dkrtt St|tztt S#tt f$r t}YKwxYw)uOne-shot running-claim TTL from ``HERMES_CRON_TIMEOUT``: unset/invalid → 600s → 1800s; ``0`` (unlimited) → the fixed floor; positive N → ``max(N * headroom, floor)``.HERMES_CRON_TIMEOUTr) rstripfloat _DEFAULT_CRON_INACTIVITY_TIMEOUT ValueError TypeErrorONESHOT_RUN_CLAIM_TTL_SECONDSmaxr )rawtimeouts r#_oneshot_run_claim_ttl_secondsr{st 0 1 7 7 9C3 #%*)I!|233 w66>[8\ ]]  "323sA""A:9A:cn ddlm}||S#t$rtj d|dYywxYw)u4True when the scheduler in THIS process is still running ``job_id``: the run_claim TTL alone cannot distinguish "claiming tick died" from "alive but slow". Lazy import: scheduler imports us. Direct liveness signal for stale-entry recovery (#62002): the run_claim TTL alone cannot distinguish "the claiming tick died" from "the run is alive but slow" — a run stalled on network I/O (or a laptop that slept mid-run) legitimately outlives the TTL. The in-process ticker and the run share this process, so the scheduler's running set settles the common single-gateway case without any claim-age guesswork. Home-scoped: one process ticks every profile, so the host-wide union of bare job ids would let profile A's running ``daily-brief`` keep profile B's stale one-shot alive forever. r)is_job_runningzsCron running-set liveness check failed for job %r; keeping the entry to avoid deleting a possibly live one-shot runTexc_info)cron.schedulerr} Exceptionloggerwarning)rBr}s r#_job_running_in_this_processrsE1f%%  C T  # s !44c0tjdz S)z=Return the advisory lock path for the current cron directory.z .jobs.lock)rjr0r4r7r#_jobs_lock_filers  ) )L 88r7rzctOtj|z} tj|tjtj zyt5ttd|jttddyy#t tf$r1tj|k\rYytjdYnwxYw)a'Bounded exclusive lock: True when acquired, False on timeout, None when no backend exists. A blocking flock(LOCK_EX) taken under the in-process lock would let a wedged sibling freeze EVERY cron function forever, so poll LOCK_NB against a deadline; the caller picks the degraded mode.NTFg?lockingLK_LOCK) fcntltime monotonicflockLOCK_EXLOCK_NBOSErrorIOErrorsleepmsvcrtgetattrfileno)lock_fdrzdeadlines r#_acquire_flockrs  >>#g-  GU]]U]]%BC " "7>>#3WVY5OQRS W% >>#x/  3 s5B&C:CCcD t%tj|tjn:t4t td|j t tdd|jy#t tf$rY"wxYw#|jwxYw)zIUnlock (best effort) and close a lock file opened for ``_acquire_flock``.NrLK_UNLCKr) rrLOCK_UNrrrrrclose)rs r#_release_flockrsy   KK /   &GFI &w~~'79TVW X   W      s$A%A88B B B  B Bc#Kttdd}|r-|dzt_ dtxjdzc_yt5dt_dt_d} t t tdd}|jdt|tdurWtjd tttjt5|j!dddd} d| t'| dt_dt_ dddy#txjdzc_wxYw#1swY\xYw#tt"f$r }tj%d |Yd}~d}~wwxYw#| t'|wwxYw#dt_dt_wxYw#1swYyxYww) uSerialize a load_jobs→modify→save_jobs critical section: in-process RLock (parallel tick threads) plus a cross-process flock on ``/.jobs.lock`` (gateway vs. CLI writes — otherwise a `cron pause` could be clobbered and keep firing). Nested calls in one thread reuse the held lock. Without a flock backend, or on flock timeout (logged loudly), it degrades to in-process-only locking: a briefly torn cross-process write beats a dead scheduler.depthrrNa+utf-8encodingFuTimed out after %.0fs waiting for the cron jobs lock (%s) — another process is holding it. Proceeding with in-process locking only so the scheduler stays alive (#60703).zSjobs.json cross-process lock unavailable (%s); proceeding with in-process lock only)r_jobs_lock_stater_jobs_file_lock load_stamp ensure_dirsopenrseekr_JOBS_LOCK_TIMEOUT_SECONDSrerror contextlibsuppressrrrrr)rres r# _jobs_lockrs $gq 1E !& (   " "a ' " "/!" '+# / J 0$I Q!'+EF%OLLA3O4E G $,,W5( ("G  ,&"7+%&  "*.  'E"/"/  " "a ' ".((W% J FGHJJ J&"7+'&'  "*.  'E"/"/s"GD1 G F;$A?E#E4 E?F F F;( G1E  GE EF *FF F  F  FF  F88F;;GGc#Ktj}|jd|}t5tj |t j}dddjtstjd|dytjj di}||vr |||jy t|dt!j"t j$|j&dz }d}d} t)|d d }|j+d t-|t}|tjd n|stjd|t/|}|||< ||j5|d||r t7|n|j9 |jy#1swYuxYw#|jwxYw#t0t2f$r!} tjd|| Yd} ~ d} ~ wwxYw#|j5|d||r t7|w|j9wwxYw#|jwxYww)aSerialize one job's owner mutations and external side effects. Unlike the global jobs lock this may be held across network delivery; scoped to one profile + job so unrelated jobs keep progressing. Fails closed when cross-process locking is unavailable.::N)rzz9Timed out waiting for local fire fence %s; failing closedFheldz.fire-z.lockrrrrz1No cross-process lock backend for cron fire fencez3Timed out waiting for fire fence %s; failing closedz&Cron fire fence unavailable for %s: %s)rjr0re_fire_fence_locks_guardr* setdefault threadingRLockacquirerrr_fire_fence_lock_state__dict__releaseruuiduuid5 NAMESPACE_URLhexrrrr!rrpoprr) rBr0lock_key local_lock held_locks lock_pathracquiredresultexcs r#_fire_job_lockrGs. #$--H""$%Rx0H O&11(IOO.s Q&8PQs$$ fire_claimby)rrnextr^ isinstancedictrXr_)rBrrjobclaim owns_claims` r#fire_claim_fencers   8K   \ CQQSWXC-7T-BCGGL)E#E40VUYYt_5VJ{:6B  C    C C  s> C B? C  B?A4B3 B?* C 3B< 8B??CC rVct|xsdj}|r:|dvs6d|vs2d|vs.t|jst|jrt d|t j|z S)zResolve a job's output directory, rejecting any path-escape attempt (``..``, absolute paths, separators): only a single safe path component is accepted.>.../\z%Invalid cron job id for output path: )rJrrr is_absolutedriverurjr2)rBtexts r#_job_output_dirrst v|  " " $D DK'3$;$$, : ! ! #tDz'7'7@ KLL  + +d 22r7skillskillsc||r|gng}nt|tr|g}n t|}g}|D]8}t|xsdj}|s#||vs(|j |:|S)zPNormalize legacy/single-skill and multi-skill inputs into a unique ordered list.r)rrJlistrrappend)rr raw_items normalizedr[rs r#_normalize_skill_listrsv ~$UG" FC H L J$4:2$$& D *   d #$ r7rct|}t|jd|jd}||d<|r|dnd|d<|S)zLReturn a job dict with canonical `skills` and legacy `skill` fields aligned.rrrN)rrrX)rrrs r#_apply_skill_fieldsrsKcJ ":>>'#:JNN8rpromptscriptrno_agentzCron job has nothing to run: the prompt is blank and no script or skill(s) are set. Provide a prompt, a script, or at least one skill.udno_agent=True requires a script — with no agent and no script there is nothing for the job to run.c$tjdjs(tjdjrytjdjdryt fddDS)zTrue when a job record has nothing runnable (blank prompt, no script, no skills) AND at least one payload field is explicitly present. ``no_agent`` already requires a script.rrFrrc3&K|]}|v ywNr4)rZkrs r#r\z'job_payload_is_empty..sIAqCxI)rrrr)rrXrrranyrs`r#job_payload_is_emptyrsm)*0026FswwxGX6Y6_6_6aSWWW-swwx/@A I!HI IIr7c$t|jdj}|r|S|jd}t|tr6dD]0}t|j|j}|s.|cSy| t |Sy)Nschedule_displayschedule)displayrexprrun_at?)rrXrrrrrJ)rrrkeyrs r#_schedule_display_for_jobrssww'9:;AACGwwz"H(D!9 C#HLL$56<<>D     8} r7ct|}t|jddx}|d<t|jdx}|d<t|jdj}|sh|xsM|jdr|ddndxs0t|jdjxs|xsd }|d d jxsd }||d<t ||d <t ||d <|S)zRead-safe job shape: legacy/hand-edited records may have nullable ``prompt``, ``name``, ``schedule_display``. Storage is untouched; consumers never crash on formatting.rVunknownrnamerrrrcron jobN2rstate)rrrXrrreffective_job_state)rrrBrr label_sources r#_normalize_job_recordr s%S)J 01Ey QQFZ $4Z^^H5M$NNFZ ! JNN62 3 9 9 ;D   +5>>(+C 8$Q'  x 89??A   CR &&(6JJv%>z%JJ!".j9Jw r7ct|jdjdk(xst|jdS)z>True when the record carries any operator-facing pause signal.r paused paused_at)rrXrrr!rs r#_has_pause_markerrs8 CGGG, - 3 3 5 A _T#''R]J^E__r7cTt|jddxr t| S)zTrue iff the scheduler may fire this job: ``enabled`` plus pause markers as a second gate so a contradictory half-paused record never fires even before self-heal runs.enabledT)r!rXrrs r#is_job_runnabler s(  4( ) H2CC2H.HHr7ct|jdj}|dvr|S|jddst|s|dk(ry|xsdS|dk(s|jdry|xsdS)zOperator-facing state derived from ``enabled``: an enabled job must never display as paused (list looked frozen while jobs kept firing). Terminal states are preserved regardless.r >r completedrTrr scheduled)rrXrrr)rstoreds r#r r szcggg. / 5 5 7F '' 779d # S !Vx%7!! SWW[1  [ r7c(|jddvS)z=Return whether a job record is in a terminal scheduler state.r >rrrWrs r#is_terminal_jobrs 777 5 55r7cz|jddk(xr&|jdxsijddvS)uTrue for a recurring job stuck in ``state=error`` (set ONLY when ``compute_next_run()`` fails for a cron/interval job: croniter missing, malformed schedule). Such a job still has future occurrences once the issue resolves, so treating it as terminal would block due-scan self-heal, pre-advance, dispatch claim and ``resume_job`` — wedging it forever. Unlike ``state=completed`` (a one-shot that genuinely has no more occurrences, ever), an error-state recurring job still has a schedule with future occurrences once the underlying issue resolves — it is stuck pending a ``next_run_at`` recompute, not truly done. See #16265. r rrkind>r%intervalrWrs r#_is_recoverable_error_jobr$sB G# L WWZ &B + +F 37K Kr7pathc ddlm}||y)zOwner-only (0700) via the shared helper, so cron/ and cron/output honor the same managed/ container/HERMES_HOME_MODE rules as the rest of HERMES_HOME (#10757).r) _secure_dirN)hermes_cli.configr )r_shared_secure_dirs r#r r 4sDtr7c ddlm}||y)zJOwner-only (0600) via the shared helper (managed/container skip included).r) _secure_fileN)r!r$)r_shared_secure_files r#r$r$;s Er7beforec|tjdk7ryttdd}ttdd}||y |}|dk7s"|j|jf||fk(rytj ||j|jy#t $r7}tjd||j|j|Yd}~yd}~wwxYw)aRestore a rewritten file's previous owner (POSIX, root writer only): atomic replace makes the file owned by the writer's euid, so a root CLI write (e.g. ``docker exec``) against the unprivileged gateway's store would flip jobs.json to root:root 0600 and lock the ticker out.NposixgeteuidgetegidruCould not restore ownership of %s to uid=%s gid=%s after rewrite: %s — if the gateway runs as a different user, its cron ticker may now be locked out (see issue #68483).) osrrst_uidst_gidchownrrr)rr&r)r*euidrs r#_preserve_file_ownershipr0As~G+b)T*Gb)T*G'/ 3y 19 64:KK  v}}fmm4 3 0 &--  3 33s.B.+B C#-CCctjtt5d|j j vr dddy dddd|j vS#1swYxYw)zTrue if *path* is under ``/profiles//`` (default/custom homes are not). Checks the resolved path (symlinked parents) and the raw path (symlinked profile homes).profilesNT)rrr RuntimeErrorrepartsrs r#_is_named_profile_pathr6Xs^   Wl 3 -- - -  ##s AA&r0cft|r|jdy|jddy)aCreate a cron directory without resurrecting a deleted profile home: a stale multiplex scheduler may still hold a deleted profile's path, so named profiles use ``parents=False`` and fail closed. Default/custom homes keep ``parents=True`` so first-run creation works.T)exist_okN)parentsr8)r6mkdir)r0s r#_ensure_cron_dirr;as-h'% NN4$N/r7ct}t|jt|jt |jt |jy)z6Ensure cron directories exist with secure permissions.N)rjr;r0r2r )stores r#rrks>  !EU^^$U%%&  !r7repeatc|yt|tr4|jj}|dvry|dvry t |}|dkrdSt |S#t $rt d|dwxYw)aUCoerce a repeat value (int or user-facing string) into ``Optional[int]``: ``'forever'``-family -> None, ``'once'``-family -> 1, numeric -> int, 0/negative -> None, else ValueError. The tool schema exposes ``repeat`` as an integer, but agents and users legitimately pass the user-facing strings ``'forever'``/``'once'`` or numeric strings (``'3'``). Uncoerced strings previously died with ``'<=' not supported between instances of 'str' and 'int'`` at create (#66824/#64520/#7142/#71987/#95706) and were stored raw by update paths, breaking ``mark_job_run`` later. N)foreverinfiniteinfnoner)onceone1xrzInvalid repeat value z': use an integer, 'forever', or 'once'.r)rrJrrlowerintru)r> repeat_strs r#normalize_repeat_valuerJvs~&#\\^))+ C C . . _F Q;4/CK/  'z2()  s AA3ri)mhdsc"|jj}tjd|}|st d|d|j drt |j dnd}|t|j ddzS)u]Parse a duration into minutes: "30m" → 30, "2h" → 120, "1d" → 1440, bare "hour" → 60.zD^(\d*)\s*(m|min|mins|minute|minutes|h|hr|hrs|hour|hours|d|day|days)$zInvalid duration: 'zQ'. Use format like '30m', '2h', '1d', or a bare unit like 'hour' (defaults to 1).rr)rrrGrematchrugrouprH_DURATION_MULTIPLIERS)rNrRrs r#parse_durationrUs  A HH\^_ `E !!%: :; ;$);;q>C A qE (Q):; ;;r7sunday0sunmonday1montuesday2tuetues wednesday3wedwedsthursday4thuthurthursfriday5frisaturday6sat*z1-5z0,6)daydailyeverydayweekdayweekdaysweekendweekendsrc|jjjdd}|sy|dvry|dk(rytjd|}|syt |j d }t |j d xsd }|j d }|rd |cxkrd ksyy|d z|dk(rd nd z}|dkDs|dkDry||fS)zParse ``9am``/``9:30am``/``14:00``/``7`` (bare 24h hour)/``noon``/``midnight`` into a 24-hour ``(hour, minute)`` tuple, or None when unrecognized. rN)noonmidday) rmidnight)rrz ^(\d{1,2})(?::(\d{2}))?(am|pm)?$rrPrr{pm;)rrrGreplacerQrRrHrS)rtrRhourminutemeridiems r#_parse_clock_timers $$S"-A  J HH8! r7restc|jjddj}|sytj |d}d}|kg}t |}t |D]<\}}|dk(r tj |}||}n||vs,|j|>|sydj|}||d}|r |ddk(r|dd}|sytdj|} | y| \} } | d| d|S) zConvert `` [at]