߲j) UdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl m Z m Z mZmZddlmZddlmZddlmZddlmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z'ddl(m)Z)dd l*m+Z+ejXd Z-d Z.d Z/d Z0dZ1d Z2dZ3de4de5fdZ6da7ee8e9d<e8de$DZ:e8de$DZ;eefee fdZdZ?dje?jdge?jddDcgc]}d|jzc}zdgzZCdZDd ZEd!ZFd"ZGd#ZHd$ZId%d&d'e5d(e4d)eJde5fd*ZKd+e5de5fd,ZLde8fd-ZMGd.d/ZNd0ejddfd1ZPycc}w)2zSchema creation, column reconciliation, and FTS DDL management for SessionDB. Plain mixin for ``hermes_state.SessionDB`` (no ``__init__``/state of its own). Must never import hermes_state (cycle); shared constants live in hermes_state_common. N)DictListOptionalSequenceget_hermes_home)report_startup_progress)safe_json_loads)DEFERRED_INDEX_SQLFTS_CJK_STALE_KEYFTS_REBUILD_DEFERRAL_KEY FTS_STALE_KEYFTS_SQLFTS_STORAGE_VERSIONFTS_TOOL_CONTENT_PREFIX_CHARSFTS_TRIGRAM_SQLLEGACY_FTS_SQLLEGACY_FTS_TRIGRAM_SQL SCHEMA_SQLSCHEMA_VERSION_FTS_CJK_TRIGGERS _FTS_TRIGGERS_ephemeral_child_sql_sql_json_extractfts_rebuild_admission)_drop_orphan_fts_shadow_tables)_read_proc_argv hermes_stateN@ g @g @pidreturncHt|}|rdj|ddSdS)N xz)rjoin)r"argvs 8/home/ubuntu/.hermes/hermes-agent/hermes_state_schema.py_holder_cmdliner*/s) 3 D#'388D>$3 D-DD_READ_PROBE_STATEMENTSc#*K|] }d|vs| yw) _trigram_N.0ns r) r39sKA+:JaKs c#2K|]}|tvs |ywN)_FTS_TRIGRAM_TRIGGERSr0s r)r3r3:sVq@U7U1Vs )FTz[COALESCE(content, '') || ' ' || COALESCE(tool_name, '') || ' ' || COALESCE(tool_calls, '') )z]CREATE INDEX IF NOT EXISTS idx_session_model_usage_session ON session_model_usage(session_id)zVCREATE INDEX IF NOT EXISTS idx_session_model_usage_model ON session_model_usage(model)a_CREATE TABLE session_model_usage ( session_id TEXT NOT NULL REFERENCES sessions(id) ON DELETE CASCADE, model TEXT NOT NULL, billing_provider TEXT NOT NULL DEFAULT '', billing_base_url TEXT NOT NULL DEFAULT '', billing_mode TEXT NOT NULL DEFAULT '', task TEXT NOT NULL DEFAULT '', api_call_count INTEGER NOT NULL DEFAULT 0, input_tokens INTEGER NOT NULL DEFAULT 0, output_tokens INTEGER NOT NULL DEFAULT 0, cache_read_tokens INTEGER NOT NULL DEFAULT 0, cache_write_tokens INTEGER NOT NULL DEFAULT 0, reasoning_tokens INTEGER NOT NULL DEFAULT 0, estimated_cost_usd REAL NOT NULL DEFAULT 0, actual_cost_usd REAL NOT NULL DEFAULT 0, cost_status TEXT, cost_source TEXT, first_seen REAL, last_seen REAL, PRIMARY KEY (session_id, model, billing_provider, billing_base_url, billing_mode, task) ) z# z )aINSERT OR IGNORE INTO session_model_usage ( session_id, model, billing_provider, billing_base_url, billing_mode, api_call_count, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens, estimated_cost_usd, actual_cost_usd, cost_status, cost_source, first_seen, last_seen ) SELECT id, COALESCE(model, 'unknown'), COALESCE(billing_provider, ''), COALESCE(billing_base_url, ''), COALESCE(billing_mode, ''), COALESCE(api_call_count, 0), COALESCE(input_tokens, 0), COALESCE(output_tokens, 0), COALESCE(cache_read_tokens, 0), COALESCE(cache_write_tokens, 0), COALESCE(reasoning_tokens, 0), COALESCE(estimated_cost_usd, 0), COALESCE(actual_cost_usd, 0), cost_status, cost_source, started_at, COALESCE(ended_at, started_at) FROM sessions WHERE COALESCE(input_tokens, 0) + COALESCE(output_tokens, 0) + COALESCE(cache_read_tokens, 0) + COALESCE(cache_write_tokens, 0) + COALESCE(reasoning_tokens, 0) > 0zfCREATE UNIQUE INDEX IF NOT EXISTS idx_sessions_title_unique ON sessions(title) WHERE title IS NOT NULLziINSERT INTO state_meta (key, value) VALUES (?, '1') ON CONFLICT(key) DO UPDATE SET value = excluded.valuegINSERT INTO state_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.valuezVDELETE FROM state_meta WHERE key IN ('fts_rebuild_high_water', 'fts_rebuild_progress')zEDELETE FROM state_meta WHERE key = 'fts_tool_full_content_high_water'F delete_firsttableindentr<cbd|z}|r|d|dnd}d||d|d|d|d |d |d |d |d dS)zNLegacy inline (pre-v23) FTS re-population script fragment (whitespace pinned).r% DELETE FROM z; r7 INSERT INTO z(rowid, content) z SELECT id, z' COALESCE(content, '') || ' ' || z) COALESCE(tool_name, '') || ' ' || z COALESCE(tool_calls, '') zFROM messages; Nr/)r=r>r<padbodys r)_legacy_inline_reinsert_sqlrFsp ,C-9cU,ugS )rD TF3%|E7*< %| %7 %9 %0 %CRz  +r+identc2d|jddzdzS)zDouble-quote an SQL identifier."z"")replace)rGs r)_qrKs sD) )C //r+ctBtjt}t dt |j DatS)a SELECT statements that fail iff a live store is behind SCHEMA_SQL. Read-only opens skip ``_reconcile_columns()`` (no DDL against another profile's live DB), so healing callers run these afterwards: a missing table/column raises at prepare time. Derived from SCHEMA_SQL (a hand-maintained list went stale within days). Columns are table-qualified: an unqualified double-quoted identifier that fails to resolve silently degrades to a string literal (SQLite misfeature) and would pass on the stale store.c3K|]:\}djdjfd|Dt<yw)zSELECT {} FROM {} LIMIT 0, c3PK|]}tdt|yw).N)rK)r1colr=s r)r3z9schema_read_probe_statements...s%8bVYBuI;a3y9Q8bs#&N)formatr'rK)r1colsr=s @r)r3z/schema_read_probe_statements..sA' t ( . .tyy8b]a8b/bdfgldm n' sAA)r,SessionSchemaMixin_parse_schema_columnsrtuplesorteditems)tabless r)schema_read_probe_statementsrZsE%#99*E!&' %flln5' "  "!r+c6eZdZdZdej ddfdZdej defdZdej ddfdZ e dej de e defd Z e d ee defd Zdej defd Ze dej defd Ze dej de ddfdZe dej de ddfdZdej deddfdZe dej de defdZdej defdZdej ddfdZe ddddej dededdfdZdej de deefdZdej defdZdddej dedefd Zdefd!Zdej dedefd"Zdej defd#Ze d$e de e e e e fffd%Z!dej ddfd&Z"e dej d'e dee#e fd(Z$e d8dej d'e d)e de d*e ddf d+Z%e d9d,Z&dej ddfd-Z'dej ddfd.Z(d/Z)dej d0ed1eddfd2Z*dej ddfd3Z+dej ddfd4Z,dej ddfd5Z-d9d6Z.dej ddfd7Z/y):rTu>See module docstring — mixin for SessionDB (Schema cluster).cursorr#NcF |jdj}|D],\}} |j ||}|jd||f.y#tj$rYywxYw#tj$r!}t j d|Yd}~yd}~wwxYw)aUMove inline prompt snapshots into the shared content-addressed table. Any ``OperationalError`` mid-loop returns instead of raising: partial migration is safe (the legacy column stays a read fallback; next init resumes), whereas propagating left the version below 25 and re-ran this on every open (gateway crash loop).zFSELECT id, system_prompt FROM sessions WHERE system_prompt IS NOT NULLNzMUPDATE sessions SET system_prompt_hash = ?, system_prompt = NULL WHERE id = ?zv25 prompt dedupe paused after contention (%s); unmigrated rows keep the legacy inline prompt and the next schema init resumes the migration.)executefetchallsqlite3OperationalError_store_system_promptloggerwarning)selfr\rows session_idprompt prompt_hashexcs r)_dedupe_legacy_system_promptsz0SessionSchemaMixin._dedupe_legacy_system_promptss  >>"jkttvD#'  J "77G c *- ''   ++ t   s(A&A,A)(A),B ?BB c |jd|jdy#tj$r-}|j|s|j |Yd}~yd}~wwxYw)Nz:CREATE VIRTUAL TABLE temp._hermes_fts5_probe USING fts5(x)z"DROP TABLE temp._hermes_fts5_probeTF)r^r`ra_is_fts5_unavailable_error_warn_fts5_unavailable)rer\rjs r)_sqlite_supports_fts5z(SessionSchemaMixin._sqlite_supports_fts5s\  NNW X NN? @'' 2237  ' ' ,  s"%A%#A  A%c|j|tD]B}tjtj 5|j d|dddDy#1swYOxYw)NDROP TRIGGER IF EXISTS )_drop_fts_triggersr contextlibsuppressr`rar^)rer\triggers r)_drop_all_fts_triggersz)SessionSchemaMixin._drop_all_fts_triggerssh '( DG$$W%=%=> D!8 BC D D D D Ds AA' namesc|sydjd|D}d|d}t|j|t|j dt |kS)z;True unless every trigger in *names* (one DDL half) exists.F,c3 K|]}dyw?Nr/r1_s r)r3z;SessionSchemaMixin._fts_triggers_missing..s33 zGSELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger' AND name IN ()r)r'intr^rVfetchonelen)r\rw placeholderssqls r)_fts_triggers_missingz(SessionSchemaMixin._fts_triggers_missings`xx3U33 WXdWeefg6>>#uU|4==?BCc%jPPr+rcv|sydj|jj}d|vxrd|vS)z?True when trigger SQL is a broad AFTER UPDATE (missing ``OF``).Fr%zAFTER UPDATE OF zAFTER UPDATE ON )r'splitupper)rcompacts r)#_fts_update_trigger_needs_narrowingz6SessionSchemaMixin._fts_update_trigger_needs_narrowings=((399;'--/!0R5G75RRr+cJ|j|}d|rdndz}djd|D}d|d}|j||j}|Dcgc]\}}|j |s|}}}|sy|D]}|jd |t |\} } |j |d | |j |d | d |vrn |j||jddj} | r|j | dr&|j|tjdtjdt|t|Scc}}w#t$r(|j|tjd wxYw)aReplace broad AFTER UPDATE FTS triggers with AFTER UPDATE OF variants (``IF NOT EXISTS`` never replaces an existing broad trigger). No FTS rebuild: correctness was already gated by WHEN clauses; OF only skips trigger evaluation. Returns the number dropped.)messages_fts_updatemessages_fts_trigram_updater/)messages_fts_cjk_updaterNc3 K|]}dywr{r/r}s r)r3zHSessionSchemaMixin._migrate_broad_fts_update_triggers..s ; ;rzHSELECT name, sql FROM sqlite_master WHERE type = 'trigger' AND name IN (rrrq messages_ftsmessages_fts_trigramrz2CJK FTS re-ensure after UPDATE OF migration failedzASELECT sql FROM sqlite_master WHERE type = 'trigger' AND name = ?zeCJK FTS UPDATE trigger missing or still broad after UPDATE OF migration; marked stale and unavailablezPMigrated %d broad FTS UPDATE trigger(s) to AFTER UPDATE OF (no rebuild required))_db_has_legacy_inline_ftsr'r^r_r_FTS_DDL_ensure_fts_schema_ensure_fts_cjk_schema Exception)_quarantine_cjk_after_update_of_migrationrc exceptionrrdinfor) rer\ legacy_layout update_namesrrrfnameto_dropbase_sql trigram_sqlrows r)"_migrate_broad_fts_update_triggersz5SessionSchemaMixin._migrate_broad_fts_update_triggerss 66v> MB%A yy ;l ;; XYeXffgh~~c<099;)-_ID#1Y1YZ]1^4__ =D NN4TF; < =!) 7+ A (> L % / ++F3 ..SUqhj $BB3q6J>>vFH  fhklshtu7|;` >>vF  !UV sE+8E+E111F"cb|jdj}|duxr d|dxsdvS)uTrue when ``messages_fts`` is still external-content over the raw ``messages`` table (FTS_STORAGE_VERSION < 3): its index holds a TRUNCATED projection for long tool rows that the checker/'delete' commands re-read as FULL content, a mismatch by construction. Such an index cannot be repaired in place — it must be 'rebuild'-filled from the aligned ``messages_fts_src`` view exactly once.zLSELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'messages_fts'Nmessages_fts_srcrrAr^r)r\rs r)_fts_index_is_misaligned_sourcez2SessionSchemaMixin._fts_index_is_misaligned_sources>nn Z (* $K#5c!fl#KKr+ddlc~t|jdj}d}d}|jD]}||dzz }t j |s!d|j }}|j}|jdr|}W|jdrL|jdddjd }|j|jdd d |k(rd}||j|d}|j||j rt jd y) uRun *ddl* statement by statement, skipping each ``DROP TRIGGER IF EXISTS x`` / ``CREATE TRIGGER IF NOT EXISTS x …`` pair whose trigger already exists with the same body. ``DROP TRIGGER IF EXISTS`` on an existing trigger takes SQLite's write lock (the ``CREATE … IF NOT EXISTS`` forms do not), so the unconditional drop+recreate that lets a changed trigger body roll out would otherwise block every open of a settled database behind a sibling process's transaction. ``sqlite_master.sql`` stores the CREATE text verbatim minus ``IF NOT EXISTS`` and the ``;``, so an exact comparison decides. z:SELECT name, sql FROM sqlite_master WHERE type = 'trigger'NrAr7rqzCREATE TRIGGER IF NOT EXISTS zIF NOT EXISTS r8;rzincomplete DDL statement)dictr^r_ splitlinesr`complete_statementstripr startswithrJrstripgetrra) r\rstored pending_drop statementlinecurrentrdesireds r)&_execute_ddl_skipping_settled_triggersz9SessionSchemaMixin._execute_ddl_skipping_settled_triggers's.fnn%abkkmn&*  NN$ $D  $I--i8!#Y__%6wIMMOE 9:&  ?@!//*:BBII#N::gmmD!4Q78GC#'L'|,# NN7 ## $$ ?? **+EF F r+cd}|jD]3}||dzz }tj|s!|j|d}5|j rtj dy)zAExecute a DDL script without ``executescript``'s implicit commit.rAr7zincomplete FTS DDL statementN)rr`rr^rra)r\rrrs r)!_execute_ddl_script_transactionalz4SessionSchemaMixin._execute_ddl_script_transactionalJso NN$ D  $I)))4y)   ?? **+IJ J r+legacyc|sjdsyjsyjdjdudfd }s+jd |jdyj |y#t$r$jdjdwxYw) uRe-point ``messages_fts`` at the stable ``messages_fts_src`` projection view and rebuild it ONCE (FTS_STORAGE_VERSION 2 -> 3). A v1/v2 base index carries token streams the raw-``messages`` external-content source cannot read back (truncated long tool rows, and tool rows whose full content was indexed under an old high-water mark), so in-place continuity is not achievable — the ONLY valid transition is a full rebuild from the view, under the shared cross-process rebuild admission. Legacy inline DBs skip this entirely (their index is self-contained; they still take the DDL on the optimize path).rNzSELECT 1 FROM messages LIMIT 1cXtD]}jd|jdjdtrjdjtjt jt dttfy)Nrqz!DROP TABLE IF EXISTS messages_ftsrz8INSERT INTO messages_fts(messages_fts) VALUES('rebuild')fts_storage_version) _FTS_BASE_TRIGGERSr^rr_CLEAR_REBUILD_MARKERS_SQL!_DROP_RETIRED_TOOL_HIGH_WATER_SQL_STATE_META_UPSERT_SQLstrr)rr\ has_messagesres r)do_alignzCSessionSchemaMixin._migrate_misaligned_fts_source..do_alignes* A!8?@ A NN> ?  # #FNG DYZ NN5 6 NN< = NN14I3ObKc3d er+zSAVEPOINT fts_align_emptyz!RELEASE SAVEPOINT fts_align_emptyz%ROLLBACK TO SAVEPOINT fts_align_emptyr#N)_sqlite_table_existsrr^r BaseException_run_admitted_startup_rebuild)rer\rrrs`` @r)_migrate_misaligned_fts_sourcez1SessionSchemaMixin._migrate_misaligned_fts_sourceVs 226>J 33F; ~~&FGPPRZ^^  f NN6 7  BC  **68< ! FGBC s (B-CrcH|jd|fjduS)Nz=SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?r)r\rs r)rz'SessionSchemaMixin._sqlite_table_exists}s(~~ KdW (*D! !r+c:|js|jry|jd}|dur|duStD]}j d|j d|j dt sy|jfdy)aInstall the source-filtered trigram view and purge historical rows (v29 cron exclusion, v30 subagent exclusion: both only change the view/trigger predicate and rebuild from it). Legacy inline indexes stay opt-in (their content is private to the vtable). A v1 external layout whose vtable still declares ``tool_calls`` is left to ``optimize-storage``: swapping the view underneath would make 'rebuild' read a column the view no longer has. Otherwise the inverted index still holds excluded rows until FTS5 rebuilds from the new view, which runs under the shared cross-process admission gate. Returns False to hold schema_version back.TrFrqz,DROP VIEW IF EXISTS messages_fts_trigram_srcc&jdS)NzHINSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild')r^r\sr)zDSessionSchemaMixin._migrate_trigram_cron_exclusion..sFNN#mnr+)r%_db_has_trigram_tool_calls_projection_fts_table_prober6r^rrr)rer\trigram_existsrs ` r)_migrate_trigram_cron_exclusionz2SessionSchemaMixin._migrate_trigram_cron_exclusions  ) )& 1T5_5_`f5g..v7MN  %"U* *) =D NN4TF; < =EF&&v/EW **  n r+cd|_ |jtd| |j dy #t$rtj ddY5wxYw#t$rtj ddYy wxYw) zFail closed after dropping the CJK UPDATE trigger mid-migration: clear availability, persist ``fts_cjk_stale``, drop any residual trigger so a later open cannot IF-NOT-EXISTS over a gap.F1rz*Could not persist CJK FTS stale breadcrumbTexc_infoz.DROP TRIGGER IF EXISTS messages_fts_cjk_updatez;Could not drop residual CJK UPDATE trigger after quarantineN)_fts_cjk_availableset_metar rrcdebugr^rer\s r)rzM9Rc NCcU34cU2NOhNiivwxcU!C58KLM  N  NN5 6r+ table_namec |jd|dy#t$r }|}Yd}~nd}~wtj$r}|j |r9|j |r|j |n|j|Yd}~ydt|jvrYd}~ydt|jvr|}Yd}~nd}~wwxYwtjd||y) aTrue = queryable, False = absent, None = FTS module/tokenizer missing or content undecodable (index degraded, store accessible). Invalid UTF-8 surfaces as a bare UnicodeDecodeError on some builds and OperationalError("Could not decode to UTF-8") on others; both are caught so the probe never raises into init/recovery flows. Anything else (malformed schema, corrupt vtable) re-raises.zSELECT * FROM z LIMIT 0TNz no such tableFzdecode to utf-8zp%s probe encountered invalid UTF-8 in FTS content; search may return incomplete results until FTS is rebuilt: %s) r^UnicodeDecodeErrorr`rarm_is_trigram_unavailable_error_warn_trigram_unavailablernrlowerrcrd)rer\rrj decode_excs r)rz#SessionSchemaMixin._fts_table_probes   NN^JK|]}d|dt|yw)zpid z: N)r*)r1r"s r)r3zBSessionSchemaMixin._defer_stale_fts_for_holders..s#V3DR(<'=>Vszstate.db FTS repair remains blocked after %d deferrals by holder(s) %s. Stop the listed processes (this process's own retry then rebuilds), or run `hermes sessions optimize-storage` with every holder stopped. `hermes doctor` reports this degraded state.zDeferred stale state.db FTS rebuild while foreign processes hold the database or WAL sidecars (%s); canonical writes and LIKE search remain available (deferral %d).)timer^r rr`Errorr isinstancerfloatrr TypeError ValueErrorrW_FTS_HOLDER_ESCALATE_ATTEMPTS_FTS_HOLDER_ESCALATE_SECONDS%_reap_inactive_orphan_desktop_holdersrcerror_foreign_state_db_holdersbool_FTS_HOLDER_FUTILE_ATTEMPTS_FTS_HOLDER_FUTILE_SECONDSjsondumps_fts_deferred_holder_pidsr'_FTS_STALE_RETRY_SECONDSrd)rer\foreign_holdersnowrparsedrecordrrrrr"_pathrreapedr diagnostics r)_defer_stale_fts_for_holdersz/SessionSchemaMixin._defer_stale_fts_for_holderssiik ..DG_Fahj  -0Q(T%fd3 Svzz,<=J6::j!459H!&**_c"BCM"6::.@!#DEI   zA~JOOjc5sQwcOP &**]3 3.11+M 4 4z9IMi9i??1M@F NOUW_#'"@"@"B$O%Wjc5sUVwc%WX k"  ; ; qm@SWq@q %3H]h*@P\b   u %tzz*'M N -1D *)4&  LLo!3#6$"> V+VVXp  6 63;KOk;k LLZ[cet   w X {}} C :& SDGCQRN AJ-1A SP&Xs<%J"A2J* J> 8J> ; K K JJJ;:J;timeout_secondsc|j}|r|j||ryt|j|5}|stj d dddy|j ||cdddS#1swYyxYw)aAtomically rebuild stale base/trigram indexes and resume syncing. *timeout_seconds* bounds the admission wait (None = full startup budget, ``0`` = non-blocking retry). Fails closed: holders or a lost admission race leave the breadcrumb set.FrzDeferred stale state.db FTS rebuild: another process holds the rebuild authority; canonical writes and LIKE search remain available.Nr)rrrdb_pathrcrd_recover_stale_fts_locked)rer\rrradmitteds r)_recover_stale_ftsz%SessionSchemaMixin._recover_stale_fts,s88: t@@Y "4<< Q IU]I  I I11&1H I I IsA;A;;BcL|jsy|jd|_d|_y|js |j yt j}t|dd}|t|ddkr?|5t|jDchc] \}}|dkDs |c}}|k(ryd|_tt|dd}|dkrt}||z|_tt|tdd zt|_ |j 5|j |js dddy|j j#}|j%|}|j'||d }|r|j)|d|_t+j,t.j05|j j3ddd|cdddScc}}w#1swYxYw#1swYyxYw#t4$rt6j9d d YywxYw)uRetry a deferred stale-FTS rebuild (gateway housekeeping tick). ``_recover_stale_fts`` fails closed at open, leaving search on LIKE; live write/search paths must never start a full rebuild, and a gateway opens state.db once for days, so "next open" never comes. Bounded doubling backoff, non-blocking admission, no new thread. True only when the index was rebuilt and sync triggers restored. Never raises. This is the in-process retry: bounded backoff from ``_FTS_STALE_RETRY_SECONDS`` doubling to ``_FTS_STALE_RETRY_MAX_SECONDS``, non-blocking admission (``timeout=0``) so a live holder is skipped and tried again later, no new thread — the caller is an existing periodic tick (gateway housekeeping). See #100108, #97940. FNgr_fts_stale_retry_afterr_fts_stale_retry_intervalg?g@)rrzUIn-process retry of the deferred stale state.db FTS rebuild failed; will retry later.Tr) _fts_stale_quarantine_reasonrr read_only_connr monotonicgetattrrWrrrminmax_FTS_STALE_RETRY_MAX_SECONDS_lockr\rrrrsrtr`rcommitrrcrd) rer  deferred_pidsr"r intervalr\r recovereds r)retry_deferred_fts_recoveryz.SessionSchemaMixin.retry_deferred_fts_recovery<s   " " $ 0+.D '-0D * >>TZZ/nn&A4H 7= =$'+'E'E'GSe3QR7S))-0D *'BCHI s?/H&)Hn#), 2C 83 >@\* &  !::%T__  ! !**,77? 33F6[^3_ //758D2((7(JJ%%'(  ! !T.(( ! !  NNgrv    sa G" G" HG4/H8A.s]G4WIQ?]sz*DROP TABLE IF EXISTS messages_fts_trigram;zODROP VIEW IF EXISTS messages_fts_trigram_src;DROP TABLE IF EXISTS messages_fts;rr;z9INSERT INTO messages_fts(messages_fts) VALUES('rebuild');zIINSERT INTO messages_fts_trigram(messages_fts_trigram) VALUES('rebuild');rzBEGIN IMMEDIATE;z&DELETE FROM state_meta WHERE key IN ('z', 'z ');COMMIT;NzfAutomatic rebuild of stale FTS indexes failed (%s); canonical writes remain enabled with FTS detached.FzVRebuilt stale state.db FTS indexes from canonical messages and restored sync triggers.)r_trigram_tokenizer_availabler` DatabaseErrorrr'rrrrFrrrrrr executescriptrsrtrrrollbackrvr%rcrr _fts_enabled_trigram_availablerd) rer\rtrigram_presentrdrop_sql rebuild_sql recovery_sqlrjs r)rz,SessionSchemaMixin._recover_stale_fts_lockedys l"33F__bee eK  )K 76}oTJbIccmn o     .  "1opU%%'9: #"O #6$$ $$W]]3 & ##% & & &  ' ' / JJ    LLEFI  sBDD:D76D7:G+ #G&0F G&F AG&&G+cfd}|jd| |jd|d |jd|y#tj$rA}|j|s|j |Yd}~|jd|yd}~wwxYw#|jd|wxYw)zAProbe trigram support without publishing a persistent FTS object.ztemp.hermes_fts5_trigram_probezDROP TABLE IF EXISTS zCREATE VIRTUAL TABLE z( USING fts5(content, tokenize='trigram')NFT)r^r`rarr)rer\proberjs r)r.z/SessionSchemaMixin._trigram_tokenizer_availables0.ug67 < NN25'9ab c NN25': ;'' 55c:  * *3 / NN25': ;   NN25': ;s)AB#B9BBBB0 schema_sqlcd}tj|jdj}t j t 5ddlm}|dz dz }tj|jd}t|tr%|jd|k(r|jd nd}t|tr+td |j!Dr |cdddSdddt#j$d } |j'|i}|j)d j+D]\}i} |j)d |dj+} | D]O\} } } }}}| r| gng}|r|s|j-d||j-d|dj/|| | <Q| ||< |j1|t j t 5|j2j5ddt7j8t;|j2d\}}t=j>|dd5}tj@||d|dddt=jB||ddd|S|S#1swYxYw#|j1wxYw#1swYMxYw#1swY|SxYw)u[Expected columns per table: execute SCHEMA_SQL in an in-memory database and read PRAGMA table_info (no regex). Memoized on disk keyed by a DDL hash (~85ms per startup otherwise); only the reference-side parse is cached — diffing the LIVE database still runs every startup. A corrupt/stale cache degrades to recomputation.Nutf-8rrcachezschema_columns.jsonencoding schema_hashrYc3K|]6}t|txr td|jD8yw)c3<K|]}t|tywr5)rr)r1vs r)r3zESessionSchemaMixin._parse_schema_columns...s.Yaz!S/A.YsN)rrallvalues)r1rSs r)r3z;SessionSchemaMixin._parse_schema_columns..s50^b 4&Y3.Y4;;=.Y+YY0s<>z:memory:zNSELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%'PRAGMA table_info("")zNOT NULLzDEFAULT r%T)parentsexist_okz.schema_columns.)dirprefixw)r@rY)"hashlibsha256encode hexdigestrsrtrhermes_constantsrrloads read_textrrrrDrEr`connectr0r^r_appendr'closeparentmkdirtempfilemkstemprosfdopendumprJ)r: cache_pathr@_homeblobrYref table_columnsrrSr_cidcol_namecol_typenotnulldefaultpkpartsfdtmpfhs r)rUz(SessionSchemaMixin._parse_schema_columnss  nnZ%6%6w%?@JJL   +  A7*-BBJ::j22G2DED+5dD+Adhh}F]alFlTXXh'rvF&$'C0flfsfsfu0-   ooj)    j )79M++`hj *(*{{%8R#@AJJLFJ5BD(Hgw*2XJEr Z0* xy%9:%(XXe_DN5&* c" * IIK  !$$Y/ ,!!''t'D"**s:3D3D/EN`aCYYr39YRIIk]SUWXY 3 +  , }O  < IIK YY , s?B!J!B7J.:A#KK7K!J+.KK KKc ~|jt}|jD]|\}} |jd|dj }|Dchc]}|d }}|jD].\}} ||vr |jd|dt|d| 0~y #t j $rYwxYwcc}w#t j $re} t| j} d| vrtjd||| Yd } ~ d | vsd | vrtjd ||| Yd } ~ d } ~ wwxYw) zADD every SCHEMA_SQL column missing from the live tables (SCHEMA_SQL is the single source of truth; column additions need no version-gated migration).rFrGr8z ALTER TABLE "z " ADD COLUMN r%zduplicate columnzreconcile %s.%s: %sNlockedbusyz;reconcile %s.%s failed; store remains behind SCHEMA_SQL: %s) rUrrXr^r_r`rarKrrrcrrd) rer\expectedr declared_colsrfr live_colsrdrerjmessages r)_reconcile_columnsz%SessionSchemaMixin._reconcile_columnssW--j9)1)9  %J  ~~(;JYZ[cZd#ef   ++  1 // !#hnn.G)W4 %:JRUV 7*f.?NNUWackmp s5#B& B?>#C&B<;B<D<5D7!D77D<r=c |jd|dj}|syt d|DdDcgc]}|d c}S#tj$rd}Y@wxYwcc}w)zPRIMARY KEY column names of *table* in key order; None when the table is missing or has no columns (SCHEMA_SQL creates it correctly).rFrGNc3,K|] }|ds |yw)Nr/)r1rs r)r3z6SessionSchemaMixin._live_pk_columns..s%>A1a%>s c |dS)Nrwr/)rxs r)rz5SessionSchemaMixin._live_pk_columns..s aPQdr+)keyr8)r^r_r`rarW)r\r=rfrxs r)_live_pk_columnsz#SessionSchemaMixin._live_pk_columns ss >>$7wb"ABKKMD$%>%>NST!TT '' D  Us#A A)A&%A& legacy_namecopy_sqlc"|j}|jrtj||||||y|j d tj|||||||j dy#t $r|j dwxYw)uRENAME *table* to *legacy_name*, CREATE it fresh from *ddl*, copy rows back with *copy_sql*, DROP the legacy copy, recreate *indexes* — as ONE write transaction. The writer connection is autocommit (``isolation_level=None``), so without an explicit BEGIN each statement commits on its own and a sibling process opening the same state.db between RENAME and CREATE runs SCHEMA_SQL's ``CREATE TABLE IF NOT EXISTS`` first: our CREATE then fails with "table already exists", every row is stranded in *legacy_name* and the live table is empty. BEGIN IMMEDIATE holds the write lock for the whole rebuild so the sibling blocks (and retries under its lock patience) instead of interleaving; any failure rolls the RENAME back.NzBEGIN IMMEDIATEROLLBACKCOMMIT) connectionin_transactionrT_rebuild_table_statementsr^r)r\r=r|rr}indexesconns r)_rebuild_tablez!SessionSchemaMixin._rebuild_tables       8 8 UXZbdk l ()   8 8 UXZbdk l x   NN: &  s A22Bc|jd|d||j||j||jd||D]}|j|y)Nz ALTER TABLE z RENAME TO z DROP TABLE r)r\r=r|rr}rrs r)rz,SessionSchemaMixin._rebuild_table_statements0saeWK }EFsx 23 C NN3  r+c|j|d}||ddgk(rytjd||j|ddddy) aRebuild ``gateway_routing`` when its PRIMARY KEY predates scoping (``session_key TEXT PRIMARY KEY``): the reconciler ADDs ``scope`` but SQLite cannot ALTER a PK, so every routing write fails (ON CONFLICT mismatch / cross-scope UNIQUE violation). Newest row wins a cross-scope session_key collision (INSERT OR REPLACE in updated_at order). Early builds of the routing-index migration (#59203) created the table with ``session_key TEXT PRIMARY KEY`` and no ``scope`` column. ``_reconcile_columns()`` ADDs the missing ``scope`` column on those databases, but SQLite cannot ALTER a primary key, so the shipped composite ``PRIMARY KEY (scope, session_key)`` never lands. On such tables every write path is broken: gateway_routingNscope session_keyz]gateway_routing has legacy primary key %r; rebuilding with composite (scope, session_key) keygateway_routing_legacy_pkzCREATE TABLE gateway_routing ( scope TEXT NOT NULL DEFAULT '', session_key TEXT NOT NULL, entry_json TEXT NOT NULL, updated_at REAL NOT NULL, PRIMARY KEY (scope, session_key) )zINSERT OR REPLACE INTO gateway_routing (scope, session_key, entry_json, updated_at) SELECT COALESCE(scope, ''), session_key, entry_json, updated_at FROM gateway_routing_legacy_pk ORDER BY updated_at ASC)r{rcrr)rer\pk_colss r)_heal_gateway_routing_pkz+SessionSchemaMixin._heal_gateway_routing_pk9sa''0AB ?g'=)AA  kmt   %'B  E r+c|j|d}|d|vrytjdt||j d |j |ddt dt|j d y#tj$r }tjd|Yd}~@d}~wwxYw#|j d wxYw) uRebuild ``session_model_usage`` when its PRIMARY KEY lacks ``task``: installs already at v22+ when ``task`` landed carry the 5-column PK, the reconciler ADDs ``task`` but SQLite cannot ALTER a PK, and the v22 rebuild is unreachable — every upsert then fails (ON CONFLICT mismatch), silently zeroing accounting. Idempotent. FK-off window: INSERT OR IGNORE does NOT suppress FK violations, so an orphaned usage row would abort the rebuild (PRAGMA foreign_keys is a no-op inside a transaction; none is open here). OR IGNORE: COALESCE(task, '') on legacy NULL rows can collide with a genuine ''-task row — keep the first. Installs whose ``state.db`` reached ``schema_version >= 22`` before the ``task`` dimension was added carry a 5-column PRIMARY KEY ``(session_id, model, billing_provider, billing_base_url, billing_mode)``. See #73823. session_model_usageNtaskzdsession_model_usage has legacy primary key %r (missing task); rebuilding with composite 6-column keyzPRAGMA foreign_keys=OFFsession_model_usage_legacy_pkaINSERT OR IGNORE INTO session_model_usage ( session_id, model, billing_provider, billing_base_url, billing_mode, task, api_call_count, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens, estimated_cost_usd, actual_cost_usd, cost_status, cost_source, first_seen, last_seen ) SELECT session_id, model, COALESCE(billing_provider, ''), COALESCE(billing_base_url, ''), COALESCE(billing_mode, ''), COALESCE(task, ''), api_call_count, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens, estimated_cost_usd, actual_cost_usd, cost_status, cost_source, first_seen, last_seen FROM session_model_usage_legacy_pkz'session_model_usage PK heal skipped: %szPRAGMA foreign_keys=ON) r{rcrrWr^r_SESSION_MODEL_USAGE_HEAL_DDL_SESSION_MODEL_USAGE_INDEX_SQLr`rar)rer\rrjs r)_heal_session_model_usage_pkz/SessionSchemaMixin._heal_session_model_usage_pkXs''0EF ?f/  r 7O  01 5   -/NPm9"/5 > NN3 4'' I LLBC H H I NN3 4s* A;;B.B)$B1)B..B11CcZtdd|jj}|jt|j ||j ||j| |jd|j|ttj tj5|jdj#|jdddd|j%|}|jd t&fj#}|du|_|j(r|j+||s|j-||jd j#}||jd t.ft0j0j3t0j4j6j9}|j;d d t=t?j@fd|fgn|jC||d||jE||r|jG||jjIy#tj$r!}tjd|Yd}~d}~wwxYw#1swYxYw)zCreate tables and FTS if missing, reconcile columns, run data migrations. Column additions are declarative via _reconcile_columns(), so reordered migrations can never skip a column; schema_version remains for data migrations only.@state_db_init_schemaphasezCREATE INDEX IF NOT EXISTS idx_messages_platform_msg_id ON messages(session_id, platform_message_id) WHERE platform_message_id IS NOT NULLz/idx_messages_platform_msg_id create skipped: %sNz3SELECT 1 FROM messages WHERE active IS NULL LIMIT 1z3UPDATE messages SET active = 1 WHERE active IS NULLz.SELECT 1 FROM state_meta WHERE key = ? LIMIT 1z*SELECT version FROM schema_version LIMIT 1z/INSERT INTO schema_version (version) VALUES (?)z;INSERT OR IGNORE INTO state_meta (key, value) VALUES (?, ?)store_instance_idstore_created_at_utcr)%r rr\r0rrtrrr^r`rarcrrr rsrtrrorrrvrrrdatetimer timezoneutc isoformat executemanyruuiduuid4_run_data_migrations_ensure_unique_title_index _init_ftsr%)rer\rjfts5_available stale_rowrnow_isos r) _init_schemazSessionSchemaMixin._init_schemas? -CD""$Z( ' %%f- ))&1 Q NNe  33F -^=OPQR/ ;< >,^=OPQR,^=OPQRPP  $ R  OBB6J R $$W%=%=> B@A B R   1 1& 9 R Nt7Y7YZ`7a MM2CM G R   . .v 6"& R N'+&J&J6&R #. 66v>Whj!''/@@H~~b()+hj! 3S9L5MV\ ] ^ +0GN NNB^DU VM[0G +s  0 O H#NN O B Bs0AH/H<I)/H9< I&I!!I&)I3c |jdjd}|ry|j|ddtdty#t j $r }tjd|Yd}~yd}~wwxYw)uv22: ``task`` joins the session_model_usage PRIMARY KEY ('' = main loop; aux calls named). SQLite cannot ALTER a PK, so rebuild; existing rows → task=''.z\SELECT COUNT(*) FROM pragma_table_info('session_model_usage') WHERE name = 'task' AND pk > 0rNrsession_model_usage_v21aINSERT INTO session_model_usage ( session_id, model, billing_provider, billing_base_url, billing_mode, task, api_call_count, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens, estimated_cost_usd, actual_cost_usd, cost_status, cost_source, first_seen, last_seen ) SELECT session_id, model, billing_provider, billing_base_url, billing_mode, '', api_call_count, input_tokens, output_tokens, cache_read_tokens, cache_write_tokens, reasoning_tokens, estimated_cost_usd, actual_cost_usd, cost_status, cost_source, first_seen, last_seen FROM session_model_usage_v21z+v22 session_model_usage rebuild skipped: %s) r^rr_SESSION_MODEL_USAGE_V22_DDLrr`rarcr)rer\ legacy_pkrjs r)rz3SessionSchemaMixin._migrate_v22_session_model_usageOs M nhjI   -/HJf ?/ "'' M LLF L L Ms$AAA9A44A9cv |jty#tj$ry |jdtj d|j |jtYy#tj$rtjdYYywxYwtj$rYywxYw)zUnique title index. Older DBs may hold duplicate aliases from before the constraint; the newest keeps the alias. Must never abort opening the DB, so the repair is guarded.agUPDATE sessions AS older SET title = NULL WHERE title IS NOT NULL AND EXISTS ( SELECT 1 FROM sessions AS newer WHERE newer.title = older.title AND newer.rowid > older.rowid )zFCleared %d duplicate session title(s) while restoring the unique indexzICould not repair duplicate session titles; unique title index not createdN) r^_TITLE_UNIQUE_INDEX_SQLr`IntegrityErrorrcrdrowcountrrrars r)rz-SessionSchemaMixin._ensure_unique_title_indexps  NN2 3%% n n \^d^m^m67== n  !lm n''   s-B8AA44(B B8B  B87B8c jtd}jsjjr;j rj ndx_x__nt\ jtxsd|v}jtxsd|v}d  fd }|rj|nZjd_j r6jd _r|rjfdj rs|sj j rjyy) a\Create/repair the FTS objects on an FTS5-capable runtime. The DDL runs even when the vtable exists so CREATE TRIGGER IF NOT EXISTS repairs trigger-only degradation. OPT-IN v23 boundary: a legacy v22 inline install keeps its inline schema + triggers (the v23 DDL would create the trigram source VIEW and leave a mixed state).)rrmessages_fts_cjkrFrrNcjd_jsyjd_jjsj yy)Nrrr)rr2r3rr)rr\ legacy_ftsrersr)ensure_and_rebuildz8SessionSchemaMixin._init_fts..ensure_and_rebuildsx$($;$;FNT\$]!((*.*A*A&J`bm*n')):t?V?V*"//7"r+c,jS)Nr)r)r\rretrigram_enabledsr)rz.SessionSchemaMixin._init_fts..sD$=$= &z?%>%r+r)rrrrrrr2r3rrrrr6rrr) rer\orphan_repairedbase_triggers_missingtrigram_triggers_missingrrrrrs `` @@@@r)rzSessionSchemaMixin._init_ftss 33F; 9 P   / /z / J ??&&vj&A++F3X]]!]D$;d>U$,Z$8 !Hk**63EFk.\kJk "**63HIvMcgvMv % 8 8%226;MN$($;$;FNT\$]!$$&*&=&=fF\^i&jO.=D+&+C::"   ? ##F+PUUUD3d6M  s BBc@tdz dz }|jsyt|dd5}tj|}dddt t sy|jD]\}}t|jdst |t s2|jd}|sF|jd }t |t r|nd} |jd |jd xs|| | jd nd|jd | | jdnd|jd| tj|nd|jds|jdrdndt|f"y#1swYOxYw)uOne-time v18 backfill of gateway metadata from sessions.json. Only fills NULL columns — never overwrites data written by newer code.rz sessions.jsonNrxr<r>r~rgoriginaiUPDATE sessions SET session_key = COALESCE(session_key, ?), chat_id = COALESCE(chat_id, ?), chat_type = COALESCE(chat_type, ?), thread_id = COALESCE(thread_id, ?), display_name = COALESCE(display_name, ?), origin_json = COALESCE(origin_json, ?), expiry_finalized = CASE WHEN COALESCE(expiry_finalized, 0) = 0 AND ? = 1 THEN 1 ELSE expiry_finalized END WHERE id = ?rchat_id chat_type thread_id display_nameexpiry_finalizedmemory_flushedr8r) rexistsopenrloadrrrXrrrr^r) rer\ sessions_filefdatarzentryrgr origin_dicts r)rz@SessionSchemaMixin._backfill_gateway_metadata_from_sessions_jsonsm()J6H ##%  -w 7 199QBL7>>LW[L\[_IIDIcgI ;T;z//4/TX/b 7>> d ".#.$sDcN7J2K..`D> U U Uc@S U U!w~~!c!!RU!ad!uy!!.   w~~ $ >657>>65d65tRhgW7>>gWCgWaegWjngWRMw~~M$MBD0A<A<4A>, -DDKKM*r+)QrrsrrMloggingrr[r`rYrrtypingrrrrrQrhermes_startup_watchdogr utilsr hermes_state_commonr r r rrrrrrrrrrrrrrhermes_state_ftsrhermes_state_holdersr getLoggerrcrrrrrr#rrr*r,rV__annotations__r6rrrrrr'rrrrrrrrrrrFrKrZrT Connectionr)lns0r)rs$   11-;! <0   > * !#!# %EEE +/. KKKVmVV_-nF\5] ^a"!, $yy"--/23'D'O'O'QRSTV'WXx"((*XY  %H!<mpnv%l!RW s C $ [^ 0c0c0 "e""WWt"+!3!3++]%Ys E9