߲j>T dZddlZddlZddlZddlmZddlmZddlm Z ddl m Z m Z m Z mZmZddlmZej$dZd Zd Zd efd Zd efd Zdej2d efdZdZdej8deed eefdZGddZ y)zFTS5 index setup for SessionDB: the CJK-bigram (cjk_unicode61) index DDL and tokenizer loader, per-table schema ensure with tokenizer-capability fallback, FTS-scoped corruption detection and the atomic fail-open trigger detach.N)Path)Sequence)get_hermes_home)FTS_CJK_STALE_KEY FTS_STALE_KEY_FTS_CJK_TRIGGERS _FTS_TRIGGERSrouted_sessions_settingis_fts_scoped_corruption_error hermes_statea\ CREATE VIEW IF NOT EXISTS messages_fts_cjk_src AS SELECT id, role, content, tool_name, tool_calls FROM messages WHERE role <> 'tool'; CREATE VIRTUAL TABLE IF NOT EXISTS messages_fts_cjk USING fts5( content, tool_name, tool_calls, content='messages_fts_cjk_src', content_rowid='id', tokenize='cjk_unicode61' ); a  CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_insert AFTER INSERT ON messages WHEN new.role <> 'tool' AND (new.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_high_water'), -1) OR new.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_progress'), -1)) BEGIN INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) VALUES (new.id, new.content, new.tool_name, new.tool_calls); END; CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_delete AFTER DELETE ON messages WHEN old.role <> 'tool' AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_high_water'), -1) OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_progress'), -1)) BEGIN INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls) VALUES ('delete', old.id, old.content, old.tool_name, old.tool_calls); END; CREATE TRIGGER IF NOT EXISTS messages_fts_cjk_update AFTER UPDATE OF content, tool_name, tool_calls, role ON messages WHEN (old.content IS NOT new.content OR old.tool_name IS NOT new.tool_name OR old.tool_calls IS NOT new.tool_calls OR old.role IS NOT new.role) AND (old.id > COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_high_water'), -1) OR old.id <= COALESCE((SELECT CAST(value AS INTEGER) FROM state_meta WHERE key = 'fts_cjk_rebuild_progress'), -1)) BEGIN INSERT INTO messages_fts_cjk(messages_fts_cjk, rowid, content, tool_name, tool_calls) SELECT 'delete', old.id, old.content, old.tool_name, old.tool_calls WHERE old.role <> 'tool'; INSERT INTO messages_fts_cjk(rowid, content, tool_name, tool_calls) SELECT new.id, new.content, new.tool_name, new.tool_calls WHERE new.role <> 'tool'; END; returnctjd}|rt|jSt dz dz S)z1Location of the cjk_unicode61 loadable extension.HERMES_FTS5_CJK_SOlibzlibfts5_cjk.so)osgetenvr expanduserr)envs 5/home/ubuntu/.hermes/hermes-agent/hermes_state_fts.pyfts5_cjk_so_pathrcs9 ))( )C%(49   !Zo.?%.GJZ.ZZcxtdd}|duxs)t|jjdvS)zKconfig.yaml ``sessions.cjk_fts`` (default on) for the profile being served.cjk_ftsHERMES_CJK_FTSN)0falseoffno)r strstriplower)values r_cjk_fts_config_enabledr$is: #I/? @E D= YCJ,,.446>YYYrconncDt}tr|jsy |jd |j t ||jdy#|jdwxYw#t $rtjd|dYywxYw)zBest-effort load of the cjk_unicode61 tokenizer; False (never raises) when the .so is absent, ``sessions.cjk_fts`` is off, or loading is compiled out.FTz#fts5_cjk extension load failed (%s)exc_info) rr$existsenable_load_extensionload_extensionr Exceptionloggerwarning)r%paths rload_fts5_cjk_extensionr0os  D " $DKKM  ""4( .   D *  & &u -  & &u - z1_drop_orphan_fts_shadow_tables..sXnabY\Xn )rz DROP TABLE ""zRDropped orphan FTS5 shadow tables of %s (%s); the index is recreated from messagesz, )executefetchone_FTS5_SHADOW_SUFFIXESjoinfetchallr-r.append) r6r7repairedfamily vtable_livesuffixshadowsroworphansnames r_drop_orphan_fts_shadow_tablesrSsH nn 3 I  (*   6KLFfXQvh'LL%+^^OPSPXPXXnfmXnPnOoop q &  (*c3q6  3D NN\$q1 2 3 ` DIIg&  + , OMs C( C-ceZdZdZedej defdZedej defdZ edejdefdZ edejdefdZ e dejdefd Zdej dd fd Zdej dd fd Zdd Zedejdd fdZdejdededefdZedej*defdZdej*defdZdZdZdZdZdefdZdZy )SessionFtsSetupMixinzOFTS table/trigger lifecycle shared by schema init, optimize and the write path.excrcvt|j}d|vxrd|vxstj|S)zONo FTS5 module, or an optional tokenizer missing (same capability-error shape).zno such modulefts5)r r"rU_is_trigram_unavailable_errorrVerrs r_is_fts5_unavailable_errorz/SessionFtsSetupMixin._is_fts5_unavailable_errors:#hnn C'9FcMu>R>p>pqt>uurcHt|j}d|vxsd|vS)zOnly an optional tokenizer is missing (trigram needs SQLite >= 3.34; cjk_unicode61 is loadable): "this one index can't be served", never "disable FTS".zno such tokenizer: trigramz no such tokenizer: cjk_unicode61)r r"rZs rrYz2SessionFtsSetupMixin._is_trigram_unavailable_errors+#hnn+s2_6X\_6__rr6cb|jdj}|duxr d|dxsdvS)umessages_fts exists in ANY pre-v23 shape: every legacy shape lacks tool_name, so "stored CREATE lacks tool_name" catches them all. False when absent. v23's messages_fts is external-content over THREE real columns (content, tool_name, tool_calls). Every pre-v23 shape lacks the tool_name/tool_calls columns — whether the old inline single-column form (v11..v22) or the even older external-content single-column form (v10-era, pre-#16751). We therefore detect "needs optimize" as "the stored CREATE lacks the tool_name column", which is the precise v23 marker and correctly catches BOTH legacy variants. zLSELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'messages_fts'N tool_namer)rErFr6rPs r_db_has_legacy_inline_ftsz.SessionFtsSetupMixin._db_has_legacy_inline_ftss=nn Z (* $D;s1v|#DDrc~|jdj}|duxrd|dxsdjvS)z[True when the trigram vtable still includes the tool_calls payload (FTS_STORAGE_VERSION 1).zTSELECT sql FROM sqlite_master WHERE type = 'table' AND name = 'messages_fts_trigram'N tool_callsrr`)rErFr"ras r%_db_has_trigram_tool_calls_projectionz:SessionFtsSetupMixin._db_has_trigram_tool_calls_projectionsFnn b (* $I= 3.34, this build is %s); CJK/substring search will fall back to LIKE: %s)getattrrjr-infodb_pathsqlite3sqlite_versionselfrVs r_warn_trigram_unavailablez.SessionFtsSetupMixin._warn_trigram_unavailables@ 46 > +/(  > LL  " "   rc|d|_|jryd|_tjd|j|y)NFTzSQLite FTS5 unavailable for %s; full-text session search disabled. Run `hermes update` to rebuild the venv with a current Python (managed uv guarantees FTS5). (underlying error: %s)) _fts_enabled_fts_unavailable_warnedr-r.rmrps r_warn_fts5_unavailablez+SessionFtsSetupMixin._warn_fts5_unavailables<!  ' ' '+$ R LL   rc t|jdj}|js|r|jddj dt Ddt j Dcgc]}|d }}|rPtjdt|jdtf|D]}|jd |d |_ y |jt|s|jdtf|jdjddkDrM|jdjd}dt!|fdfD]\}}|jd||f|jdtfjrd |_ y |jt"|jdj} | |_ y cc}w#tj$r!tjd d d |_ Yy wxYw#tj$r!tjdd d |_ Yy wxYw)uCreate / repair / self-heal the CJK-bigram index (see the module comment). Sets ``_fts_cjk_available``; never raises. Loaded + absent → create (a populated DB gets backfill markers and is NOT served until optimize-storage backfills); loaded + present → ensure triggers, honour the stale breadcrumb; NOT loaded + live triggers → drop them (INSERTs must not fail at trigger time).zNSELECT 1 FROM sqlite_master WHERE type = 'table' AND name = 'messages_fts_cjk'zCSELECT name FROM sqlite_master WHERE type = 'trigger' AND name IN (r:c3 K|]}dywr<r>r?s rrAz>SessionFtsSetupMixin._ensure_fts_cjk_schema..0P0PrBrCru messages_fts_cjk triggers present but the cjk_unicode61 tokenizer is unavailable (%s) — dropping the cjk triggers so message writes keep working. CJK search falls back to trigram/LIKE; run `hermes sessions optimize-storage` on a host with the extension to rebuild.z^INSERT INTO state_meta (key, value) VALUES (?, '1') ON CONFLICT(key) DO UPDATE SET value = '1'DROP TRIGGER IF EXISTS FNzHmessages_fts_cjk presence check failed; CJK search stays on trigram/LIKETr'z$DELETE FROM state_meta WHERE key = ?z2SELECT COUNT(*) FROM messages WHERE role <> 'tool'z)SELECT COALESCE(MAX(id), 0) FROM messagesfts_cjk_rebuild_high_water)fts_cjk_rebuild_progressrzgINSERT INTO state_meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.valuez&SELECT 1 FROM state_meta WHERE key = ?zISELECT 1 FROM state_meta WHERE key = 'fts_cjk_rebuild_high_water' LIMIT 1z@messages_fts_cjk ensure failed; CJK search stays on trigram/LIKE)boolrErF_fts_cjk_loadedrHrrIr-r.rr_fts_cjk_availablernOperationalError executescriptFTS_CJK_TABLE_SQLr FTS_CJK_TRIGGER_SQL) rqr6 cjk_presentrlivetrighwkvbackfill_pendings r_ensure_fts_cjk_schemaz+SessionFtsSetupMixin._ensure_fts_cjk_schemas % v~~` hjK''*0..((+0P>O0P(P'QQRT)+hj "QAaD"D" = -.I.0 %)MD"NN-DTF+KLM*/'5(D ,  !2 3EHYG[\>>"VW``bcdehii(ST]]_`abB5s2w?Ab!1TF~~FIZH\]ffh*/'  !4 5%~~[ hj +;&:D #s"2''  NN)-  ',D #   B'' , NN)-  ',D #  ,s>A:G5< G0AG5&C H,3    %'' 223711#6..s3++C0 s)B ABB ct|S)afCorruption SQLite identifies as FTS-scoped (SQLITE_CORRUPT_VTAB, or an ``fts5:`` report naming ``messages_fts*`` on builds without result codes); a bare malformed image is structural. One rule, shared with ``classify_persistence_error`` and the gateway transcript retry: see :func:`hermes_state_errors.is_fts_scoped_corruption_error`.r )rVs r_is_fts_write_corruption_errorz3SessionFtsSetupMixin._is_fts_write_corruption_errorYs .c22rc|jr|j|sy|j |j5|j |j |j d|j jd |j jdtf|j jddjd tDd tj}|r!|j jdtf|j|j j|j j dddd |_d|_d|_d|_t(j+d |y #t $r|j j#wxYw#1swYdxYw#t$j&$r }t(j+d |Yd}~yd}~wwxYw)zDetach corrupt FTS indexes so canonical writes can continue. Breadcrumb + trigger drop commit atomically: once triggers are absent the index has a gap of unknown extent, so nobody may reinstall them without a full rebuild.FNwrite)contextzBEGIN IMMEDIATEziINSERT INTO state_meta (key, value) VALUES (?, '1') ON CONFLICT(key) DO UPDATE SET value = excluded.valuez@SELECT 1 FROM sqlite_master WHERE type = 'trigger' AND name IN (r:c3 K|]}dywr<r>r?s rrAz.vryrBz ) LIMIT 1zNCould not detach corrupt FTS indexes; canonical write still cannot proceed: %sTzstate.db FTS indexes remain corrupt (%s); disabled FTS sync and retrying the canonical write. Search temporarily uses LIKE until a later SessionDB open rebuilds the indexes.)rtr_raise_if_db_corrupt_lock_raise_if_db_replaced_conn_reopen_after_close_lockedrErrHrrFr_drop_all_fts_triggersr6commit BaseExceptionrollbackrnErrorr-error _fts_stale_trigram_availabler)rqrVcjk_triggers_present detach_excs r_enter_fts_fail_openz)SessionFtsSetupMixin._enter_fts_fail_openas  (K(KC(P !!#"  **,::%33G3D ""#45JJ&&P&( ,0::+=+=((+0P>O0P(P'QR""* , hj ) , **T.0 // 0A0A0CDJJ%%'1 D!"'"'  ;   '%JJ'')3  8}}  LL`   sB G A F>CFG %F;;F>>GG G=G88G=ig@g?fts_v22_trash_c t|jd|jjdddzfj S)zTrue when demoted v22 shadow tables are still awaiting teardown. Caller must hold ``self._lock`` (or pass a migration-time cursor).zSSELECT 1 FROM sqlite_master WHERE type = 'table' AND name LIKE ? ESCAPE '\' LIMIT 1r9z\_%)r}rE_FTS_TRASH_PREFIXreplacerF)rqr%s r_has_fts_trashz#SessionFtsSetupMixin._has_fts_trashsFDLL b  # # + +C 7# = ?  (* r) messages_ftsmessages_fts_trigrammessages_fts_cjk)rN)__name__ __module__ __qualname____doc__ staticmethodrnrr}r\rYCursorrbre classmethodrhrrrvrrr r DatabaseErrorrr_FTS_REBUILD_CHUNK_ROWS_FTS_REBUILD_DUTY_FACTOR_FTS_REBUILD_MIN_PAUSErr _FTS_TABLESr>rrrUrUsYv(@(@vTvv `7+C+C```  E'.. ET E EJgnnJJJj7>>jdjj  W-E-E  $    '*B*B  t  K,Z7>>dSsW[$3G,A,A3d334(=(=4$4x""  )dOKrrU)!rloggingrrnpathlibrtypingrhermes_constantsrhermes_state_commonrrrr r hermes_state_errorsr getLoggerr-rrrr}r$ Connectionr0rGrr listrSrUr>rrrsL ,>   > *, )X[$[ ZZ '"4"4,H7>>Xc]W[\_W`DFOFOr