Ë
    ãß²j±&  ã            
       ó>  — d Z ddlZddlZddlmZmZmZ  ej                  d«      ZdZ	dZ
d\  ZZded	efd
„Zded	efd„Zd	efd„Zded	efd„Zdeded	ee   fd„Zdededed	eeef   fd„Zdee   d	efd„ZdZded	efd„Zdee   deded	dfd„Zdee   d	ee   fd„Zy)a�  Truncate-and-store pipeline for web_extract (no LLM).

Pages at or under the char budget are returned whole; larger pages become a head+tail window plus a
footer that says how much is shown, where the full text is stored (cache/web) and the exact read_file
call that pages the omitted middle. Inline base64 images become ``[IMAGE: alt]`` placeholders. Logs under the
origin (tools.web_tools) logger.
é    N)ÚAnyÚListÚOptionalztools.web_toolsi˜:  i€„ )iÐ  i ¡ ÚvalueÚreturnc                 óP   — t        t        t        t        | «      t        «      «      S )zžClamp to [2k, 500k] (below 2k the truncation footer dominates; a config typo must not blow up context);
    raises TypeError/ValueError for non-numeric input.)ÚmaxÚ_CHAR_LIMIT_FLOORÚminÚintÚ_CHAR_LIMIT_CEILING©r   s    ú=/home/ubuntu/.hermes/hermes-agent/tools/web_tools_truncate.pyÚ_clamp_char_limitr      s   € ô Ô ¤#¤c¨%£jÔ2EÓ"FÓGÐGó    c                 ó`   — 	 | €t         S t        | «      S # t        t        f$ r	 t         cY S w xY w)zV``_clamp_char_limit(value)``; ``None`` or non-numeric input falls back to the default.)ÚDEFAULT_EXTRACT_CHAR_LIMITr   Ú	TypeErrorÚ
ValueErrorr   s    r   Ú_clamp_or_defaultr   !   s8   € ð*Ø-2¨]Ô)ÐXÔ@QÐRWÓ@XÐXøÜ”zÐ"ò *Ü)Ò)ð*ús   ‚ Š
 •-¬-c                  óL   — ddl m}  t         | «       j                  d«      «      S )zE``web.extract_char_limit`` clamped to a sane range, else the default.r   ©Ú_load_web_configÚextract_char_limit)Útools.web_toolsr   r   Úgetr   s    r   Ú_get_extract_char_limitr   )   s    € å0ÜÑ-Ó/×3Ñ3Ð4HÓIÓJÐJr   Útextc                 ó¤   — dddt         fd„}t        j                  d|| «      }t        j                  dd|«      }t        j                  dd|«      S )	a+  Replace inline base64 image blobs (token bombs) with ``[IMAGE: alt]`` placeholders: markdown images
    (alt kept), parenthesised blobs, and bare ``data:image/...;base64,`` payloads. Real http(s) markdown
    image links are left untouched so the agent can ``web_extract`` / ``vision_analyze`` them.Úmzre.Match[str]r   c                 ó\   — | j                  d«      xs dj                  «       x}rd|› d�S dS )NÚaltÚ z[IMAGE: ú]ú[IMAGE])ÚgroupÚstrip)r    r"   s     r   Ú_md_replz0convert_base64_images_to_links.<locals>._md_repl3   s7   € Ø-.¯W©W°U«^Ò-A¸r×,HÑ,HÓ,JÐ%J SÐ%J�˜#˜˜aÐ ÐZÐQZÐZr   zD!\[(?P<alt>[^\]]*)\]\(\s*data:image/[^;]+;base64,[A-Za-z0-9+/=\s]+\)z0\(\s*data:image/[^;]+;base64,[A-Za-z0-9+/=\s]+\)r%   z'data:image/[^;]+;base64,[A-Za-z0-9+/=]+)ÚstrÚreÚsub)r   r(   Úouts      r   Úconvert_base64_images_to_linksr-   /   sW   € ð[�Oð [¬ó [ô �&‰&ÐXÐZbÐdhÓ
i€CÜ
�&‰&ÐDÀiÐQTÓ
U€CÜ�6‰6Ð<¸iÈÓMÐMr   ÚurlÚcontentc                 óØ  — 	 ddl }ddlm} ddlm}  |dd«      }|j                  dd¬«       | || «      › d	|j                  | j                  d
«      «      j                  «       dd › d�z  }t        |«      t        kD  r"|dt         dt        d›dt        |«      d›d�z   }ddlm}  |||dd¬«       t        |«      S # t        $ r!}t        j!                  d| |«       Y d}~yd}~ww xY w)a  Write the full page to cache/web; absolute path or None (best-effort: the truncated content is still
    returned). cache/web is mounted read-only into remote backends (credential_files _CACHE_DIRS) so
    read_file can page the complete text on any backend.r   N)Úget_hermes_dir)Ú
_host_slugz	cache/webÚ	web_cacheT)ÚparentsÚexist_okÚ-zutf-8é
   z.mdz 

[... stored copy truncated at ú,z
 chars of z2; re-extract a more specific URL for the rest ...])Úwrite_text_exclusiveF)ÚprivateÚ	overwritez0Failed to store full web_extract text for %s: %s)ÚhashlibÚhermes_constantsr1   Útools.web_result_cacher2   ÚmkdirÚsha256ÚencodeÚ	hexdigestÚlenÚMAX_STORED_TEXT_CHARSÚtools.spill_safetyr9   r)   Ú	ExceptionÚloggerÚdebug)	r.   r/   r<   r1   r2   Ú	cache_dirÚpathr9   Úexcs	            r   Ú_store_full_textrL   ;   s÷   € ðÛÝ3Ý5Ù" ;°Ó<ˆ	Ø�‰ ¨tˆÔ4Ø™j¨›oÐ.¨a°·±¸s¿z¹zÈ'Ó?RÓ0S×0]Ñ0]Ó0_Ð`cÐacÐ0dÐ/eÐehÐiÑiˆÜˆw‹<Ô/Ò/ØÐ4Ô4Ð5Ø4Ô5JÈ1Ð4Mð NÜ˜'“l 1Ð%Ð%WðYñˆGõ 	<ñ 	˜T 7°EÀTÕJÜ�4‹yÐøÜò Ü�‰ÐGÈÈcÔRÜûðús   ‚B<B? Â?	C)ÃC$Ã$C)Ú
char_limitc           
      óZ  — t        | «      |k  r| dfS t        |dz  «      }||z
  }| d| | | d }}|j                  d«      x}|dz  kD  r|d| }d|j                  d«      x}cxk  r|dz  k  rn n||dz   d }t	        || «      }|rddlm}	  |	|«      }d	d
dt        |«      d›dt        |«      d›dt        | «      d›d�g}
|r&|j                  d«      dz   }|
d|› �d|› d|› d�gz  }
n|
j                  d«       |
j                  d«       |dz   |z   }|dz   dj                  |
«      z   dfS )a  Return (model_text, was_truncated). Pages over ``char_limit`` become a ~75% head / ~25% tail window cut
    on line boundaries, plus a footer saying how much is shown, where the full text is stored, and the
    read_file call that pages the omitted middle. Deterministic.Fg      è?NÚ
g      à?r   é   )Úto_agent_visible_cache_pathr#   u=   â”€â”€â”€â”€â”€â”€â”€â”€ [TRUNCATED] â”€â”€â”€â”€â”€â”€â”€â”€zShowing r8   z chars (head) + z chars (tail) of z total clean characters.é   zFull text saved to: z,To read the omitted middle: read_file path="z	" offset=zS limit=200  (the file is the complete page; raise/lower offset to page through it).zwFull text could not be stored; re-run web_extract on a more specific URL or use browser_navigate for the complete page.uW   â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€â”€u+   

[... middle omitted â€” see footer ...]

T)
rC   r   ÚrfindÚfindrL   Útools.credential_filesrQ   ÚcountÚappendÚjoin)r/   r.   rM   Úhead_budgetÚtail_budgetÚheadÚtailÚnlÚstored_pathrQ   Úfooter_linesÚmiddle_start_lineÚ
model_texts                r   Ú_truncate_with_footerrb   V   s¢  € ô ˆ7ƒ|�zÒ!Ø˜ˆ~ÐÜ�j 4Ñ'Ó(€KØ˜{Ñ*€KØ˜˜+Ð&¨°°°Ð(>ˆ$€Dà�j‰j˜ÓÐˆ +°Ñ"3Ò3Ø�C�RˆyˆØ�4—9‘9˜T“?Ð"ˆRÔ7 k°CÑ&7Õ7Ø�B˜‘F�Gˆ}ˆä" 3¨Ó0€KÙõ 	GÙ1°+Ó>ˆà
Ð3Ø
”3�t“9˜Q�-Ð/´°D³	¸!¨}ð =Ü�'‹l˜1ÐÐ5ð	7ð€Lñ
 à ŸJ™J tÓ,¨qÑ0ÐØØ" ; -Ð0Ø:¸;¸-ð HØ'Ð(ð )6ð7ð
ñ 	
‰ð 	×ÑðJô	
ð ×Ñ˜
Ô#ØÐIÑIÈDÑP€JØ˜Ñ˜tŸy™y¨Ó6Ñ6¸Ð<Ð<r   c                 ó0   — | �t        | «      S t        «       S )z[Caller's ``char_limit`` (else config) clamped; non-numeric input falls back to the default.)r   r   )rM   s    r   Ú_effective_char_limitrd   ƒ   s   € à,6Ð,BÔ˜ZÓ(ÐaÔH_ÓHaÐar   )ÚSQLiteÚZIPÚgzipÚbzip2Úxzz7-ZipÚELFzMach-OÚPNGÚJPEGÚGIFÚTIFFÚFLACÚOggc                 óî   — ddl m} | dd j                  dd«      j                  dd«      }|D ]E  \  }}|j                  dd«      }|sŒ|j	                  t
        «      sŒ1|j	                  |«      sŒC|c S  y	)
a  Magic-byte type name when a fetched body is a raw binary file, else ``""``. Backends return
    the body as text with NUL bytes dropped, so signatures are compared NUL-stripped on both sides.
    Only the file tools' own signature table; HTML/markdown/JSON never start with one.r   )Ú_MAGIC_SIGNATURESNé    zlatin-1Úignoreó    r   r#   )Útools.file_operationsrr   rA   ÚreplaceÚ
startswithÚ_UNAMBIGUOUS_BINARY_KINDS)r   rr   r[   ÚprefixÚnameÚsigs         r   Ú_binary_payload_kindr}   ‹   st   € õ 8à��ˆ9×Ñ˜I xÓ0×8Ñ8¸À#ÓF€DØ)ò ‰ˆ�Ø�n‰n˜W cÓ*ˆÚ�4—?‘?Ô#<Õ=À$Ç/Á/ÐRUÕBVØŠKðð r   ÚresultsÚdebug_call_datac           	      ór  — | D �]1  }|j                  dd«      }|j                  dd«      xs |j                  dd«      }|j                  d«      s|sŒPt        |«      }|r0d|d<   d|› d�|d<   t        j                  d||t	        |«      «       Œ�t        |«      }t        |||«      \  }}	||d<   |	rc|d	xx   d
z  cc<   |d   j                  |t	        |«      t	        |«      dœ«       t        j                  d|t	        |«      t	        |«      «       �Œt        j                  d|t	        |«      «       �Œ4 y)z–In place: replace each successful entry's content with its base64-cleaned, budgeted text;
    per-page truncation metrics go into ``debug_call_data``.r.   r#   Úraw_contentr/   ÚerrorzURL returned binary content (z�), not a page. Download it with the terminal (curl -L -o) and use read_file (SQLite/Office/PDF auto-extract) or terminal utilities on the file.z)%s (binary payload: %s, %d chars dropped)Úpages_truncatedrP   Útruncation_metrics)r.   Úoriginal_sizeÚ	sent_sizez%s (truncated %d -> %d chars)z%s (%d chars, whole)N)r   r}   rG   ÚinforC   r-   rb   rW   )
r~   rM   r   Úresultr.   r�   Úbinary_kindÚcleanra   Ú	truncateds
             r   Ú_truncate_resultsrŒ   ™   s=  € ð ó AˆØ�j‰j˜ Ó#ˆØ—j‘j °Ó3ÒP°v·z±zÀ)ÈRÓ7PˆØ�:‰:�gÔ¡kØÜ*¨;Ó7ˆÙð !#ˆF�9Ñà/°¨}ð =uð uð �7‰Oô �K‰KÐCÀSÈ+ÔWZÐ[fÓWgÔhØÜ.¨{Ó;ˆÜ 5°e¸SÀ*Ó MÑˆ
�IØ&ˆˆyÑÙØÐ-Ó.°!Ñ3Ó.ØÐ0Ñ1×8Ñ8Ø¬c°%«jÄsÈ:ÃÑWôô �K‰KÐ7¸¼cÀ%»jÌ#ÈjË/ÖZä�K‰KÐ.°´S¸³ZÖ@ñ3Ar   c           	      óÎ   — | D �cg c]U  }|j                  dd«      |j                  dd«      |j                  dd«      |j                  d«      dœd|v rd|d   ini ¥‘ŒW c}S c c}w )zOKeep only url/title/content/error per entry (+ blocked_by_policy when present).r.   r#   Útitler/   r‚   )r.   rŽ   r/   r‚   Úblocked_by_policy)r   )r~   Úrs     r   Ú_trim_resultsr‘   ¸   s€   € ð öð ð	 —5‘5˜ Ó#¨a¯e©e°G¸RÓ.@ÈQÏUÉUÐS\Ð^`ÓMaØ—U‘U˜7“^ñ	
ð ATÐWXÑ@XÐ# QÐ':Ñ%;Ñ<Ð^`ò	
òð ùò s   …AA")Ú__doc__Úloggingr*   Útypingr   r   r   Ú	getLoggerrG   r   rD   r
   r   r   r   r   r   r)   r-   rL   ÚtupleÚboolrb   rd   ry   r}   ÚdictrŒ   r‘   © r   r   ú<module>rš      sZ  ðñó Û 	ß &Ñ &à	ˆ×	Ñ	Ð,Ó	-€ð #Ð ð "Ð à)6Ñ &Ð Ð&ðH˜Sð H Só Hð*˜Sð * Só *ðK ó Kð	N¨ð 	N°ó 	Nð˜#ð ¨ð °¸±ó ð6*= 3ð *=¨Sð *=¸cð *=ÀeÈCÐQUÈIÑFVó *=ðZb h¨s¡mð b¸ó bð
 LÐ ð˜sð  só ðA˜t D™zð A°sð AÈTð AÐVZó Að>	˜4 ™:ð 	¨$¨t©*ô 	r   