߲jBEUdZddlmZddlZddlZddlmZmZmZm Z ddl Z ddl m Z m Z mZmZmZmZmZddlmZddlmZddlmZdd lmZej8eZd Zda d e!d <d.d Z"GddZ#e#Z$dZ%d/dZ&d0dZ'd1dZ(d1dZ)GddZ*d2dZ+d1dZ,d1dZ-d2dZ.d3dZ/d4dZ0d5dZ1d6dZ2d7dZ3d d!dd"d8d#Z4d$Z5d%Z6d9d&Z7Gd'd(e Z8dd)lm9Z9dd*lm:Z:ddl;Z;d+d,iZrr s r__repr__z_FirecrawlProxy.__repr__3s1rN)r!rr"rreturnr)r&rr+boolr+str)__name__ __module__ __qualname____doc__ __slots__r#r'r*rrrrr(sTI662rrcddlm}|S)zbClient cache slots live on tools.web_tools so tests that reset ``_firecrawl_client`` there see it.rN)tools.web_tools web_tools)_mods r_wtr8:s " KrcBddlm}||xsdjS)Nr) get_env_value)hermes_cli.configr:strip)namer:s r_envr?@s/ $  %2 , , ..rctd}tdjd}|s|r-dd|fd|ffDcic] \}}|s || c}}d|xsd|xsdffStrd dtid tdffSycc}}w) uDirect Firecrawl ``(mode, kwargs, cache_key)`` or None. ``mode`` is ``"sdk"`` (keyed / self-hosted) or ``"keyless"`` (explicit selection + no credentials → anonymous public cloud; the explicit selection is required so an unconfigured install never silently routes to it).FIRECRAWL_API_KEYFIRECRAWL_API_URL/sdkapi_keyapi_urldirectNkeylesszdirect-keyless)r?rstrip _is_explicit_firecrawl_selection_FIRECRAWL_CLOUD_API_URL)rErFkvs r_get_direct_firecrawl_configrNEs&'G&'..s3G')W)= 7?S(TZ1XYq!tZ]egngvrvxyHDH]II I')9&>?BRTlnrAsss [s A4A4cddlm}|dS)Nr_web_config_selectsr)plugins.web.keyless_mcprQrPs rrJrJRs; { ++rctds tdryddlmmddlm}fddfD] } |ry|d d S#t $rY#wxYw) zRoute via the keyless ring only with no direct credentials, when the managed Nous gateway isn't the selected path, and the keyless tier isn't disabled or pinned paid.rArBFrNOUS_MANAGED_PROVIDERread_selection) use_keylesscdk(S)NwebrrTsrz#_use_keyless_ring.._s./3HHrc0txr t Sr)_is_tool_gateway_readyrJrrrrZz#_use_keyless_ring.._s&RhRjSVtTtVpVrrr;)r?tools.tool_backend_helpersrUrVrRrW Exception)rWproberUrVs @@r_use_keyless_ringr`Wsp  D)<$=P3HKVW w {B ''   sA  AAc6eZdZdZefd dZd dZdddZdZy) _KeylessFirecrawlClientzMinimal REST client for Firecrawl's keyless cloud mode; duck-types the SDK's ``search`` / ``scrape`` and never sends an Authorization header.c0|jd|_y)NrC)rIrF)r rFs r__init__z _KeylessFirecrawlClient.__init__ls~~c* rctj|j||ddid}|j|j S)Nz Content-Typezapplication/jsongN@)jsonheaderstimeout)httpxpostrFraise_for_statusrf)r pathpayloadresponses r_postz_KeylessFirecrawlClient._postosE::tf5Gn^pMq{A!!#}}r)limitc,|jd||dS)Nz /v2/searchqueryrqro)r rtrqs rrZz _KeylessFirecrawlClient.tsTZZ PU`eFf-grc,|jd||dS)Nz /v2/scrapeurlformatsru)r rxrys rrZz _KeylessFirecrawlClient.us4::lC\cDd+erN)rFr.)rlr.rmDict[str, Any]r+rz) r/r0r1r2rKrdrosearchscraperrrrbrbhs&H'?+ +, gF eFrrbc,tjdS)Nr)_gatewaybuild_vendor_gateway_urlrrr_get_firecrawl_gateway_urlrxs  , ,[ 99rcPtjdtjduS)z._managedsh  2 2;XMlMl m :""4"4ARARSVdfhfwfwy{zLzLVMM Mrcnd}tjr|dzS|dztjdzS)NzWeb tools are not configured. Set FIRECRAWL_API_KEY for cloud Firecrawl or set FIRECRAWL_API_URL for a self-hosted Firecrawl instance.z With your Nous subscription you can also use the Tool Gateway. run `hermes tools` and select Nous Subscription as the web provider. zmanaged Firecrawl web tools)rr%nous_tool_gateway_unavailable_message)messages r_unconfigured_messagez4_get_firecrawl_client.._unconfigured_messagesG[  6 6 8dd d}/UUVstttrzRthe Nous Subscription web selection is stored but the tool gateway is unavailable.cddS)NrYzDthe Nous Tool Gateway is not available (not entitled or unreachable)r)rUrsrrZz'_get_firecrawl_client..sL[ (*pLrrNzMdirect Firecrawl selected but FIRECRAWL_API_KEY/FIRECRAWL_API_URL is not set.c dxsddS)NrYrz6neither FIRECRAWL_API_KEY nor FIRECRAWL_API_URL is setr)rrsrrZz'_get_firecrawl_client..s JY 8*{,dJfrz,missing direct config and tool-gateway auth.z*Firecrawl client initialization failed: %s_firecrawl_client_firecrawl_client_configrHrF)rFr-r)r8r]rUrVrrNrJloggererror ValueErrorgetattrrbrrr)wtrV direct_configrrresolvedlogr client_moder" client_configcachedrUrrs @@@r_get_firecrawl_clientrs0 Baae$H02MM u((!).BDrw#  !A!C "/0Bfw#  " !)-[]rw# A3G##)1&K R,d 3F gb*DdK}\ Q\`iQi26);LMoxpC|BpCB"/B  rc |*t|ttttt t fr|SddfddffD]\}}t||s ||cS|S#t$rY,wxYw)uRSDK objects (pydantic ``model_dump`` / ``__dict__``) → plain data when possible. model_dumpc"|jSr)r)rMs rrZz"_to_plain_object..s 1<<>r__dict__c|jjDcic]\}}|jdr||c}}Scc}}w)N_)ritems startswith)rMrLxs rrZz"_to_plain_object..samnmwmwm}m}m\ZeiefhiHIHTHTUXHY\]_`\`\Z\Zs AA) r%dictlistr.intfloatr,hasattrr^)valueattrconverts r_to_plain_objectrs } 54sC*MN ')ABZRZE[\ g 5$  u~% L  s A A&%A&ct|tr/tt|Dcgc]}t|ts|c}SgScc}wr)r%rmaprr)valuesps r_normalize_result_listrs8PZ[acgPhs+V4 L! 1d8KA Lpnpp Ls AAct|}t|tr|jd}t|tr t |St|tr"|jd|jdgng}||jd|jdgzD]}t |}|s|cSt |drt t|dgSgS)z9Search results across SDK/direct/gateway response shapes.datarYresults)rr%rgetrrrr)rnplainr candidates candidate normalizeds r_extract_web_search_resultsrs X &E%yy  dD !)$/ /?I$PT?Udhhuotxx ':;[] #uyy'799M&NN "I/ :J!! "x%ghr&BCC Irct|}t|tsiSt|jdtr|dS|S)Nr)rr%rr) scrape_resultrs r_extract_scrape_payloadrs< ] +E eT " &uyy'8$?5=JUJrr;Ftitlerawblockedch|rddDcic]}||| c}ini}||dd|rddinid|i|Scc}w)zPer-URL extract failure. ``raw`` adds ``raw_content`` (post-scrape failures carry it, pre-scrape ones don't); ``blocked`` adds ``blocked_by_policy``.blocked_by_policy)hostrulesourcer;)rxrcontent raw_contentrr)rxrrrrrLpolicys r _error_entryrs\\c!;U#VaAwqzM#V WhjF2 xQT=":MZ\ x_fhm xqw xx$Ws /ucScrape timed out after 60s — page may be too large or unresponsive. Try browser_navigate instead.z:Blocked: URL targets a private or internal network addresscKt|x}r.tjd|d|dt||d|S tjd| t j t j tj||d d {}t|}|jd i}t|ts)t!|d r|j#n t%|di}|jdd|jd|}}t'|s)tjd|t|t(|dSt|x} r0tjd| d| dt|| d|d| S|jd|jd} } |dk(s|| r| n | xs| xsd} ||| | |dS7/#tj$r)tjd |t|tcYSwxYw#t*$r8} tj-d|| t|t/| dcYd } ~ Sd } ~ wwxYww)zScrape one URL (60s timeout) and re-check SSRF + website policy against the post-redirect URL. Never raises for scrape errors; returns an error entry instead.z%Blocked web_extract for %s by rule %srrr)rzFirecrawl scraping: %srw<)rhNz!Firecrawl scrape timed out for %smetadatarrrr; sourceURLz7Blocked redirected web_extract for unsafe final URL: %sT)rrz0Blocked redirected web_extract for %s by rule %srmarkdownhtml)rxrrrrz"Firecrawl scrape failed for %s: %s)r)rrinforasynciowait_for to_threadrr| TimeoutErrorwarning_SCRAPE_TIMEOUT_MSGrrr%rrrrr_UNSAFE_REDIRECT_MSGr^debugr.)rxryformatrrrmrr final_url final_blockedrrr scrape_errs r _scrape_oners4's++w+ ;WV_gV\o^C!3WEE< ,c2 :")"2"273D3DEZE\EcEcilv}3~IK#LLM*-8;;z2.(D)07,0Ox**,U\]egqsuUvH#<<4hll;PS6Ty9% KKQS\ ] +?uRVW W0; ;= ; KKJMZ`Lacpqwcx y =+C5VZdqr r Z0'++f2E$$ 2v~((Y]YoaiYomo 5WU\jrss%L## : NN> D%89 9 :$ < 93 KCZd;;H I G 9HH I HH I -I?I I I  I cBeZdZdZdZdZdZdZd dZd d dZ d dZ ddZ y )FirecrawlWebSearchProviderz9Firecrawl search + extract provider with dual auth paths.rrTctSr)rr)s r is_availablez'FirecrawlWebSearchProvider.is_availables &((rcddlm}|r tdStrt dd||t St j d||t} t|j||}t j dt|t|S#t$r.}t jd |td |cYd }~Sd }~wwxYw) zPre-flight errors (ValueError / ImportError) propagate so the dispatcher emits the legacy ``tool_error`` envelope; in-flight errors become failure dicts.ris_interrupted Interruptedrrz!Firecrawl search: '%s' (limit=%d)rsz"Firecrawl: found %d search resultszFirecrawl search error: %szFirecrawl search failed: N)tools.interruptrr r`r rrrrr{lenr r^r)r rtrqrclient web_resultsexcs rr{z!FirecrawlWebSearchProvider.search!s 3  }- -  !+{E5&Q Q 7F&( B5fmm%W\m6]^K KKN O[) ) B NN7 =!:3%@A A BsAB C'#C CCcfKddlm}|r|Dcgc]}|ddd c}Str)tjt dd|t d{S|jd }|d vr|gnd d g}|Dcgc]$}|r|dddnt|||d{&c}Scc}w7U7cc}ww) zPer-URL scrape; failures become items with an ``error`` field. ``format``: "markdown" | "html" | both (markdown preferred).rrrr;)rxrrrrNr)rrrr) rrr`rrr rrr)r urlsr"_is_interruptedurryrxs rextractz"FirecrawlWebSearchProvider.extract3s F  MQRA CR R   **?KVZ\bcc cH%$(<<6(:vBV BQARC-" =^ijmovx~^X   ScY@ s?B1 B#1B1B($B17"B,B* B, B1*B,,B1c"tddddddS)Nru keyless/paid · optional gatewayzYFull search + extract; supports keyless cloud, direct API, and Nous tool-gateway routing.rAzBFirecrawl API key (optional; blank = keyless cloud or self-hosted)z'https://docs.firecrawl.dev/introduction)rr)s rget_setup_schemaz+FirecrawlWebSearchProvider.get_setup_schemaBs% ; g !ehQ  rNr+r,)rp)rtr.rqrr+rz)r List[str]r"rr+List[Dict[str, Any]])r+rz) r/r0r1r2NAME DISPLAY_NAMEEXTRACTKEYLESSrr{rrrrrrrs.C DLGG)B$   rr)NoReturn) TYPE_CHECKINGWebSearchProvider)zagent.web_search_providerr ctj|}|tdtd|ddl}ddlm}|t|g|t|j|d|dS)Nzmodule z has no attribute r) warn_once) _PLUGIN_COMPAT_LAZYrAttributeErrorr/ importlibhermes_cli.plugin_compatr r import_module)r>targetrr s r __getattr__rXse $ $T *F ~wxl2DTHMNN2 h&v& 9**6!95vay AAr)r+type)r>r.r+r.)r+zOptional[tuple]rr-)r+r)rrr+r)rrr+r)rnrr+r)rrr+rz) rxr.rr.rr.rr,rzOptional[Dict[str, Any]]r+rz)rxr.ryrrz Optional[str]r+rz)>r2 __future__rrloggingtypingrrrrriplugins.web._commonrr r r r r rtoolsrr~rrtools.url_safetyrtools.website_policyr getLoggerr/rrKr__annotations__rrrr8r?rNrJr`rbrr\rrrrrrrrrrrrr r osrrrrrr!s& #,, JJJ2:(5   8 $6(,n+  2 2    / , (" f f :x ij. d q"K8:urvy|S