fix(pegas-memory): web fetch SOCKS5h proxy fallback + stale_guard supported double-count fix
- ingest.parse_web: direct fetch; on block/timeout retry via socks5h://127.0.0.1:1080 (SOCKS_FALLBACK_PROXY); raises ValueError if both fail. - config: add SOCKS_FALLBACK_PROXY default (socks5h://127.0.0.1:1080). - self_improve.stale_guard: supported LLM verdict now sets status_change='supported' (authoritative), no fallthrough to age fallback -> no double-checked/skipped_no_source. - README: document auto-fallback.
This commit is contained in:
1 parent
eca8a11e23
commit
13e0b87446
4 files changed
+44
-15
No files matched your search
+36
-13
@@ -59,26 +59,49 @@ def parse_epub(path: str) -> ParsedDoc:
|
||||
doc.kind = "epub"
|
||||
return doc
|
||||
|
||||
def _http_client():
|
||||
"""httpx.Client с опциональным прокси и единым User-Agent."""
|
||||
def _http_client(proxy=None, timeout: int = 30):
|
||||
"""httpx.Client с единым User-Agent.
|
||||
|
||||
proxy: явный прокси (None = прямой запрос). Если проси не передан, но
|
||||
config.PROXY_URL задан — используется он.
|
||||
"""
|
||||
import httpx
|
||||
from . import config
|
||||
kwargs = dict(timeout=30, follow_redirects=True,
|
||||
kwargs = dict(timeout=timeout, follow_redirects=True,
|
||||
headers={"User-Agent": config.USER_AGENT})
|
||||
if config.PROXY_URL:
|
||||
kwargs["proxy"] = config.PROXY_URL
|
||||
use_proxy = proxy or (config.PROXY_URL or None)
|
||||
if use_proxy:
|
||||
kwargs["proxy"] = use_proxy
|
||||
return httpx.Client(**kwargs)
|
||||
|
||||
def parse_web(url: str) -> ParsedDoc:
|
||||
def _fetch_text(url: str, proxy: str | None = None) -> str | None:
|
||||
"""Fetch + extract main content. Tries direct; on error/empty retries `proxy`.
|
||||
|
||||
trafilatura.extract has no timeout -> fetch via httpx with timeout.
|
||||
"""
|
||||
import trafilatura
|
||||
# fetch через httpx с таймаутом + прокси (trafilatura.fetch_url не имеет таймаута и виснет)
|
||||
with _http_client() as client:
|
||||
r = client.get(url)
|
||||
r.raise_for_status()
|
||||
text = trafilatura.extract(r.text, include_comments=False, include_tables=True,
|
||||
url=url)
|
||||
for _lbl, _prx in (("direct", None), ("proxy", proxy)):
|
||||
try:
|
||||
with _http_client(proxy=_prx, timeout=30) as client:
|
||||
r = client.get(url)
|
||||
r.raise_for_status()
|
||||
text = trafilatura.extract(r.text, include_comments=False,
|
||||
include_tables=True, url=url)
|
||||
if text:
|
||||
return text
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
def parse_web(url: str) -> ParsedDoc:
|
||||
from . import config
|
||||
# direct fetch + socks5h fallback on block/timeout
|
||||
text = _fetch_text(url, proxy=config.SOCKS_FALLBACK_PROXY)
|
||||
if not text:
|
||||
raise ValueError(f"trafilatura: no main content for {url}")
|
||||
raise ValueError(
|
||||
f"no main content for {url} "
|
||||
f"(direct + {config.SOCKS_FALLBACK_PROXY} fallback failed)"
|
||||
)
|
||||
doc = parse_text(text, uri=url, title=url)
|
||||
doc.kind = "web"
|
||||
return doc
|
||||
Reference in new issue
Block a user