Pegas
13e0b87446
- ingest.parse_web: direct fetch; on block/timeout retry via socks5h://127.0.0.1:1080 (SOCKS_FALLBACK_PROXY); raises ValueError if both fail. - config: add SOCKS_FALLBACK_PROXY default (socks5h://127.0.0.1:1080). - self_improve.stale_guard: supported LLM verdict now sets status_change='supported' (authoritative), no fallthrough to age fallback -> no double-checked/skipped_no_source. - README: document auto-fallback.
108 lines
3.7 KiB
Python
108 lines
3.7 KiB
Python
"""Парсеры источников: text/pdf/epub/web → чанки."""
|
||
from __future__ import annotations
|
||
import hashlib
|
||
from dataclasses import dataclass, field
|
||
|
||
@dataclass
|
||
class ParsedChunk:
|
||
seq: int
|
||
content: str
|
||
locator: dict = field(default_factory=dict)
|
||
|
||
@dataclass
|
||
class ParsedDoc:
|
||
kind: str
|
||
uri: str | None
|
||
title: str | None
|
||
chunks: list[ParsedChunk]
|
||
content_hash: str = ""
|
||
|
||
def parse_text(text: str, *, chunk_size: int = 2000, overlap: int = 200,
|
||
title: str | None = None, uri: str | None = None) -> ParsedDoc:
|
||
"""Разбить сырой текст на чанки с перекрытием."""
|
||
text = text.strip()
|
||
chunks: list[ParsedChunk] = []
|
||
start = 0
|
||
seq = 0
|
||
while start < len(text):
|
||
end = min(start + chunk_size, len(text))
|
||
# не резать посреди слова
|
||
if end < len(text):
|
||
space = text.rfind(" ", start, end)
|
||
if space > start + chunk_size // 2:
|
||
end = space
|
||
chunks.append(ParsedChunk(seq=seq, content=text[start:end].strip()))
|
||
seq += 1
|
||
if end >= len(text):
|
||
break
|
||
start = end - overlap
|
||
return ParsedDoc(kind="text", uri=uri, title=title, chunks=chunks,
|
||
content_hash=hashlib.sha256(text.encode()).hexdigest())
|
||
|
||
def parse_pdf(path: str) -> ParsedDoc:
|
||
from pypdf import PdfReader
|
||
reader = PdfReader(path)
|
||
text = "\n\n".join((page.extract_text() or "") for page in reader.pages)
|
||
doc = parse_text(text, title=path.split("/")[-1], uri=path)
|
||
doc.kind = "pdf"
|
||
return doc
|
||
|
||
def parse_epub(path: str) -> ParsedDoc:
|
||
import ebooklib
|
||
from ebooklib import epub
|
||
book = epub.read_epub(path)
|
||
parts = []
|
||
for item in book.get_items_of_type(ebooklib.ITEM_DOCUMENT):
|
||
parts.append(item.get_content().decode("utf-8", errors="ignore"))
|
||
text = "\n\n".join(parts)
|
||
doc = parse_text(text, title=path.split("/")[-1], uri=path)
|
||
doc.kind = "epub"
|
||
return doc
|
||
|
||
def _http_client(proxy=None, timeout: int = 30):
|
||
"""httpx.Client с единым User-Agent.
|
||
|
||
proxy: явный прокси (None = прямой запрос). Если проси не передан, но
|
||
config.PROXY_URL задан — используется он.
|
||
"""
|
||
import httpx
|
||
from . import config
|
||
kwargs = dict(timeout=timeout, follow_redirects=True,
|
||
headers={"User-Agent": config.USER_AGENT})
|
||
use_proxy = proxy or (config.PROXY_URL or None)
|
||
if use_proxy:
|
||
kwargs["proxy"] = use_proxy
|
||
return httpx.Client(**kwargs)
|
||
|
||
def _fetch_text(url: str, proxy: str | None = None) -> str | None:
|
||
"""Fetch + extract main content. Tries direct; on error/empty retries `proxy`.
|
||
|
||
trafilatura.extract has no timeout -> fetch via httpx with timeout.
|
||
"""
|
||
import trafilatura
|
||
for _lbl, _prx in (("direct", None), ("proxy", proxy)):
|
||
try:
|
||
with _http_client(proxy=_prx, timeout=30) as client:
|
||
r = client.get(url)
|
||
r.raise_for_status()
|
||
text = trafilatura.extract(r.text, include_comments=False,
|
||
include_tables=True, url=url)
|
||
if text:
|
||
return text
|
||
except Exception:
|
||
continue
|
||
return None
|
||
|
||
def parse_web(url: str) -> ParsedDoc:
|
||
from . import config
|
||
# direct fetch + socks5h fallback on block/timeout
|
||
text = _fetch_text(url, proxy=config.SOCKS_FALLBACK_PROXY)
|
||
if not text:
|
||
raise ValueError(
|
||
f"no main content for {url} "
|
||
f"(direct + {config.SOCKS_FALLBACK_PROXY} fallback failed)"
|
||
)
|
||
doc = parse_text(text, uri=url, title=url)
|
||
doc.kind = "web"
|
||
return doc
|