From 13e0b8744686fe83e7850bddeb74ca050555632f Mon Sep 17 00:00:00 2001
From: Pegas <pegas@local>
Date: Thu, 15 Oct 2026 22:25:22 +0500
Subject: [PATCH] fix(pegas-memory): web fetch SOCKS5h proxy fallback +
 stale_guard supported double-count fix

- ingest.parse_web: direct fetch; on block/timeout retry via socks5h://127.0.0.1:1080 (SOCKS_FALLBACK_PROXY); raises ValueError if both fail.
- config: add SOCKS_FALLBACK_PROXY default (socks5h://127.0.0.1:1080).
- self_improve.stale_guard: supported LLM verdict now sets status_change='supported' (authoritative), no fallthrough to age fallback -> no double-checked/skipped_no_source.
- README: document auto-fallback.
---
 README.md                    |  5 +++-
 pegas_memory/config.py       |  3 ++-
 pegas_memory/ingest.py       | 49 ++++++++++++++++++++++++++----------
 pegas_memory/self_improve.py |  2 ++
 4 files changed, 44 insertions(+), 15 deletions(-)

diff --git a/README.md b/README.md
index 23e46df..e7d2eb8 100644
--- a/README.md
+++ b/README.md
@@ -76,7 +76,10 @@ Cloudflare anti-bot (neo4j.com — 403). Поведение research-цикла:
 ```bash
 export PROXY_URL="socks5://127.0.0.1:1080"   # hysteria2
 ```
-Когда прокси задан, `parse_web` ходит через него; иначе — напрямую.
+
+**Авто-повор:** `parse_web` сначала грузит URL напрямую; при таймауте/блокировке повторяет запрос через `socks5h://127.0.0.1:1080` (дефолт,
+`SOCKS_FALLBACK_PROXY`). Если и проки недоступен — падает с
+`ValueError: no main content ... (direct + ... fallback failed)`.
 Поддержка SOCKS5 уже установлена (`httpx[socks]`).
 
 ## Управление пробелами (gaps)
diff --git a/pegas_memory/config.py b/pegas_memory/config.py
index defd173..9e38994 100644
--- a/pegas_memory/config.py
+++ b/pegas_memory/config.py
@@ -19,5 +19,6 @@ ATOM_STATUSES = ("candidate", "verified", "stale", "contested")
 
 DEDUP_THRESHOLD = float(os.getenv("DEDUP_THRESHOLD", "0.90"))  # порог семантической дедупликации
 
-PROXY_URL = os.getenv("PROXY_URL", "")  # опционально: socks5://127.0.0.1:1080 или http://...
+PROXY_URL = os.getenv("PROXY_URL", "")  # явный прокси: используется всегда (socks5://… или http://…)
+SOCKS_FALLBACK_PROXY = os.getenv("SOCKS_FALLBACK_PROXY", "socks5h://127.0.0.1:1080")  # авто-повор через прокси при ошибке прямого запроса
 USER_AGENT = os.getenv("USER_AGENT", "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36")
diff --git a/pegas_memory/ingest.py b/pegas_memory/ingest.py
index 1e210d8..cc84ab8 100644
--- a/pegas_memory/ingest.py
+++ b/pegas_memory/ingest.py
@@ -59,26 +59,49 @@ def parse_epub(path: str) -> ParsedDoc:
     doc.kind = "epub"
     return doc
 
-def _http_client():
-    """httpx.Client с опциональным прокси и единым User-Agent."""
+def _http_client(proxy=None, timeout: int = 30):
+    """httpx.Client с единым User-Agent.
+
+    proxy: явный прокси (None = прямой запрос). Если проси не передан, но
+    config.PROXY_URL задан — используется он.
+    """
     import httpx
     from . import config
-    kwargs = dict(timeout=30, follow_redirects=True,
+    kwargs = dict(timeout=timeout, follow_redirects=True,
                   headers={"User-Agent": config.USER_AGENT})
-    if config.PROXY_URL:
-        kwargs["proxy"] = config.PROXY_URL
+    use_proxy = proxy or (config.PROXY_URL or None)
+    if use_proxy:
+        kwargs["proxy"] = use_proxy
     return httpx.Client(**kwargs)
 
-def parse_web(url: str) -> ParsedDoc:
+def _fetch_text(url: str, proxy: str | None = None) -> str | None:
+    """Fetch + extract main content. Tries direct; on error/empty retries `proxy`.
+
+    trafilatura.extract has no timeout -> fetch via httpx with timeout.
+    """
     import trafilatura
-    # fetch через httpx с таймаутом + прокси (trafilatura.fetch_url не имеет таймаута и виснет)
-    with _http_client() as client:
-        r = client.get(url)
-    r.raise_for_status()
-    text = trafilatura.extract(r.text, include_comments=False, include_tables=True,
-                               url=url)
+    for _lbl, _prx in (("direct", None), ("proxy", proxy)):
+        try:
+            with _http_client(proxy=_prx, timeout=30) as client:
+                r = client.get(url)
+            r.raise_for_status()
+            text = trafilatura.extract(r.text, include_comments=False,
+                                       include_tables=True, url=url)
+            if text:
+                return text
+        except Exception:
+            continue
+    return None
+
+def parse_web(url: str) -> ParsedDoc:
+    from . import config
+    # direct fetch + socks5h fallback on block/timeout
+    text = _fetch_text(url, proxy=config.SOCKS_FALLBACK_PROXY)
     if not text:
-        raise ValueError(f"trafilatura: no main content for {url}")
+        raise ValueError(
+            f"no main content for {url} "
+            f"(direct + {config.SOCKS_FALLBACK_PROXY} fallback failed)"
+        )
     doc = parse_text(text, uri=url, title=url)
     doc.kind = "web"
     return doc
diff --git a/pegas_memory/self_improve.py b/pegas_memory/self_improve.py
index c76cc92..3948c3e 100644
--- a/pegas_memory/self_improve.py
+++ b/pegas_memory/self_improve.py
@@ -198,6 +198,8 @@ def stale_guard(conn, *, atom_id=None, all: bool = False, limit: int = 20,
                     status_change = "stale"
                     reason = f"source_{v['verdict']}: {v.get('reason', '')}"
                 else:
+                    # content-staleness re-fetch succeeded -> verdict is authoritative
+                    status_change = "supported"
                     reason = "supported"
         if status_change is None:
             # proxy-free fallback: возраст контента