Initial commit: Pegas Memory (PG + Qdrant + Memgraph, MCP server, 14 tools)

pegas_memory/research.py

@@ -0,0 +1,84 @@
+"""Цикл исследования пробелов: detect → plan → acquire → extract → verify → commit."""
+from __future__ import annotations
+import json
+import httpx
+from . import config, db, pipeline
+
+_client = httpx.Client(timeout=600)
+
+def detect_gaps(conn) -> list[dict]:
+    """Найти пробелы в базе знаний."""
+    with conn.cursor() as cur:
+        cur.execute(
+            "SELECT id, question, reason, priority, status FROM pm_research_gap "
+            "WHERE status = 'open' ORDER BY priority DESC, id LIMIT 20"
+        )
+        return cur.fetchall()
+
+def add_gap(conn, question: str, reason: str = "user_query", priority: float = 0.5):
+    with conn.cursor() as cur:
+        cur.execute(
+            "INSERT INTO pm_research_gap (question, reason, priority) VALUES (%s, %s, %s) RETURNING id",
+            (question, reason, priority),
+        )
+    conn.commit()
+
+def web_search(query: str, engine: str = "searxng") -> list[dict]:
+    """Поиск через SearXNG. Возвращает [{title, url, snippet}]."""
+    # локальный SearXNG (MCP mcp__searxng__search); здесь — прямой HTTP JSON API
+    import urllib.parse
+    q = urllib.parse.quote(query)
+    # типичный endpoint локального SearXNG
+    for base in ("http://127.0.0.1:8889", "http://127.0.0.1:8080"):
+        try:
+            r = _client.get(f"{base}/search", params={"q": query, "format": "json"}, timeout=15)
+            r.raise_for_status()
+            results = r.json().get("results", [])
+            return [{"title": x.get("title"), "url": x.get("url"), "snippet": x.get("content")}
+                    for x in results[:3]]
+        except Exception:
+            continue
+    return []
+
+def research_question(conn, question: str, *, top_k: int = 5, model: str | None = None) -> dict:
+    """Полный цикл исследования одного вопроса.
+
+    1. web-поиск (SearXNG);
+    2. для каждого источника: полная загрузка URL; при недоступности (гео-блок,
+       нет прокси) — дистилляция сниппета из поиска;
+    3. пометка вопроса resolved.
+    """
+    result = {"question": question, "sources": [], "atoms_created": 0}
+    sources = web_search(question)
+    result["sources"] = [s["url"] for s in sources if s.get("url")]
+
+    for s in sources:
+        url = s.get("url")
+        if not url:
+            continue
+        # попытка полной загрузки
+        ingested = False
+        try:
+            stats = pipeline.ingest_url(conn, url, min_confidence=0.4)
+            result["atoms_created"] += stats["atoms_created"]
+            ingested = True
+        except Exception as e:
+            print(f"[research] ingest {url} failed: {e}")
+        # fallback: сниппет из поиска (когда сайт недоступен напрямую)
+        if not ingested and s.get("snippet"):
+            try:
+                title = s.get("title") or url
+                stats = pipeline.ingest_text(
+                    conn, s["snippet"], title=title, uri=url, min_confidence=0.4,
+                )
+                result["atoms_created"] += stats["atoms_created"]
+            except Exception as e:
+                print(f"[research] snippet fallback {url} failed: {e}")
+
+    with conn.cursor() as cur:
+        cur.execute(
+            "UPDATE pm_research_gap SET status = 'resolved' WHERE question = %s AND status = 'open'",
+            (question,),
+        )
+    conn.commit()
+    return result