85 lines
3.6 KiB
Python
85 lines
3.6 KiB
Python
"""Цикл исследования пробелов: detect → plan → acquire → extract → verify → commit."""
|
|
from __future__ import annotations
|
|
import json
|
|
import httpx
|
|
from . import config, db, pipeline
|
|
|
|
_client = httpx.Client(timeout=600)
|
|
|
|
def detect_gaps(conn) -> list[dict]:
|
|
"""Найти пробелы в базе знаний."""
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"SELECT id, question, reason, priority, status FROM pm_research_gap "
|
|
"WHERE status = 'open' ORDER BY priority DESC, id LIMIT 20"
|
|
)
|
|
return cur.fetchall()
|
|
|
|
def add_gap(conn, question: str, reason: str = "user_query", priority: float = 0.5):
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"INSERT INTO pm_research_gap (question, reason, priority) VALUES (%s, %s, %s) RETURNING id",
|
|
(question, reason, priority),
|
|
)
|
|
conn.commit()
|
|
|
|
def web_search(query: str, engine: str = "local") -> list[dict]:
|
|
"""Веб-поиск (локальный инстанс). Возвращает [{title, url, snippet}]."""
|
|
# прямого MCP-инструмента searxng больше нет; internal HTTP JSON API остаётся
|
|
import urllib.parse
|
|
q = urllib.parse.quote(query)
|
|
# endpoint локального поискового инстанса
|
|
for base in ("http://127.0.0.1:8889", "http://127.0.0.1:8080"):
|
|
try:
|
|
r = _client.get(f"{base}/search", params={"q": query, "format": "json"}, timeout=15)
|
|
r.raise_for_status()
|
|
results = r.json().get("results", [])
|
|
return [{"title": x.get("title"), "url": x.get("url"), "snippet": x.get("content")}
|
|
for x in results[:3]]
|
|
except Exception:
|
|
continue
|
|
return []
|
|
|
|
def research_question(conn, question: str, *, top_k: int = 5, model: str | None = None) -> dict:
|
|
"""Полный цикл исследования одного вопроса.
|
|
|
|
1. веб-поиск;
|
|
2. для каждого источника: полная загрузка URL; при недоступности (гео-блок,
|
|
нет прокси) — дистилляция сниппета из поиска;
|
|
3. пометка вопроса resolved.
|
|
"""
|
|
result = {"question": question, "sources": [], "atoms_created": 0}
|
|
sources = web_search(question)
|
|
result["sources"] = [s["url"] for s in sources if s.get("url")]
|
|
|
|
for s in sources:
|
|
url = s.get("url")
|
|
if not url:
|
|
continue
|
|
# попытка полной загрузки
|
|
ingested = False
|
|
try:
|
|
stats = pipeline.ingest_url(conn, url, min_confidence=0.4)
|
|
result["atoms_created"] += stats["atoms_created"]
|
|
ingested = True
|
|
except Exception as e:
|
|
print(f"[research] ingest {url} failed: {e}")
|
|
# fallback: сниппет из поиска (когда сайт недоступен напрямую)
|
|
if not ingested and s.get("snippet"):
|
|
try:
|
|
title = s.get("title") or url
|
|
stats = pipeline.ingest_text(
|
|
conn, s["snippet"], title=title, uri=url, min_confidence=0.4,
|
|
)
|
|
result["atoms_created"] += stats["atoms_created"]
|
|
except Exception as e:
|
|
print(f"[research] snippet fallback {url} failed: {e}")
|
|
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"UPDATE pm_research_gap SET status = 'resolved' WHERE question = %s AND status = 'open'",
|
|
(question,),
|
|
)
|
|
conn.commit()
|
|
return result
|