From a4c4a766c143668c7b938603c51c607837cade6f Mon Sep 17 00:00:00 2001 From: n1kt4r Date: Thu, 15 Oct 2026 17:47:40 +0500 Subject: [PATCH] Gaps: full lifecycle management (get/update/delete), pure PostgreSQL (no web search) --- AGENTS.md | 30 ++++++++ ARCHITECTURE.md | 13 ++-- README.md | 23 +++++- mcp_server.py | 121 ++++++++++++++++++++++++++--- pegas_memory/research.py | 160 +++++++++++++++++++++++---------------- 5 files changed, 266 insertions(+), 81 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 7e2ec9e..3be9aec 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -40,6 +40,10 @@ Read this file before answering any question about knowledge, memory, or researc | "Find a function/class named X" | `memory_search_code` | | "Show relations / graph edges" | `memory_query_graph` — `kind` is an **edge** type (`imports`/`depends_on`/`calls`/`inherits`/`related`), not an atom type | | "Remember to research X later" | `memory_add_gap` | +| "Edit a gap" (text/priority/reason) | `memory_update_gap` | +| "Close / take a gap" | `memory_update_gap` with `status` = `researching` / `resolved` / `dismissed` | +| "Show one gap" | `memory_get_gap` | +| "Delete a gap" | `memory_delete_gap` | | "Run logical inference" | `memory_infer` | **Ambiguity rule:** if the question is about *knowledge content* → memory tools. @@ -53,6 +57,7 @@ If it is about *the outside world right now* → `web_search` (built-in). 1. DETECT memory_stats / memory_list_gaps → what is missing? 2. CHECK memory_search → do I already know it? 3. PLAN memory_add_gap(question) → register the gap + TRACK memory_update_gap(id, status) → researching → resolved/dismissed 4. ACQUIRE web_search / web_fetch / read URL → find sources 5. EXTRACT memory_ingest(text|url|path) → distill into atoms 6. VERIFY memory_verify → candidate → verified/contested @@ -102,3 +107,28 @@ Every atom needs a **claim** (one short sentence) and, where possible, provenanc - If a fact is `candidate` (unverified), say so. - If the base has nothing, say "not in the knowledge base" and offer to research it. - Never present web content as memory, and never present memory as live web data. + +--- + +## 7. Managing gaps (full lifecycle) + +A **gap** is a question you know you cannot answer yet. Manage them, don't just collect them: + +| Operation | Tool | +|---|---| +| Create | `memory_add_gap(question, reason, priority)` | +| List (filter by status) | `memory_list_gaps(status?, limit?)` | +| Read one | `memory_get_gap(gap_id)` | +| Edit text / reason / priority / status | `memory_update_gap(gap_id, ...)` | +| Delete | `memory_delete_gap(gap_id)` | + +**Statuses:** `open` → `researching` → `resolved` | `dismissed` + +**Reasons:** `low_coverage` · `low_confidence` · `stale` · `contested` · `user_query` + +**Rules:** +- When you start working on a gap → set `status='researching'`. +- When the knowledge is in the base → set `status='resolved'`. +- If the question turned out irrelevant → `status='dismissed'` (prefer this over deleting). +- Raise `priority` when the user asks about the topic; lower it when it stops mattering. +- Keep questions **specific and answerable** ("How does HNSW build its graph?" not "HNSW"). diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index c44f16a..0fbba7d 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -1,7 +1,7 @@ # Pegas Memory — Architecture Pegas Memory is the long-term memory and knowledge base for the **Pegas** research agent. -It is a Python service exposed to the agent as an **MCP server** (13 tools). +It is a Python service exposed to the agent as an **MCP server** (16 tools). --- @@ -14,7 +14,7 @@ It is a Python service exposed to the agent as an **MCP server** (13 tools). └───────────────────┬──────────────────────┘ │ JSON-RPC 2.0 / stdio ┌───────────────────▼──────────────────────┐ - │ mcp_server.py (tool layer, 13 tools) │ + │ mcp_server.py (tool layer, 16 tools) │ └───────────────────┬──────────────────────┘ │ Python API ┌────────────────────────────┼────────────────────────────┐ @@ -96,7 +96,7 @@ agent's experience, atoms are curated facts. | `outbox_sync.py` | CDC: PostgreSQL outbox → Qdrant and Memgraph | | `dedup.py` | semantic deduplication by embedding cosine (threshold 0.90) | | `verify.py` | self-critique (LLM verdict) + cross-confirmation (multiple sources) | -| `research.py` | gap-filling loop: web search → ingest → commit | +| `research.py` | research-gap management (PostgreSQL only): add / list / get / update / delete | | `code_symbols.py` | tree-sitter symbol extraction | | `code_graph.py` | tree-sitter dependency extraction (calls/imports/inherits) | | `inference.py` | forward-chaining logical rules over the graph until fixpoint | @@ -153,7 +153,7 @@ Only single-token kinds (no spaces) are accepted — human-readable rules from d --- -## 6. MCP tools (13) +## 6. MCP tools (16) | Tool | Purpose | |---|---| @@ -164,7 +164,10 @@ Only single-token kinds (no spaces) are accepted — human-readable rules from d | `memory_ingest` | ingest text/URL/file → atoms | | `memory_verify` | self-critique + cross-confirmation | | `memory_add_gap` | register a research gap | -| `memory_list_gaps` | list open gaps | +| `memory_list_gaps` | list gaps (optional status filter) | +| `memory_get_gap` | read one gap by id | +| `memory_update_gap` | edit question / reason / priority / status | +| `memory_delete_gap` | delete a gap | | `memory_store_code` | extract + store code symbols | | `memory_search_code` | lexical search over code symbols | | `memory_code_graph` | build symbol + dependency graph | diff --git a/README.md b/README.md index 33e7db6..83a66c3 100644 --- a/README.md +++ b/README.md @@ -42,8 +42,10 @@ schema.sql — DDL PostgreSQL - **Сервер**: [mcp_server.py](./mcp_server.py) — чистый stdlib, JSON-RPC 2.0, stdio. - **Инструменты** (видны модели как `mcp__pegas-memory__*`): - `memory_search`, `memory_store`, `memory_ingest`, `memory_add_gap`, - `memory_list_gaps`, `memory_stats`, `memory_get_context`. + `memory_search`, `memory_store`, `memory_ingest`, `memory_stats`, `memory_get_context`, + `memory_verify`, `memory_store_code`, `memory_search_code`, `memory_code_graph`, + `memory_infer`, `memory_query_graph`, + `memory_add_gap`, `memory_list_gaps`, `memory_get_gap`, `memory_update_gap`, `memory_delete_gap`. - **Регистрация**: в `~/.dsh/profiles/web/cordis.patch.yml` через `@deepseek-ai/dsh-mcp-client`. - **Live-reload**: профиль web имеет `patchReload: live` — изменения конфига подхватываются без рестарта (проверено: сервер спавнится автоматически). @@ -75,3 +77,20 @@ export PROXY_URL="socks5://127.0.0.1:1080" # hysteria2 ``` Когда прокси задан, `parse_web` ходит через него; иначе — напрямую. Поддержка SOCKS5 уже установлена (`httpx[socks]`). + +## Управление пробелами (gaps) + +Пробел — вопрос, на который в базе пока нет ответа. Полный жизненный цикл: + +| Операция | Инструмент | +|---|---| +| Создать | `memory_add_gap(question, reason, priority)` | +| Список (с фильтром) | `memory_list_gaps(status?, limit?)` | +| Прочитать один | `memory_get_gap(gap_id)` | +| Редактировать | `memory_update_gap(gap_id, question?/reason?/priority?/status?)` | +| Удалить | `memory_delete_gap(gap_id)` | + +Статусы: `open` → `researching` → `resolved` | `dismissed`. +Причины: `low_coverage`, `low_confidence`, `stale`, `contested`, `user_query`. + +Работает целиком на PostgreSQL — без веб-поиска. Поиск выполняет модель встроенным `web_search`. diff --git a/mcp_server.py b/mcp_server.py index e061dff..a2550ac 100644 --- a/mcp_server.py +++ b/mcp_server.py @@ -35,6 +35,13 @@ def _conn(): def dict_row(cursor): + """Row factory: dict по именам колонок. + + cursor.description может быть None (UPDATE/DELETE без RETURNING) — + в этом случае возвращаем строку как есть, иначе падаем на итерации None. + """ + if cursor.description is None: + return lambda row: row cols = [d.name for d in cursor.description] return lambda row: dict(zip(cols, row)) @@ -114,16 +121,67 @@ def _tool_add_gap(args): question = str(args.get("question") or "").strip() if not question: return json.dumps({"error": "пустой question"}, ensure_ascii=False), True - research.add_gap(_conn(), question, - reason=str(args.get("reason") or "user_query"), - priority=float(args.get("priority") or 0.5)) - return json.dumps({"added": question}, ensure_ascii=False), False + try: + gap_id = research.add_gap(_conn(), question, + reason=str(args.get("reason") or "user_query"), + priority=float(args.get("priority") or 0.5)) + except ValueError as e: + return json.dumps({"error": str(e)}, ensure_ascii=False), True + return json.dumps({"gap_id": gap_id, "question": question, "status": "open"}, + ensure_ascii=False), False def _tool_list_gaps(args): from pegas_memory import research - gaps = research.detect_gaps(_conn()) - return json.dumps({"gaps": gaps}, ensure_ascii=False, indent=2), False + status = args.get("status") or None + limit = int(args.get("limit") or 50) + try: + gaps = research.list_gaps(_conn(), status=status, limit=limit) + except ValueError as e: + return json.dumps({"error": str(e)}, ensure_ascii=False), True + return json.dumps({"status": status or "all", "count": len(gaps), "gaps": gaps}, + ensure_ascii=False, indent=2, default=str), False + + +def _tool_get_gap(args): + from pegas_memory import research + gap_id = args.get("gap_id") + if gap_id is None: + return json.dumps({"error": "нужен gap_id"}, ensure_ascii=False), True + gap = research.get_gap(_conn(), int(gap_id)) + if gap is None: + return json.dumps({"error": f"пробел {gap_id} не найден"}, ensure_ascii=False), True + return json.dumps(gap, ensure_ascii=False, indent=2, default=str), False + + +def _tool_update_gap(args): + from pegas_memory import research + gap_id = args.get("gap_id") + if gap_id is None: + return json.dumps({"error": "нужен gap_id"}, ensure_ascii=False), True + fields = {k: args[k] for k in ("question", "reason", "priority", "status") + if args.get(k) is not None} + if not fields: + return json.dumps({"error": "нечего обновлять (укажи question/reason/priority/status)"}, + ensure_ascii=False), True + try: + ok = research.update_gap(_conn(), int(gap_id), **fields) + except ValueError as e: + return json.dumps({"error": str(e)}, ensure_ascii=False), True + if not ok: + return json.dumps({"error": f"пробел {gap_id} не найден"}, ensure_ascii=False), True + return json.dumps({"updated": int(gap_id), "gap": research.get_gap(_conn(), int(gap_id))}, + ensure_ascii=False, indent=2, default=str), False + + +def _tool_delete_gap(args): + from pegas_memory import research + gap_id = args.get("gap_id") + if gap_id is None: + return json.dumps({"error": "нужен gap_id"}, ensure_ascii=False), True + if not research.delete_gap(_conn(), int(gap_id)): + return json.dumps({"error": f"пробел {gap_id} не найден"}, ensure_ascii=False), True + return json.dumps({"deleted": int(gap_id)}, ensure_ascii=False), False def _tool_stats(args): @@ -296,9 +354,51 @@ TOOLS = [ }, { "name": "memory_list_gaps", - "description": "Список открытых пробелов (вопросов, на которые пока нет ответа) в базе, " - "отсортированный по приоритету. Вызывай, чтобы узнать, что ещё нужно исследовать.", - "inputSchema": {"type": "object", "properties": {}, "required": []}, + "description": "Список пробелов (вопросов без ответа). По умолчанию — открытые, по приоритету. " + "Вызывай, чтобы узнать, что ещё нужно исследовать.", + "inputSchema": { + "type": "object", + "properties": { + "status": {"type": "string", "description": "Фильтр: open | researching | resolved | dismissed (опционально)."}, + "limit": {"type": "integer", "default": 50, "description": "Сколько вернуть (1-200)."}, + }, + "required": [], + }, + }, + { + "name": "memory_get_gap", + "description": "Получить один пробел по id (все поля).", + "inputSchema": { + "type": "object", + "properties": {"gap_id": {"type": "integer", "description": "ID пробела."}}, + "required": ["gap_id"], + }, + }, + { + "name": "memory_update_gap", + "description": "Отредактировать пробел: текст вопроса, причину, приоритет и/или статус. " + "Статусы: open (открыт) | researching (взят в работу) | resolved (закрыт) | dismissed (не актуален).", + "inputSchema": { + "type": "object", + "properties": { + "gap_id": {"type": "integer", "description": "ID пробела."}, + "question": {"type": "string", "description": "Новая формулировка вопроса."}, + "reason": {"type": "string", "description": "low_coverage|low_confidence|stale|contested|user_query."}, + "priority": {"type": "number", "description": "Новый приоритет 0-1."}, + "status": {"type": "string", "description": "open | researching | resolved | dismissed."}, + }, + "required": ["gap_id"], + }, + }, + { + "name": "memory_delete_gap", + "description": "Удалить пробел по id (если он больше не нужен). " + "Обычно вместо удаления лучше перевести в status=dismissed или resolved.", + "inputSchema": { + "type": "object", + "properties": {"gap_id": {"type": "integer", "description": "ID пробела."}}, + "required": ["gap_id"], + }, }, { "name": "memory_stats", @@ -441,6 +541,9 @@ def handle(msg): "memory_ingest": _tool_ingest, "memory_add_gap": _tool_add_gap, "memory_list_gaps": _tool_list_gaps, + "memory_get_gap": _tool_get_gap, + "memory_update_gap": _tool_update_gap, + "memory_delete_gap": _tool_delete_gap, "memory_stats": _tool_stats, "memory_store_code": _tool_store_code, "memory_search_code": _tool_search_code, diff --git a/pegas_memory/research.py b/pegas_memory/research.py index 64f6e46..6b0a664 100644 --- a/pegas_memory/research.py +++ b/pegas_memory/research.py @@ -1,84 +1,114 @@ -"""Цикл исследования пробелов: detect → plan → acquire → extract → verify → commit.""" +"""Управление пробелами в знаниях (research gaps). + +Работает **только с PostgreSQL** — никакого веб-поиска. Поиск выполняет сама +модель встроенным инструментом \`web_search\` (SearXNG и автоматический +research-цикл убраны). + +Жизненный цикл пробела: + open → researching → resolved | dismissed + +Поля: + question — что нужно исследовать (формулировка) + reason — почему пробел обнаружен + priority — 0.0 .. 1.0 (больше = важнее) + status — open | researching | resolved | dismissed +""" from __future__ import annotations -import json -import httpx -from . import config, db, pipeline -_client = httpx.Client(timeout=600) +VALID_REASONS = ("low_coverage", "low_confidence", "stale", "contested", "user_query") +VALID_STATUSES = ("open", "researching", "resolved", "dismissed") -def detect_gaps(conn) -> list[dict]: - """Найти пробелы в базе знаний.""" +_COLS = "id, question, reason, priority, status, created_at" + + +def list_gaps(conn, *, status: str | None = None, limit: int = 50) -> list[dict]: + """Список пробелов. Без фильтра — сначала открытые, затем по приоритету.""" + limit = max(1, min(int(limit), 200)) with conn.cursor() as cur: - cur.execute( - "SELECT id, question, reason, priority, status FROM pm_research_gap " - "WHERE status = 'open' ORDER BY priority DESC, id LIMIT 20" - ) + if status: + if status not in VALID_STATUSES: + raise ValueError(f"status must be one of {VALID_STATUSES}") + cur.execute( + f"SELECT {_COLS} FROM pm_research_gap WHERE status = %s " + "ORDER BY priority DESC, id LIMIT %s", + (status, limit), + ) + else: + cur.execute( + f"SELECT {_COLS} FROM pm_research_gap " + "ORDER BY (status = 'open') DESC, priority DESC, id LIMIT %s", + (limit,), + ) return cur.fetchall() -def add_gap(conn, question: str, reason: str = "user_query", priority: float = 0.5): + +def get_gap(conn, gap_id: int) -> dict | None: + """Один пробел по id (или None).""" + with conn.cursor() as cur: + cur.execute(f"SELECT {_COLS} FROM pm_research_gap WHERE id = %s", (gap_id,)) + return cur.fetchone() + + +def add_gap(conn, question: str, reason: str = "user_query", priority: float = 0.5) -> int: + """Зарегистрировать пробел. Возвращает id.""" + question = (question or "").strip() + if not question: + raise ValueError("question must not be empty") + if reason not in VALID_REASONS: + raise ValueError(f"reason must be one of {VALID_REASONS}") + priority = max(0.0, min(1.0, float(priority))) with conn.cursor() as cur: cur.execute( - "INSERT INTO pm_research_gap (question, reason, priority) VALUES (%s, %s, %s) RETURNING id", + "INSERT INTO pm_research_gap (question, reason, priority, status) " + "VALUES (%s, %s, %s, 'open') RETURNING id", (question, reason, priority), ) + row = cur.fetchone() conn.commit() + return row["id"] -def web_search(query: str, engine: str = "local") -> list[dict]: - """Веб-поиск (локальный инстанс). Возвращает [{title, url, snippet}].""" - # прямого MCP-инструмента searxng больше нет; internal HTTP JSON API остаётся - import urllib.parse - q = urllib.parse.quote(query) - # endpoint локального поискового инстанса - for base in ("http://127.0.0.1:8889", "http://127.0.0.1:8080"): - try: - r = _client.get(f"{base}/search", params={"q": query, "format": "json"}, timeout=15) - r.raise_for_status() - results = r.json().get("results", []) - return [{"title": x.get("title"), "url": x.get("url"), "snippet": x.get("content")} - for x in results[:3]] - except Exception: - continue - return [] -def research_question(conn, question: str, *, top_k: int = 5, model: str | None = None) -> dict: - """Полный цикл исследования одного вопроса. +def update_gap(conn, gap_id: int, *, question: str | None = None, + reason: str | None = None, priority: float | None = None, + status: str | None = None) -> bool: + """Обновить поля пробела. Возвращает True, если запись найдена. - 1. веб-поиск; - 2. для каждого источника: полная загрузка URL; при недоступности (гео-блок, - нет прокси) — дистилляция сниппета из поиска; - 3. пометка вопроса resolved. + Типичные переходы: status='researching' (взял в работу), + status='resolved' (закрыт), status='dismissed' (не актуален). """ - result = {"question": question, "sources": [], "atoms_created": 0} - sources = web_search(question) - result["sources"] = [s["url"] for s in sources if s.get("url")] + sets: list[str] = [] + vals: list = [] + if question is not None: + question = question.strip() + if not question: + raise ValueError("question must not be empty") + sets.append("question = %s"); vals.append(question) + if reason is not None: + if reason not in VALID_REASONS: + raise ValueError(f"reason must be one of {VALID_REASONS}") + sets.append("reason = %s"); vals.append(reason) + if priority is not None: + sets.append("priority = %s"); vals.append(max(0.0, min(1.0, float(priority)))) + if status is not None: + if status not in VALID_STATUSES: + raise ValueError(f"status must be one of {VALID_STATUSES}") + sets.append("status = %s"); vals.append(status) - for s in sources: - url = s.get("url") - if not url: - continue - # попытка полной загрузки - ingested = False - try: - stats = pipeline.ingest_url(conn, url, min_confidence=0.4) - result["atoms_created"] += stats["atoms_created"] - ingested = True - except Exception as e: - print(f"[research] ingest {url} failed: {e}") - # fallback: сниппет из поиска (когда сайт недоступен напрямую) - if not ingested and s.get("snippet"): - try: - title = s.get("title") or url - stats = pipeline.ingest_text( - conn, s["snippet"], title=title, uri=url, min_confidence=0.4, - ) - result["atoms_created"] += stats["atoms_created"] - except Exception as e: - print(f"[research] snippet fallback {url} failed: {e}") + if not sets: + return get_gap(conn, gap_id) is not None + vals.append(gap_id) with conn.cursor() as cur: - cur.execute( - "UPDATE pm_research_gap SET status = 'resolved' WHERE question = %s AND status = 'open'", - (question,), - ) + cur.execute(f"UPDATE pm_research_gap SET {', '.join(sets)} WHERE id = %s", vals) + found = cur.rowcount > 0 conn.commit() - return result + return found + + +def delete_gap(conn, gap_id: int) -> bool: + """Удалить пробел. Возвращает True, если запись существовала.""" + with conn.cursor() as cur: + cur.execute("DELETE FROM pm_research_gap WHERE id = %s", (gap_id,)) + found = cur.rowcount > 0 + conn.commit() + return found