Initial commit: Pegas Memory (PG + Qdrant + Memgraph, MCP server, 14 tools)

pegas_memory/pipeline.py

@@ -0,0 +1,83 @@
+"""Ингест-пайплайн: источник → чанки → атомы → PG + outbox."""
+from __future__ import annotations
+import psycopg
+from . import db, distill, ingest
+from .ingest import ParsedDoc
+
+def ingest_doc(conn, doc: ParsedDoc, *, verify: bool = False,
+               min_confidence: float = 0.4, dedup: bool = True,
+               dedup_threshold: float = 0.93) -> dict:
+    """Прогнать документ через пайплайн. Возвращает статистику.
+
+    dedup: семантическая дедупликация по эмбеддингам (Qdrant) против
+    существующих атомов — вариации одного факта от LLM не дублируются.
+    """
+    doc_id = db.upsert_document(
+        conn, kind=doc.kind, uri=doc.uri, title=doc.title,
+        content_hash=doc.content_hash,
+    )
+    stats = {"doc_id": doc_id, "chunks": 0, "atoms_created": 0,
+             "atoms_skipped": 0, "duplicates": 0}
+
+    with conn.cursor() as cur:
+        for ch in doc.chunks:
+            cur.execute(
+                "INSERT INTO pm_chunk (doc_id, seq, content, locator) VALUES (%s, %s, %s, %s) RETURNING id",
+                (doc_id, ch.seq, ch.content, __import__("json").dumps(ch.locator, ensure_ascii=False)),
+            )
+            stats["chunks"] += 1
+
+            # дистилляция чанка в атомы
+            atoms = distill.distill_chunk(ch.content)
+            for a in atoms:
+                if a.get("confidence", 0.5) < min_confidence:
+                    stats["atoms_skipped"] += 1
+                    continue
+                # семантическая дедупликация
+                if dedup:
+                    from . import dedup as _dedup
+                    dup_id = _dedup.find_duplicate(
+                        (a.get("claim") or "") + " " + str(a["content"])[:800],
+                        threshold=dedup_threshold,
+                    )
+                    if dup_id is not None:
+                        stats["duplicates"] += 1
+                        stats["atoms_skipped"] += 1
+                        continue
+                atom_id, created = db.upsert_atom(
+                    conn,
+                    type=a["type"],
+                    claim=a.get("claim"),
+                    content=a["content"],
+                    confidence=a.get("confidence", 0.5),
+                    doc_id=doc_id,
+                    locator=ch.locator,
+                    metadata=a.get("metadata") or a.get("content_json"),
+                )
+                if created:
+                    stats["atoms_created"] += 1
+                    db.enqueue_outbox(conn, entity_type="atom", entity_id=atom_id)
+                else:
+                    stats["atoms_skipped"] += 1
+    conn.commit()
+    return stats
+
+def ingest_text(conn, text: str, *, title: str | None = None, uri: str | None = None,
+                **kw) -> dict:
+    doc = ingest.parse_text(text, title=title, uri=uri)
+    return ingest_doc(conn, doc, **kw)
+
+def ingest_file(conn, path: str, **kw) -> dict:
+    low = path.lower()
+    if low.endswith(".pdf"):
+        doc = ingest.parse_pdf(path)
+    elif low.endswith(".epub"):
+        doc = ingest.parse_epub(path)
+    else:
+        doc = ingest.parse_text(open(path, encoding="utf-8").read(),
+                                title=path.split("/")[-1], uri=path)
+    return ingest_doc(conn, doc, **kw)
+
+def ingest_url(conn, url: str, **kw) -> dict:
+    doc = ingest.parse_web(url)
+    return ingest_doc(conn, doc, **kw)