84 lines
3.5 KiB
Python
84 lines
3.5 KiB
Python
"""Ингест-пайплайн: источник → чанки → атомы → PG + outbox."""
|
|
from __future__ import annotations
|
|
import psycopg
|
|
from . import db, distill, ingest
|
|
from .ingest import ParsedDoc
|
|
|
|
def ingest_doc(conn, doc: ParsedDoc, *, verify: bool = False,
|
|
min_confidence: float = 0.4, dedup: bool = True,
|
|
dedup_threshold: float = 0.93) -> dict:
|
|
"""Прогнать документ через пайплайн. Возвращает статистику.
|
|
|
|
dedup: семантическая дедупликация по эмбеддингам (Qdrant) против
|
|
существующих атомов — вариации одного факта от LLM не дублируются.
|
|
"""
|
|
doc_id = db.upsert_document(
|
|
conn, kind=doc.kind, uri=doc.uri, title=doc.title,
|
|
content_hash=doc.content_hash,
|
|
)
|
|
stats = {"doc_id": doc_id, "chunks": 0, "atoms_created": 0,
|
|
"atoms_skipped": 0, "duplicates": 0}
|
|
|
|
with conn.cursor() as cur:
|
|
for ch in doc.chunks:
|
|
cur.execute(
|
|
"INSERT INTO pm_chunk (doc_id, seq, content, locator) VALUES (%s, %s, %s, %s) RETURNING id",
|
|
(doc_id, ch.seq, ch.content, __import__("json").dumps(ch.locator, ensure_ascii=False)),
|
|
)
|
|
stats["chunks"] += 1
|
|
|
|
# дистилляция чанка в атомы
|
|
atoms = distill.distill_chunk(ch.content)
|
|
for a in atoms:
|
|
if a.get("confidence", 0.5) < min_confidence:
|
|
stats["atoms_skipped"] += 1
|
|
continue
|
|
# семантическая дедупликация
|
|
if dedup:
|
|
from . import dedup as _dedup
|
|
dup_id = _dedup.find_duplicate(
|
|
(a.get("claim") or "") + " " + str(a["content"])[:800],
|
|
threshold=dedup_threshold,
|
|
)
|
|
if dup_id is not None:
|
|
stats["duplicates"] += 1
|
|
stats["atoms_skipped"] += 1
|
|
continue
|
|
atom_id, created = db.upsert_atom(
|
|
conn,
|
|
type=a["type"],
|
|
claim=a.get("claim"),
|
|
content=a["content"],
|
|
confidence=a.get("confidence", 0.5),
|
|
doc_id=doc_id,
|
|
locator=ch.locator,
|
|
metadata=a.get("metadata") or a.get("content_json"),
|
|
)
|
|
if created:
|
|
stats["atoms_created"] += 1
|
|
db.enqueue_outbox(conn, entity_type="atom", entity_id=atom_id)
|
|
else:
|
|
stats["atoms_skipped"] += 1
|
|
conn.commit()
|
|
return stats
|
|
|
|
def ingest_text(conn, text: str, *, title: str | None = None, uri: str | None = None,
|
|
**kw) -> dict:
|
|
doc = ingest.parse_text(text, title=title, uri=uri)
|
|
return ingest_doc(conn, doc, **kw)
|
|
|
|
def ingest_file(conn, path: str, **kw) -> dict:
|
|
low = path.lower()
|
|
if low.endswith(".pdf"):
|
|
doc = ingest.parse_pdf(path)
|
|
elif low.endswith(".epub"):
|
|
doc = ingest.parse_epub(path)
|
|
else:
|
|
doc = ingest.parse_text(open(path, encoding="utf-8").read(),
|
|
title=path.split("/")[-1], uri=path)
|
|
return ingest_doc(conn, doc, **kw)
|
|
|
|
def ingest_url(conn, url: str, **kw) -> dict:
|
|
doc = ingest.parse_web(url)
|
|
return ingest_doc(conn, doc, **kw)
|