stage4: graph_outbox на AGE (Cypher, fixed password+syntax)
This commit is contained in:
neo committed
1 parent
cae85627fe
commit
a93aa24b54
4 files changed
+469
-23
No files matched your search
@@ -0,0 +1,167 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MVP stage 3: compositions — сборка выходов из атомов.
|
||||
|
||||
Собирает composition (пост/рассылку) из атомов по рецепту:
|
||||
- hook (цепляющий заголовок) → stat (доказательство) → cta (призыв к действию)
|
||||
- порядок атомов задаётся recipe (JSONB)
|
||||
|
||||
Запись в compositions + atom_ids.
|
||||
|
||||
Запуск:
|
||||
python3 stage3/compositions.py --channel telegram --project rabota-ufa
|
||||
python3 stage3/compositions.py --channel newsletter --project rabota-ufa
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
|
||||
import psycopg2
|
||||
|
||||
POSTGRES = {
|
||||
"host": os.environ.get("POSTGRES_HOST", "127.0.0.1"),
|
||||
"port": int(os.environ.get("POSTGRES_PORT", "5434")),
|
||||
"user": os.environ.get("POSTGRES_USER", "atom"),
|
||||
"dbname": os.environ.get("POSTGRES_DB", "content_atom"),
|
||||
"password": os.environ.get("POSTGRES_PASSWORD", "atompass123"),
|
||||
}
|
||||
|
||||
|
||||
def get_conn():
|
||||
return psycopg2.connect(**POSTGRES)
|
||||
|
||||
|
||||
def get_project_id(conn, project: str) -> str:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id FROM projects WHERE slug=%s", (project,))
|
||||
row = cur.fetchone()
|
||||
if row:
|
||||
return row[0]
|
||||
cur.execute(
|
||||
"INSERT INTO projects (slug, name, kind, role, status) VALUES (%s, %s, 'site', 'satellite', 'active') RETURNING id",
|
||||
(project, project),
|
||||
)
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def get_atoms(conn, project_id: str) -> list[dict]:
|
||||
"""Атомы по типам для сборки (hook → stat → cta)."""
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
SELECT id, type, plain_text, word_count
|
||||
FROM atoms
|
||||
WHERE project_id = %s AND plain_text IS NOT NULL
|
||||
ORDER BY created_at
|
||||
""", (project_id,))
|
||||
cols = [d[0] for d in cur.description]
|
||||
return [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
|
||||
|
||||
def build_composition(atoms: list[dict], channel: str, project: str) -> dict:
|
||||
"""Сборка: hook → stat → cta (в порядке recipe)."""
|
||||
# recipe: порядок типов
|
||||
recipe_order = ["hook", "stat", "comparison", "cta", "claim"]
|
||||
|
||||
ordered = []
|
||||
for t in recipe_order:
|
||||
for atom in atoms:
|
||||
if atom["type"] == t:
|
||||
ordered.append(atom)
|
||||
# дополняем оставшимися
|
||||
for atom in atoms:
|
||||
if atom["type"] not in recipe_order:
|
||||
ordered.append(atom)
|
||||
|
||||
atom_ids = [str(a["id"]) for a in ordered]
|
||||
|
||||
# текст композиции
|
||||
if channel == "telegram":
|
||||
content = _build_telegram_post(ordered)
|
||||
else:
|
||||
content = _build_newsletter(ordered)
|
||||
|
||||
return {
|
||||
"slug": f"{project}_{channel}",
|
||||
"title": f"{channel} — {project}",
|
||||
"recipe": {"order": recipe_order, "types": list({a["type"] for a in ordered})},
|
||||
"atom_ids": atom_ids,
|
||||
"content": content,
|
||||
}
|
||||
|
||||
|
||||
def _build_telegram_post(atoms: list[dict]) -> str:
|
||||
"""Telegram-пост: hook + stat + cta."""
|
||||
lines = []
|
||||
for atom in atoms[:8]: # не более 8 атомов в пост
|
||||
text = atom["plain_text"]
|
||||
if len(text) > 200:
|
||||
text = text[:200].rstrip() + "…"
|
||||
t = atom["type"]
|
||||
if t == "hook":
|
||||
lines.append(f"🔥 {text}")
|
||||
elif t == "stat":
|
||||
lines.append(f"📊 {text}")
|
||||
elif t == "cta":
|
||||
lines.append(f"👉 {text}")
|
||||
else:
|
||||
lines.append(text)
|
||||
return "\n\n".join(lines)
|
||||
|
||||
|
||||
def _build_newsletter(atoms: list[dict]) -> str:
|
||||
"""Рассылка: hook (заголовок) + body."""
|
||||
lines = []
|
||||
for atom in atoms[:10]:
|
||||
text = atom["plain_text"]
|
||||
if len(text) > 300:
|
||||
text = text[:300].rstrip() + "…"
|
||||
lines.append(text)
|
||||
return "\n\n".join(lines)
|
||||
|
||||
|
||||
def insert_composition(conn, project_id: str, comp: dict, channel: str) -> str:
|
||||
"""Вставить composition. Возвращает composition_id."""
|
||||
composition_id = str(uuid.uuid4())
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
INSERT INTO compositions
|
||||
(id, project_id, channel, slug, title, status, recipe, atom_ids, content)
|
||||
VALUES (%s, %s, %s, %s, %s, %s, %s, %s::uuid[], %s)
|
||||
ON CONFLICT (project_id, channel, slug) DO UPDATE
|
||||
SET content = EXCLUDED.content, atom_ids = EXCLUDED.atom_ids
|
||||
""", (
|
||||
composition_id, project_id, channel, comp["slug"], comp["title"],
|
||||
"ready", json.dumps(comp["recipe"]), comp["atom_ids"], comp["content"],
|
||||
))
|
||||
conn.commit()
|
||||
return composition_id
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="MVP stage 3: compositions")
|
||||
ap.add_argument("--channel", default="telegram", help="канал")
|
||||
ap.add_argument("--project", default="rabota-ufa", help="project slug")
|
||||
args = ap.parse_args()
|
||||
|
||||
conn = get_conn()
|
||||
project_id = get_project_id(conn, args.project)
|
||||
atoms = get_atoms(conn, project_id)
|
||||
print(f" • {len(atoms)} атомов для {args.channel}")
|
||||
|
||||
start = time.time()
|
||||
comp = build_composition(atoms, args.channel, args.project)
|
||||
comp_id = insert_composition(conn, project_id, comp, args.channel)
|
||||
conn.close()
|
||||
|
||||
print(f" ✓ composition: {comp['slug']} за {time.time()-start:.1f}с")
|
||||
print(f" • atom_ids: {len(comp['atom_ids'])}")
|
||||
print(f"\nMVP stage 3 compositions OK: {comp['slug']} → {comp_id}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
@@ -0,0 +1,114 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
MVP stage 3: cost tracking — стоимость на атом + runs.
|
||||
|
||||
Считает cost/quality для каждого запуска (runs) и cost на атом (atoms.cost).
|
||||
На основе эмбеддингов (bge-m3) и atom_versions (compositions).
|
||||
|
||||
Запуск:
|
||||
python3 stage3/cost_tracking.py --project rabota-ufa
|
||||
python3 stage3/cost_tracking.py --all
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import psycopg2
|
||||
|
||||
POSTGRES = {
|
||||
"host": os.environ.get("POSTGRES_HOST", "127.0.0.1"),
|
||||
"port": int(os.environ.get("POSTGRES_PORT", "5434")),
|
||||
"user": os.environ.get("POSTGRES_USER", "atom"),
|
||||
"dbname": os.environ.get("POSTGRES_DB", "content_atom"),
|
||||
"password": os.environ.get("POSTGRES_PASSWORD", "atompass123"),
|
||||
}
|
||||
|
||||
# Стоимость bge-m3 в Ollama: ~0 за локальный запуск, но учитываем CPU-нагрузку
|
||||
# Для юнит-экономики: embed ~0.0001 USD/atom, version ~0.0002 USD/atom
|
||||
COST_PER_ATOM = 0.0001
|
||||
COST_PER_VERSION = 0.0002
|
||||
|
||||
|
||||
def get_conn():
|
||||
return psycopg2.connect(**POSTGRES)
|
||||
|
||||
|
||||
def get_projects(conn) -> list[dict]:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id, slug FROM projects")
|
||||
cols = [d[0] for d in cur.description]
|
||||
return [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
|
||||
|
||||
def get_atoms(conn, project_id: str) -> list[dict]:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT id, plain_text FROM atoms WHERE project_id=%s", (project_id,))
|
||||
cols = [d[0] for d in cur.description]
|
||||
return [dict(zip(cols, row)) for row in cur.fetchall()]
|
||||
|
||||
|
||||
def get_versions(conn, atom_id: str) -> int:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("SELECT count(*) FROM atom_versions WHERE atom_id=%s", (atom_id,))
|
||||
return cur.fetchone()[0]
|
||||
|
||||
|
||||
def track_runs(conn, project_id: str, project_slug: str) -> dict:
|
||||
"""Отслеживает cost на атомах проекта."""
|
||||
atoms = get_atoms(conn, project_id)
|
||||
n_atoms = len(atoms)
|
||||
n_versions = 0
|
||||
|
||||
start = time.time()
|
||||
for atom in atoms:
|
||||
n_versions += get_versions(conn, atom["id"])
|
||||
version_time = time.time() - start
|
||||
|
||||
# runs: один запуск на проект
|
||||
with conn.cursor() as cur:
|
||||
cur.execute("""
|
||||
INSERT INTO jobs (kind, status, finished_at)
|
||||
VALUES ('index', 'done', now()) RETURNING id
|
||||
""")
|
||||
job_id = cur.fetchone()[0]
|
||||
cur.execute("""
|
||||
INSERT INTO runs (job_id, stage, model, tokens_in, tokens_out, seconds, cost_usd, quality)
|
||||
VALUES (%s, 'embed', 'bge-m3', %s, %s, %s, %s, 7.0)
|
||||
""", (job_id, n_atoms, n_versions, round(version_time, 3), round(n_atoms * COST_PER_ATOM + n_versions * COST_PER_VERSION, 6)))
|
||||
cur.execute("""
|
||||
UPDATE atoms SET cost = %s WHERE project_id=%s
|
||||
""", (json.dumps({"cost_usd": round(n_atoms * COST_PER_ATOM, 6),
|
||||
"embed_seconds": round(version_time, 3)}), project_id))
|
||||
conn.commit()
|
||||
|
||||
return {"atoms": n_atoms, "versions": n_versions, "cost_usd": round(n_atoms * COST_PER_ATOM, 6), "seconds": round(version_time, 3)}
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser(description="MVP stage 3: cost tracking")
|
||||
ap.add_argument("--project", default=None, help="project slug")
|
||||
ap.add_argument("--all", dest="all", action="store_true", help="все проекты")
|
||||
args = ap.parse_args()
|
||||
|
||||
conn = get_conn()
|
||||
projects = get_projects(conn)
|
||||
|
||||
total_cost = 0.0
|
||||
for proj in projects:
|
||||
if args.project and proj["slug"] != args.project:
|
||||
continue
|
||||
result = track_runs(conn, proj["id"], proj["slug"])
|
||||
print(f" • {proj['slug']}: {result['atoms']} атомов, {result['versions']} версий, ${result['cost_usd']}")
|
||||
total_cost += result["cost_usd"]
|
||||
|
||||
conn.close()
|
||||
print(f"\nMVP stage 3 cost tracking OK: ${total_cost:.6f} за {len(projects)} проектов")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
sys.exit(main())
|
||||
Reference in new issue
Block a user