425 lines
16 KiB
Python
Executable File
425 lines
16 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""
|
||
Perplexity-style search agent для Hermes (async).
|
||
|
||
Использование:
|
||
python3 perplexity_search.py "запрос"
|
||
python3 perplexity_search.py "запрос" --category news --results 5
|
||
python3 perplexity_search.py "запрос" --raw # только JSON
|
||
|
||
Опции:
|
||
--category Категория SearXNG: general (default), news, it, science
|
||
--results Сколько результатов fetch'ить (default: 3, макс: 5)
|
||
--raw Только сбор данных, JSON на stdout (без LLM-синтеза)
|
||
--model Имя LLM модели (env: LLM_MODEL)
|
||
--temperature Temperature (env: LLM_TEMPERATURE)
|
||
"""
|
||
|
||
import argparse
|
||
import asyncio
|
||
import json
|
||
import os
|
||
import random
|
||
import re
|
||
import subprocess
|
||
import sys
|
||
import time
|
||
import urllib.parse
|
||
from collections import defaultdict
|
||
from datetime import datetime
|
||
|
||
import aiohttp
|
||
from bs4 import BeautifulSoup
|
||
|
||
# ─── Конфигурация ─────────────────────────────────────────────────────────────
|
||
|
||
SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://localhost:8888")
|
||
LLM_URL = os.environ.get("LLM_URL", "http://192.168.1.152:8080/v1/chat/completions")
|
||
LLM_MODEL = os.environ.get("LLM_MODEL", "Qwen3.6-35B-A3B-UD-IQ4_NL.gguf")
|
||
LLM_TEMPERATURE = float(os.environ.get("LLM_TEMPERATURE", "0.7"))
|
||
MAX_ARTICLE_CHARS = 5000
|
||
LLM_TIMEOUT = 120
|
||
|
||
# ─── Anti-blocking: User-Agent ротация ────────────────────────────────────────
|
||
|
||
USER_AGENTS = [
|
||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15",
|
||
"Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
|
||
"Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||
]
|
||
|
||
|
||
def random_ua():
|
||
return random.choice(USER_AGENTS)
|
||
|
||
|
||
def build_headers(referer=None):
|
||
h = {
|
||
"User-Agent": random_ua(),
|
||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||
"Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
||
"Accept-Encoding": "gzip, deflate",
|
||
"DNT": "1",
|
||
"Connection": "keep-alive",
|
||
}
|
||
if referer:
|
||
h["Referer"] = referer
|
||
return h
|
||
|
||
|
||
# ─── Anti-blocking: Domain Rate Limiter ────────────────────────────────────────
|
||
|
||
|
||
class DomainRateLimiter:
|
||
def __init__(self, min_interval=1.5, jitter=0.5):
|
||
self._last_request = defaultdict(float)
|
||
self._min_interval = min_interval
|
||
self._jitter = jitter
|
||
|
||
async def acquire(self, domain):
|
||
elapsed = time.monotonic() - self._last_request[domain]
|
||
base_wait = max(0, self._min_interval - elapsed)
|
||
jittered = base_wait + random.uniform(-self._jitter, self._jitter)
|
||
wait = max(0, jittered)
|
||
if wait > 0:
|
||
await asyncio.sleep(wait)
|
||
self._last_request[domain] = time.monotonic()
|
||
|
||
|
||
_limiter = DomainRateLimiter(min_interval=1.5, jitter=0.5)
|
||
|
||
|
||
# ─── Helpers ──────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def log(msg):
|
||
print(f"[perplexity] {msg}", file=sys.stderr)
|
||
|
||
|
||
def _extract_text(html, max_chars=MAX_ARTICLE_CHARS):
|
||
"""Извлечь чистый текст из HTML."""
|
||
try:
|
||
soup = BeautifulSoup(html, "html.parser")
|
||
for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
|
||
tag.decompose()
|
||
text = soup.get_text(separator=" ")
|
||
text = re.sub(r"\s+", " ", text).strip()
|
||
if len(text) < 100:
|
||
return None
|
||
return text[:max_chars]
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
# ─── Async поиск и парсинг ────────────────────────────────────────────────────
|
||
|
||
|
||
async def searxng_search(session, query, category="general", max_results=20):
|
||
"""Поиск через SearXNG (async)."""
|
||
params = {
|
||
"q": query,
|
||
"format": "json",
|
||
"categories": category,
|
||
"pageno": 1,
|
||
}
|
||
try:
|
||
async with session.get(
|
||
SEARXNG_URL, params=params, headers=build_headers(),
|
||
timeout=aiohttp.ClientTimeout(total=30)
|
||
) as resp:
|
||
resp.raise_for_status()
|
||
data = await resp.json()
|
||
results = data.get("results", [])
|
||
# Только с валидными URL
|
||
valid = []
|
||
for r in results:
|
||
url = r.get("url", "")
|
||
if url and url.startswith("http"):
|
||
valid.append(r)
|
||
return valid[:max_results]
|
||
except Exception as e:
|
||
log(f"SearXNG error: {e}")
|
||
return []
|
||
|
||
|
||
async def fetch_content(session, url, max_chars=MAX_ARTICLE_CHARS):
|
||
"""Скачать и извлечь текст (aiohttp + curl fallback)."""
|
||
domain = urllib.parse.urlparse(url).hostname
|
||
await _limiter.acquire(domain)
|
||
headers = build_headers(referer="https://www.google.com/")
|
||
|
||
# Метод 1: aiohttp
|
||
try:
|
||
async with session.get(
|
||
url, headers=headers, timeout=aiohttp.ClientTimeout(total=15)
|
||
) as resp:
|
||
resp.raise_for_status()
|
||
html = await resp.text()
|
||
text = _extract_text(html, max_chars)
|
||
if text:
|
||
return text
|
||
except Exception:
|
||
pass
|
||
|
||
# Метод 2: curl fallback
|
||
try:
|
||
result = await asyncio.to_thread(
|
||
subprocess.run,
|
||
["curl", "-sL", "--max-time", "15", url,
|
||
"-H", f"User-Agent: {headers['User-Agent']}",
|
||
"-H", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||
"-H", "Accept-Language: ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7"],
|
||
capture_output=True, text=True, timeout=20,
|
||
)
|
||
if result.returncode == 0 and len(result.stdout) >= 300:
|
||
return _extract_text(result.stdout, max_chars)
|
||
except Exception:
|
||
pass
|
||
|
||
return None
|
||
|
||
|
||
def deduplicate(results):
|
||
seen = set()
|
||
unique = []
|
||
for r in results:
|
||
url = r.get("url", "")
|
||
if url and url not in seen:
|
||
seen.add(url)
|
||
unique.append(r)
|
||
return unique
|
||
|
||
|
||
# ─── Sync LLM call ────────────────────────────────────────────────────────────
|
||
|
||
|
||
def llm_chat(system, prompt, max_tokens=4096):
|
||
"""Запрос к OpenAI-совместимому API (llama-server)."""
|
||
payload = {
|
||
"model": LLM_MODEL,
|
||
"messages": [
|
||
{"role": "system", "content": system},
|
||
{"role": "user", "content": prompt}
|
||
],
|
||
"temperature": LLM_TEMPERATURE,
|
||
"max_tokens": max_tokens,
|
||
"stream": False
|
||
}
|
||
try:
|
||
result = subprocess.run(
|
||
["curl", "-sL", "--max-time", str(LLM_TIMEOUT + 30), LLM_URL,
|
||
"-H", "Content-Type: application/json",
|
||
"-d", json.dumps(payload)],
|
||
capture_output=True, text=True, timeout=LLM_TIMEOUT + 45,
|
||
)
|
||
data = json.loads(result.stdout)
|
||
return data.get("choices", [{}])[0].get("message", {}).get("content", "").strip()
|
||
except Exception:
|
||
return ""
|
||
|
||
|
||
# ─── Pipeline ─────────────────────────────────────────────────────────────────
|
||
|
||
|
||
async def perplexity_pipeline(query, category="general", max_results=3):
|
||
connector = aiohttp.TCPConnector(limit=10, ttl_dns_cache=300)
|
||
async with aiohttp.ClientSession(connector=connector) as session:
|
||
print(f"\n{'='*60}")
|
||
print(f" Perplexity Search: {query}")
|
||
print(f"{'='*60}\n")
|
||
|
||
# PHASE 1: SEARCH
|
||
print(" Phase 1/3: Searching SearXNG...", flush=True)
|
||
results = await searxng_search(session, query, category)
|
||
results = deduplicate(results)
|
||
print(f" Found {len(results)} results")
|
||
|
||
if not results:
|
||
print("\n No results found.")
|
||
return
|
||
|
||
results.sort(key=lambda r: r.get("score", 0), reverse=True)
|
||
top = results[:max_results]
|
||
|
||
for i, r in enumerate(top, 1):
|
||
print(f" [{i}] {r.get('title','?')[:80]}")
|
||
print(f" ({r.get('engine','?')}) {r.get('url','')[:90]}")
|
||
|
||
# PHASE 2: FETCH (параллельно!)
|
||
print(f"\n Phase 2/3: Fetching ({len(top)} articles in parallel)...", flush=True)
|
||
tasks = [fetch_content(session, r.get("url", "")) for r in top]
|
||
contents = await asyncio.gather(*tasks, return_exceptions=True)
|
||
|
||
sources = []
|
||
for i, (r, content) in enumerate(zip(top, contents), 1):
|
||
title = r.get("title", "?")
|
||
url = r.get("url", "")
|
||
if isinstance(content, Exception) or not content:
|
||
# Fallback: snippet from SearXNG
|
||
snippet = r.get("content", "")
|
||
if snippet and len(snippet) > 100:
|
||
print(f" [{i}] {title[:40]}... using snippet ({len(snippet)} chars)")
|
||
sources.append({
|
||
"id": i, "title": title.strip(), "url": url,
|
||
"engine": r.get("engine", "?"), "content": snippet
|
||
})
|
||
else:
|
||
print(f" [{i}] {title[:40]}... FAIL")
|
||
else:
|
||
print(f" [{i}] {title[:40]}... OK ({len(content)} chars)")
|
||
sources.append({
|
||
"id": i, "title": title.strip(), "url": url,
|
||
"engine": r.get("engine", "?"), "content": content
|
||
})
|
||
|
||
if not sources:
|
||
print("\n Could not fetch any articles.")
|
||
return
|
||
|
||
print(f" Fetched: {len(sources)}/{len(top)}")
|
||
|
||
# PHASE 3: LLM SYNTHESIS
|
||
print(f"\n Phase 3/3: Synthesizing ({LLM_MODEL})...", flush=True)
|
||
|
||
context_parts = []
|
||
for s in sources:
|
||
snippet = s["content"][:1500]
|
||
context_parts.append(
|
||
f"[{s['id']}] {s['title']}\nURL: {s['url']}\nContent: {snippet}\n"
|
||
)
|
||
context = "\n---\n".join(context_parts)
|
||
|
||
system = """Ты — Perplexity-style AI research assistant.
|
||
На основе источников [1][2][3] напиши ответ на русском.
|
||
|
||
Правила:
|
||
1. Каждый факт — ссылка на источник [1], [2], [3]
|
||
2. Противоречия в источниках — отмечай
|
||
3. Только из предоставленных источников
|
||
4. В конце — 3 related questions
|
||
|
||
Формат:
|
||
## [Тема]
|
||
|
||
**Ключевой вывод:** [1-2 предложения]
|
||
|
||
### Обзор
|
||
[синтез с ссылками]
|
||
|
||
### Тренды
|
||
- **Тренд:** описание [1]
|
||
|
||
### Детали
|
||
[подробно]
|
||
|
||
### Related questions
|
||
- [Вопрос 1]?
|
||
- [Вопрос 2]?
|
||
- [Вопрос 3]?
|
||
|
||
### Источники
|
||
[1] Название — URL
|
||
[2] Название — URL"""
|
||
|
||
user_prompt = f"Запрос: {query}\n\nИсточники:\n{context}"
|
||
response = llm_chat(system, user_prompt, max_tokens=2000)
|
||
|
||
if response:
|
||
# Всегда добавляем блок источников программно
|
||
if not re.search(r'(?:###?\s*)?(?:Источники|Sources)', response):
|
||
src_block = "\n### Источники\n"
|
||
for s in sources:
|
||
src_block += f"[{s['id']}] {s['title']} — {s['url']}\n"
|
||
response += src_block
|
||
|
||
print("\n")
|
||
print(response)
|
||
else:
|
||
print("\n LLM временно недоступен. Raw-отчёт:\n")
|
||
print(f"## {query}\n")
|
||
for s in sources:
|
||
preview = s["content"][:200].replace("\n", " ")
|
||
print(f"\n[{s['id']}] {s['title']}")
|
||
print(f" {s['url']} ({s['engine']})")
|
||
print(f" {preview}...")
|
||
print(f"\n### Источники")
|
||
for s in sources:
|
||
print(f"[{s['id']}] {s['title']} — {s['url']}")
|
||
|
||
print(f"\n{'─'*50}")
|
||
print(f" {len(results)} results | {len(sources)} fetched | Model: {LLM_MODEL} | q: {query}")
|
||
|
||
|
||
async def go_raw(query, category="general", max_results=3):
|
||
"""Режим --raw: только сбор данных, JSON на stdout."""
|
||
connector = aiohttp.TCPConnector(limit=10, ttl_dns_cache=300)
|
||
async with aiohttp.ClientSession(connector=connector) as session:
|
||
results = await searxng_search(session, query, category)
|
||
results = deduplicate(results)
|
||
results.sort(key=lambda r: r.get("score", 0), reverse=True)
|
||
top = results[:max_results]
|
||
|
||
tasks = [fetch_content(session, r.get("url", "")) for r in top]
|
||
contents = await asyncio.gather(*tasks, return_exceptions=True)
|
||
|
||
sources = []
|
||
for r, content in zip(top, contents):
|
||
url = r.get("url", "")
|
||
if not url:
|
||
continue
|
||
if isinstance(content, Exception) or not content:
|
||
snippet = r.get("content", "")
|
||
content = snippet if len(snippet) > 100 else None
|
||
sources.append({
|
||
"title": r.get("title", "").strip(),
|
||
"url": url,
|
||
"engine": r.get("engine", ""),
|
||
"content": (content or "")[:2000]
|
||
})
|
||
|
||
output = {"query": query, "total_results": len(results), "sources": sources}
|
||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||
|
||
|
||
# ─── CLI ──────────────────────────────────────────────────────────────────────
|
||
|
||
|
||
def main():
|
||
parser = argparse.ArgumentParser(
|
||
description="Perplexity-style search agent (async)"
|
||
)
|
||
parser.add_argument("query", nargs="?", help="Поисковый запрос")
|
||
parser.add_argument("--category", default="general",
|
||
choices=["general", "news", "it", "science"])
|
||
parser.add_argument("--results", type=int, default=3,
|
||
help="Сколько статей fetch'ить (макс 5)")
|
||
parser.add_argument("--raw", action="store_true",
|
||
help="Только JSON-данные, без LLM-синтеза")
|
||
parser.add_argument("--model", help="LLM модель")
|
||
parser.add_argument("--temperature", type=float, help="LLM temperature")
|
||
args = parser.parse_args()
|
||
|
||
if not args.query:
|
||
parser.print_help()
|
||
sys.exit(1)
|
||
|
||
global LLM_MODEL, LLM_TEMPERATURE
|
||
if args.model:
|
||
LLM_MODEL = args.model
|
||
if args.temperature is not None:
|
||
LLM_TEMPERATURE = args.temperature
|
||
|
||
max_results = min(args.results, 5)
|
||
|
||
if args.raw:
|
||
asyncio.run(go_raw(args.query, args.category, max_results))
|
||
else:
|
||
asyncio.run(perplexity_pipeline(args.query, args.category, max_results))
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|