fix: SEARXNG_URL no longer includes /search suffix (provider adds it), sync all files
This commit is contained in:
nik committed
1 parent
e5e9b5c664
commit
d4731bee20
11 files changed
+2891
No files matched your search
Regular → Executable
+424
@@ -0,0 +1,424 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Perplexity-style search agent для Hermes (async).
|
||||
|
||||
Использование:
|
||||
python3 perplexity_search.py "запрос"
|
||||
python3 perplexity_search.py "запрос" --category news --results 5
|
||||
python3 perplexity_search.py "запрос" --raw # только JSON
|
||||
|
||||
Опции:
|
||||
--category Категория SearXNG: general (default), news, it, science
|
||||
--results Сколько результатов fetch'ить (default: 3, макс: 5)
|
||||
--raw Только сбор данных, JSON на stdout (без LLM-синтеза)
|
||||
--model Имя LLM модели (env: LLM_MODEL)
|
||||
--temperature Temperature (env: LLM_TEMPERATURE)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
import urllib.parse
|
||||
from collections import defaultdict
|
||||
from datetime import datetime
|
||||
|
||||
import aiohttp
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
# ─── Конфигурация ─────────────────────────────────────────────────────────────
|
||||
|
||||
SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://localhost:8888")
|
||||
LLM_URL = os.environ.get("LLM_URL", "http://192.168.1.152:8080/v1/chat/completions")
|
||||
LLM_MODEL = os.environ.get("LLM_MODEL", "Qwen3.6-35B-A3B-UD-IQ4_NL.gguf")
|
||||
LLM_TEMPERATURE = float(os.environ.get("LLM_TEMPERATURE", "0.3"))
|
||||
MAX_ARTICLE_CHARS = 5000
|
||||
LLM_TIMEOUT = 120
|
||||
|
||||
# ─── Anti-blocking: User-Agent ротация ────────────────────────────────────────
|
||||
|
||||
USER_AGENTS = [
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15",
|
||||
"Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
|
||||
"Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
|
||||
def random_ua():
|
||||
return random.choice(USER_AGENTS)
|
||||
|
||||
|
||||
def build_headers(referer=None):
|
||||
h = {
|
||||
"User-Agent": random_ua(),
|
||||
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
|
||||
"Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
|
||||
"Accept-Encoding": "gzip, deflate",
|
||||
"DNT": "1",
|
||||
"Connection": "keep-alive",
|
||||
}
|
||||
if referer:
|
||||
h["Referer"] = referer
|
||||
return h
|
||||
|
||||
|
||||
# ─── Anti-blocking: Domain Rate Limiter ────────────────────────────────────────
|
||||
|
||||
|
||||
class DomainRateLimiter:
|
||||
def __init__(self, min_interval=1.5, jitter=0.5):
|
||||
self._last_request = defaultdict(float)
|
||||
self._min_interval = min_interval
|
||||
self._jitter = jitter
|
||||
|
||||
async def acquire(self, domain):
|
||||
elapsed = time.monotonic() - self._last_request[domain]
|
||||
base_wait = max(0, self._min_interval - elapsed)
|
||||
jittered = base_wait + random.uniform(-self._jitter, self._jitter)
|
||||
wait = max(0, jittered)
|
||||
if wait > 0:
|
||||
await asyncio.sleep(wait)
|
||||
self._last_request[domain] = time.monotonic()
|
||||
|
||||
|
||||
_limiter = DomainRateLimiter(min_interval=1.5, jitter=0.5)
|
||||
|
||||
|
||||
# ─── Helpers ──────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def log(msg):
|
||||
print(f"[perplexity] {msg}", file=sys.stderr)
|
||||
|
||||
|
||||
def _extract_text(html, max_chars=MAX_ARTICLE_CHARS):
|
||||
"""Извлечь чистый текст из HTML."""
|
||||
try:
|
||||
soup = BeautifulSoup(html, "html.parser")
|
||||
for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
|
||||
tag.decompose()
|
||||
text = soup.get_text(separator=" ")
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
if len(text) < 100:
|
||||
return None
|
||||
return text[:max_chars]
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
# ─── Async поиск и парсинг ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def searxng_search(session, query, category="general", max_results=20):
|
||||
"""Поиск через SearXNG (async)."""
|
||||
params = {
|
||||
"q": query,
|
||||
"format": "json",
|
||||
"categories": category,
|
||||
"pageno": 1,
|
||||
}
|
||||
try:
|
||||
async with session.get(
|
||||
SEARXNG_URL, params=params, headers=build_headers(),
|
||||
timeout=aiohttp.ClientTimeout(total=30)
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
data = await resp.json()
|
||||
results = data.get("results", [])
|
||||
# Только с валидными URL
|
||||
valid = []
|
||||
for r in results:
|
||||
url = r.get("url", "")
|
||||
if url and url.startswith("http"):
|
||||
valid.append(r)
|
||||
return valid[:max_results]
|
||||
except Exception as e:
|
||||
log(f"SearXNG error: {e}")
|
||||
return []
|
||||
|
||||
|
||||
async def fetch_content(session, url, max_chars=MAX_ARTICLE_CHARS):
|
||||
"""Скачать и извлечь текст (aiohttp + curl fallback)."""
|
||||
domain = urllib.parse.urlparse(url).hostname
|
||||
await _limiter.acquire(domain)
|
||||
headers = build_headers(referer="https://www.google.com/")
|
||||
|
||||
# Метод 1: aiohttp
|
||||
try:
|
||||
async with session.get(
|
||||
url, headers=headers, timeout=aiohttp.ClientTimeout(total=15)
|
||||
) as resp:
|
||||
resp.raise_for_status()
|
||||
html = await resp.text()
|
||||
text = _extract_text(html, max_chars)
|
||||
if text:
|
||||
return text
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Метод 2: curl fallback
|
||||
try:
|
||||
result = await asyncio.to_thread(
|
||||
subprocess.run,
|
||||
["curl", "-sL", "--max-time", "15", url,
|
||||
"-H", f"User-Agent: {headers['User-Agent']}",
|
||||
"-H", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"-H", "Accept-Language: ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7"],
|
||||
capture_output=True, text=True, timeout=20,
|
||||
)
|
||||
if result.returncode == 0 and len(result.stdout) >= 300:
|
||||
return _extract_text(result.stdout, max_chars)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return None
|
||||
|
||||
|
||||
def deduplicate(results):
|
||||
seen = set()
|
||||
unique = []
|
||||
for r in results:
|
||||
url = r.get("url", "")
|
||||
if url and url not in seen:
|
||||
seen.add(url)
|
||||
unique.append(r)
|
||||
return unique
|
||||
|
||||
|
||||
# ─── Sync LLM call ────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def llm_chat(system, prompt, max_tokens=2000):
|
||||
"""Запрос к OpenAI-совместимому API (llama-server)."""
|
||||
payload = {
|
||||
"model": LLM_MODEL,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": prompt}
|
||||
],
|
||||
"temperature": LLM_TEMPERATURE,
|
||||
"max_tokens": max_tokens,
|
||||
"stream": False
|
||||
}
|
||||
try:
|
||||
result = subprocess.run(
|
||||
["curl", "-sL", "--max-time", str(LLM_TIMEOUT + 30), LLM_URL,
|
||||
"-H", "Content-Type: application/json",
|
||||
"-d", json.dumps(payload)],
|
||||
capture_output=True, text=True, timeout=LLM_TIMEOUT + 45,
|
||||
)
|
||||
data = json.loads(result.stdout)
|
||||
return data.get("choices", [{}])[0].get("message", {}).get("content", "").strip()
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
# ─── Pipeline ─────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
async def perplexity_pipeline(query, category="general", max_results=3):
|
||||
connector = aiohttp.TCPConnector(limit=10, ttl_dns_cache=300)
|
||||
async with aiohttp.ClientSession(connector=connector) as session:
|
||||
print(f"\n{'='*60}")
|
||||
print(f" Perplexity Search: {query}")
|
||||
print(f"{'='*60}\n")
|
||||
|
||||
# PHASE 1: SEARCH
|
||||
print(" Phase 1/3: Searching SearXNG...", flush=True)
|
||||
results = await searxng_search(session, query, category)
|
||||
results = deduplicate(results)
|
||||
print(f" Found {len(results)} results")
|
||||
|
||||
if not results:
|
||||
print("\n No results found.")
|
||||
return
|
||||
|
||||
results.sort(key=lambda r: r.get("score", 0), reverse=True)
|
||||
top = results[:max_results]
|
||||
|
||||
for i, r in enumerate(top, 1):
|
||||
print(f" [{i}] {r.get('title','?')[:80]}")
|
||||
print(f" ({r.get('engine','?')}) {r.get('url','')[:90]}")
|
||||
|
||||
# PHASE 2: FETCH (параллельно!)
|
||||
print(f"\n Phase 2/3: Fetching ({len(top)} articles in parallel)...", flush=True)
|
||||
tasks = [fetch_content(session, r.get("url", "")) for r in top]
|
||||
contents = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
sources = []
|
||||
for i, (r, content) in enumerate(zip(top, contents), 1):
|
||||
title = r.get("title", "?")
|
||||
url = r.get("url", "")
|
||||
if isinstance(content, Exception) or not content:
|
||||
# Fallback: snippet from SearXNG
|
||||
snippet = r.get("content", "")
|
||||
if snippet and len(snippet) > 100:
|
||||
print(f" [{i}] {title[:40]}... using snippet ({len(snippet)} chars)")
|
||||
sources.append({
|
||||
"id": i, "title": title.strip(), "url": url,
|
||||
"engine": r.get("engine", "?"), "content": snippet
|
||||
})
|
||||
else:
|
||||
print(f" [{i}] {title[:40]}... FAIL")
|
||||
else:
|
||||
print(f" [{i}] {title[:40]}... OK ({len(content)} chars)")
|
||||
sources.append({
|
||||
"id": i, "title": title.strip(), "url": url,
|
||||
"engine": r.get("engine", "?"), "content": content
|
||||
})
|
||||
|
||||
if not sources:
|
||||
print("\n Could not fetch any articles.")
|
||||
return
|
||||
|
||||
print(f" Fetched: {len(sources)}/{len(top)}")
|
||||
|
||||
# PHASE 3: LLM SYNTHESIS
|
||||
print(f"\n Phase 3/3: Synthesizing ({LLM_MODEL})...", flush=True)
|
||||
|
||||
context_parts = []
|
||||
for s in sources:
|
||||
snippet = s["content"][:1500]
|
||||
context_parts.append(
|
||||
f"[{s['id']}] {s['title']}\nURL: {s['url']}\nContent: {snippet}\n"
|
||||
)
|
||||
context = "\n---\n".join(context_parts)
|
||||
|
||||
system = """Ты — Perplexity-style AI research assistant.
|
||||
На основе источников [1][2][3] напиши ответ на русском.
|
||||
|
||||
Правила:
|
||||
1. Каждый факт — ссылка на источник [1], [2], [3]
|
||||
2. Противоречия в источниках — отмечай
|
||||
3. Только из предоставленных источников
|
||||
4. В конце — 3 related questions
|
||||
|
||||
Формат:
|
||||
## [Тема]
|
||||
|
||||
**Ключевой вывод:** [1-2 предложения]
|
||||
|
||||
### Обзор
|
||||
[синтез с ссылками]
|
||||
|
||||
### Тренды
|
||||
- **Тренд:** описание [1]
|
||||
|
||||
### Детали
|
||||
[подробно]
|
||||
|
||||
### Related questions
|
||||
- [Вопрос 1]?
|
||||
- [Вопрос 2]?
|
||||
- [Вопрос 3]?
|
||||
|
||||
### Источники
|
||||
[1] Название — URL
|
||||
[2] Название — URL"""
|
||||
|
||||
user_prompt = f"Запрос: {query}\n\nИсточники:\n{context}"
|
||||
response = llm_chat(system, user_prompt, max_tokens=2000)
|
||||
|
||||
if response:
|
||||
# Всегда добавляем блок источников программно
|
||||
if not re.search(r'(?:###?\s*)?(?:Источники|Sources)', response):
|
||||
src_block = "\n### Источники\n"
|
||||
for s in sources:
|
||||
src_block += f"[{s['id']}] {s['title']} — {s['url']}\n"
|
||||
response += src_block
|
||||
|
||||
print("\n")
|
||||
print(response)
|
||||
else:
|
||||
print("\n LLM временно недоступен. Raw-отчёт:\n")
|
||||
print(f"## {query}\n")
|
||||
for s in sources:
|
||||
preview = s["content"][:200].replace("\n", " ")
|
||||
print(f"\n[{s['id']}] {s['title']}")
|
||||
print(f" {s['url']} ({s['engine']})")
|
||||
print(f" {preview}...")
|
||||
print(f"\n### Источники")
|
||||
for s in sources:
|
||||
print(f"[{s['id']}] {s['title']} — {s['url']}")
|
||||
|
||||
print(f"\n{'─'*50}")
|
||||
print(f" {len(results)} results | {len(sources)} fetched | Model: {LLM_MODEL} | q: {query}")
|
||||
|
||||
|
||||
async def go_raw(query, category="general", max_results=3):
|
||||
"""Режим --raw: только сбор данных, JSON на stdout."""
|
||||
connector = aiohttp.TCPConnector(limit=10, ttl_dns_cache=300)
|
||||
async with aiohttp.ClientSession(connector=connector) as session:
|
||||
results = await searxng_search(session, query, category)
|
||||
results = deduplicate(results)
|
||||
results.sort(key=lambda r: r.get("score", 0), reverse=True)
|
||||
top = results[:max_results]
|
||||
|
||||
tasks = [fetch_content(session, r.get("url", "")) for r in top]
|
||||
contents = await asyncio.gather(*tasks, return_exceptions=True)
|
||||
|
||||
sources = []
|
||||
for r, content in zip(top, contents):
|
||||
url = r.get("url", "")
|
||||
if not url:
|
||||
continue
|
||||
if isinstance(content, Exception) or not content:
|
||||
snippet = r.get("content", "")
|
||||
content = snippet if len(snippet) > 100 else None
|
||||
sources.append({
|
||||
"title": r.get("title", "").strip(),
|
||||
"url": url,
|
||||
"engine": r.get("engine", ""),
|
||||
"content": (content or "")[:2000]
|
||||
})
|
||||
|
||||
output = {"query": query, "total_results": len(results), "sources": sources}
|
||||
print(json.dumps(output, ensure_ascii=False, indent=2))
|
||||
|
||||
|
||||
# ─── CLI ──────────────────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Perplexity-style search agent (async)"
|
||||
)
|
||||
parser.add_argument("query", nargs="?", help="Поисковый запрос")
|
||||
parser.add_argument("--category", default="general",
|
||||
choices=["general", "news", "it", "science"])
|
||||
parser.add_argument("--results", type=int, default=3,
|
||||
help="Сколько статей fetch'ить (макс 5)")
|
||||
parser.add_argument("--raw", action="store_true",
|
||||
help="Только JSON-данные, без LLM-синтеза")
|
||||
parser.add_argument("--model", help="LLM модель")
|
||||
parser.add_argument("--temperature", type=float, help="LLM temperature")
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.query:
|
||||
parser.print_help()
|
||||
sys.exit(1)
|
||||
|
||||
global LLM_MODEL, LLM_TEMPERATURE
|
||||
if args.model:
|
||||
LLM_MODEL = args.model
|
||||
if args.temperature is not None:
|
||||
LLM_TEMPERATURE = args.temperature
|
||||
|
||||
max_results = min(args.results, 5)
|
||||
|
||||
if args.raw:
|
||||
asyncio.run(go_raw(args.query, args.category, max_results))
|
||||
else:
|
||||
asyncio.run(perplexity_pipeline(args.query, args.category, max_results))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user