#!/usr/bin/env python3
"""
Perplexity-style search agent для Hermes (async).

Использование:
  python3 perplexity_search.py "запрос"
  python3 perplexity_search.py "запрос" --category news --results 5
  python3 perplexity_search.py "запрос" --raw          # только JSON

Опции:
  --category  Категория SearXNG: general (default), news, it, science
  --results   Сколько результатов fetch'ить (default: 3, макс: 5)
  --raw       Только сбор данных, JSON на stdout (без LLM-синтеза)
  --model     Имя LLM модели (env: LLM_MODEL)
  --temperature  Temperature (env: LLM_TEMPERATURE)
"""

import argparse
import asyncio
import json
import os
import random
import re
import subprocess
import sys
import time
import urllib.parse
from collections import defaultdict
from datetime import datetime

import aiohttp
from bs4 import BeautifulSoup

# ─── Конфигурация ─────────────────────────────────────────────────────────────

SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://localhost:8888")
LLM_URL = os.environ.get("LLM_URL", "http://192.168.1.152:8080/v1/chat/completions")
LLM_MODEL = os.environ.get("LLM_MODEL", "Qwen3.6-35B-A3B-UD-IQ4_NL.gguf")
LLM_TEMPERATURE = float(os.environ.get("LLM_TEMPERATURE", "0.7"))
MAX_ARTICLE_CHARS = 5000
LLM_TIMEOUT = 120

# ─── Anti-blocking: User-Agent ротация ────────────────────────────────────────

USER_AGENTS = [
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36",
    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/18.0 Safari/605.1.15",
    "Mozilla/5.0 (X11; Linux x86_64; rv:133.0) Gecko/20100101 Firefox/133.0",
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:132.0) Gecko/20100101 Firefox/132.0",
    "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36",
]


def random_ua():
    return random.choice(USER_AGENTS)


def build_headers(referer=None):
    h = {
        "User-Agent": random_ua(),
        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
        "Accept-Language": "ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7",
        "Accept-Encoding": "gzip, deflate",
        "DNT": "1",
        "Connection": "keep-alive",
    }
    if referer:
        h["Referer"] = referer
    return h


# ─── Anti-blocking: Domain Rate Limiter ────────────────────────────────────────


class DomainRateLimiter:
    def __init__(self, min_interval=1.5, jitter=0.5):
        self._last_request = defaultdict(float)
        self._min_interval = min_interval
        self._jitter = jitter

    async def acquire(self, domain):
        elapsed = time.monotonic() - self._last_request[domain]
        base_wait = max(0, self._min_interval - elapsed)
        jittered = base_wait + random.uniform(-self._jitter, self._jitter)
        wait = max(0, jittered)
        if wait > 0:
            await asyncio.sleep(wait)
        self._last_request[domain] = time.monotonic()


_limiter = DomainRateLimiter(min_interval=1.5, jitter=0.5)


# ─── Helpers ──────────────────────────────────────────────────────────────────


def log(msg):
    print(f"[perplexity] {msg}", file=sys.stderr)


def _extract_text(html, max_chars=MAX_ARTICLE_CHARS):
    """Извлечь чистый текст из HTML."""
    try:
        soup = BeautifulSoup(html, "html.parser")
        for tag in soup(["script", "style", "nav", "footer", "header", "aside"]):
            tag.decompose()
        text = soup.get_text(separator=" ")
        text = re.sub(r"\s+", " ", text).strip()
        if len(text) < 100:
            return None
        return text[:max_chars]
    except Exception:
        return None


# ─── Async поиск и парсинг ────────────────────────────────────────────────────


async def searxng_search(session, query, category="general", max_results=20):
    """Поиск через SearXNG (async)."""
    params = {
        "q": query,
        "format": "json",
        "categories": category,
        "pageno": 1,
    }
    try:
        async with session.get(
            SEARXNG_URL, params=params, headers=build_headers(),
            timeout=aiohttp.ClientTimeout(total=30)
        ) as resp:
            resp.raise_for_status()
            data = await resp.json()
        results = data.get("results", [])
        # Только с валидными URL
        valid = []
        for r in results:
            url = r.get("url", "")
            if url and url.startswith("http"):
                valid.append(r)
        return valid[:max_results]
    except Exception as e:
        log(f"SearXNG error: {e}")
        return []


async def fetch_content(session, url, max_chars=MAX_ARTICLE_CHARS):
    """Скачать и извлечь текст (aiohttp + curl fallback)."""
    domain = urllib.parse.urlparse(url).hostname
    await _limiter.acquire(domain)
    headers = build_headers(referer="https://www.google.com/")

    # Метод 1: aiohttp
    try:
        async with session.get(
            url, headers=headers, timeout=aiohttp.ClientTimeout(total=15)
        ) as resp:
            resp.raise_for_status()
            html = await resp.text()
        text = _extract_text(html, max_chars)
        if text:
            return text
    except Exception:
        pass

    # Метод 2: curl fallback
    try:
        result = await asyncio.to_thread(
            subprocess.run,
            ["curl", "-sL", "--max-time", "15", url,
             "-H", f"User-Agent: {headers['User-Agent']}",
             "-H", "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
             "-H", "Accept-Language: ru-RU,ru;q=0.9,en-US;q=0.8,en;q=0.7"],
            capture_output=True, text=True, timeout=20,
        )
        if result.returncode == 0 and len(result.stdout) >= 300:
            return _extract_text(result.stdout, max_chars)
    except Exception:
        pass

    return None


def deduplicate(results):
    seen = set()
    unique = []
    for r in results:
        url = r.get("url", "")
        if url and url not in seen:
            seen.add(url)
            unique.append(r)
    return unique


# ─── Sync LLM call ────────────────────────────────────────────────────────────


def llm_chat(system, prompt, max_tokens=4096):
    """Запрос к OpenAI-совместимому API (llama-server)."""
    payload = {
        "model": LLM_MODEL,
        "messages": [
            {"role": "system", "content": system},
            {"role": "user", "content": prompt}
        ],
        "temperature": LLM_TEMPERATURE,
        "max_tokens": max_tokens,
        "stream": False
    }
    try:
        result = subprocess.run(
            ["curl", "-sL", "--max-time", str(LLM_TIMEOUT + 30), LLM_URL,
             "-H", "Content-Type: application/json",
             "-d", json.dumps(payload)],
            capture_output=True, text=True, timeout=LLM_TIMEOUT + 45,
        )
        data = json.loads(result.stdout)
        return data.get("choices", [{}])[0].get("message", {}).get("content", "").strip()
    except Exception:
        return ""


# ─── Pipeline ─────────────────────────────────────────────────────────────────


async def perplexity_pipeline(query, category="general", max_results=3):
    connector = aiohttp.TCPConnector(limit=10, ttl_dns_cache=300)
    async with aiohttp.ClientSession(connector=connector) as session:
        print(f"\n{'='*60}")
        print(f"  Perplexity Search: {query}")
        print(f"{'='*60}\n")

        # PHASE 1: SEARCH
        print("  Phase 1/3: Searching SearXNG...", flush=True)
        results = await searxng_search(session, query, category)
        results = deduplicate(results)
        print(f"  Found {len(results)} results")

        if not results:
            print("\n  No results found.")
            return

        results.sort(key=lambda r: r.get("score", 0), reverse=True)
        top = results[:max_results]

        for i, r in enumerate(top, 1):
            print(f"    [{i}] {r.get('title','?')[:80]}")
            print(f"        ({r.get('engine','?')}) {r.get('url','')[:90]}")

        # PHASE 2: FETCH (параллельно!)
        print(f"\n  Phase 2/3: Fetching ({len(top)} articles in parallel)...", flush=True)
        tasks = [fetch_content(session, r.get("url", "")) for r in top]
        contents = await asyncio.gather(*tasks, return_exceptions=True)

        sources = []
        for i, (r, content) in enumerate(zip(top, contents), 1):
            title = r.get("title", "?")
            url = r.get("url", "")
            if isinstance(content, Exception) or not content:
                # Fallback: snippet from SearXNG
                snippet = r.get("content", "")
                if snippet and len(snippet) > 100:
                    print(f"    [{i}] {title[:40]}... using snippet ({len(snippet)} chars)")
                    sources.append({
                        "id": i, "title": title.strip(), "url": url,
                        "engine": r.get("engine", "?"), "content": snippet
                    })
                else:
                    print(f"    [{i}] {title[:40]}... FAIL")
            else:
                print(f"    [{i}] {title[:40]}... OK ({len(content)} chars)")
                sources.append({
                    "id": i, "title": title.strip(), "url": url,
                    "engine": r.get("engine", "?"), "content": content
                })

        if not sources:
            print("\n  Could not fetch any articles.")
            return

        print(f"  Fetched: {len(sources)}/{len(top)}")

        # PHASE 3: LLM SYNTHESIS
        print(f"\n  Phase 3/3: Synthesizing ({LLM_MODEL})...", flush=True)

        context_parts = []
        for s in sources:
            snippet = s["content"][:1500]
            context_parts.append(
                f"[{s['id']}] {s['title']}\nURL: {s['url']}\nContent: {snippet}\n"
            )
        context = "\n---\n".join(context_parts)

        system = """Ты — Perplexity-style AI research assistant.
На основе источников [1][2][3] напиши ответ на русском.

Правила:
1. Каждый факт — ссылка на источник [1], [2], [3]
2. Противоречия в источниках — отмечай
3. Только из предоставленных источников
4. В конце — 3 related questions

Формат:
## [Тема]

**Ключевой вывод:** [1-2 предложения]

### Обзор
[синтез с ссылками]

### Тренды
- **Тренд:** описание [1]

### Детали
[подробно]

### Related questions
- [Вопрос 1]?
- [Вопрос 2]?
- [Вопрос 3]?

### Источники
[1] Название — URL
[2] Название — URL"""

        user_prompt = f"Запрос: {query}\n\nИсточники:\n{context}"
        response = llm_chat(system, user_prompt, max_tokens=2000)

        if response:
            # Всегда добавляем блок источников программно
            if not re.search(r'(?:###?\s*)?(?:Источники|Sources)', response):
                src_block = "\n### Источники\n"
                for s in sources:
                    src_block += f"[{s['id']}] {s['title']} — {s['url']}\n"
                response += src_block

            print("\n")
            print(response)
        else:
            print("\n  LLM временно недоступен. Raw-отчёт:\n")
            print(f"## {query}\n")
            for s in sources:
                preview = s["content"][:200].replace("\n", " ")
                print(f"\n[{s['id']}] {s['title']}")
                print(f"    {s['url']}  ({s['engine']})")
                print(f"    {preview}...")
            print(f"\n### Источники")
            for s in sources:
                print(f"[{s['id']}] {s['title']} — {s['url']}")

        print(f"\n{'─'*50}")
        print(f"  {len(results)} results | {len(sources)} fetched | Model: {LLM_MODEL} | q: {query}")


async def go_raw(query, category="general", max_results=3):
    """Режим --raw: только сбор данных, JSON на stdout."""
    connector = aiohttp.TCPConnector(limit=10, ttl_dns_cache=300)
    async with aiohttp.ClientSession(connector=connector) as session:
        results = await searxng_search(session, query, category)
        results = deduplicate(results)
        results.sort(key=lambda r: r.get("score", 0), reverse=True)
        top = results[:max_results]

        tasks = [fetch_content(session, r.get("url", "")) for r in top]
        contents = await asyncio.gather(*tasks, return_exceptions=True)

        sources = []
        for r, content in zip(top, contents):
            url = r.get("url", "")
            if not url:
                continue
            if isinstance(content, Exception) or not content:
                snippet = r.get("content", "")
                content = snippet if len(snippet) > 100 else None
            sources.append({
                "title": r.get("title", "").strip(),
                "url": url,
                "engine": r.get("engine", ""),
                "content": (content or "")[:2000]
            })

        output = {"query": query, "total_results": len(results), "sources": sources}
        print(json.dumps(output, ensure_ascii=False, indent=2))


# ─── CLI ──────────────────────────────────────────────────────────────────────


def main():
    parser = argparse.ArgumentParser(
        description="Perplexity-style search agent (async)"
    )
    parser.add_argument("query", nargs="?", help="Поисковый запрос")
    parser.add_argument("--category", default="general",
                        choices=["general", "news", "it", "science"])
    parser.add_argument("--results", type=int, default=3,
                        help="Сколько статей fetch'ить (макс 5)")
    parser.add_argument("--raw", action="store_true",
                        help="Только JSON-данные, без LLM-синтеза")
    parser.add_argument("--model", help="LLM модель")
    parser.add_argument("--temperature", type=float, help="LLM temperature")
    args = parser.parse_args()

    if not args.query:
        parser.print_help()
        sys.exit(1)

    global LLM_MODEL, LLM_TEMPERATURE
    if args.model:
        LLM_MODEL = args.model
    if args.temperature is not None:
        LLM_TEMPERATURE = args.temperature

    max_results = min(args.results, 5)

    if args.raw:
        asyncio.run(go_raw(args.query, args.category, max_results))
    else:
        asyncio.run(perplexity_pipeline(args.query, args.category, max_results))


if __name__ == "__main__":
    main()