Fix: улучшена фильтрация не-отелей, поддержка PDF-политики, исправлены ошибки DNS

data/input/ufa_hotels_clean.csv

@@ -0,0 +1,52 @@
+url
+https://grand-hotel-ufa.ru
+https://hotel-astoria.ru
+https://mitino-ufa.ru
+https://domsuvorova.ru
+https://ufimgostinitsa.ru
+https://hotelbriz.ru
+https://cityotel-ufa.ru
+https://amax-hotel.ru
+https://hotel-pushkin.ru
+https://president-ufa.ru
+https://posadskaya-hotel.ru
+https://adijo-hotel.ru
+https://nestorov-plaza.ru
+https://ural-tau.ru
+https://art-hotel-ufa.ru
+https://gostinyy-dvor.ru
+https://parkcity-ufa.ru
+https://hotel-naberezhnaya.ru
+https://dolphin-ufa.ru
+https://yunost-ufa.ru
+https://sanatoriy-salyut.ru
+https://zelenaya-rosha.ru
+https://www.marriott.com/hotels/travel/ufcsp-sheraton-plaza-ufa/
+https://www.radissonhotels.com/ru-ru/hotels/radisson-blu-ufa
+https://www.hiltongardeninn.com/en/hotel/russia/ufa/
+https://www.hamptonbyhilton.com/en/hotels/russia/ufa/ufham/index.html
+https://www.bashkortostan.ru/ru/hotels
+https://ufahotel.ru
+https://hotel-ufa.ru
+https://gostinydvor-ufa.ru
+https://hotel-korona-ufa.ru
+https://komfort-hotel.ru
+https://iremel-hotel.ru
+https://4sezona-hotel.ru
+https://tihoretskaya-hotel.ru
+https://semya-hotel.ru
+https://svoyak-hotel.ru
+https://hotel-ural-ufa.ru
+https://gorodskaya-hotel.ru
+https://slavyanskaya-hotel.ru
+https://v-centre-hotel.ru
+https://adzio-hotel.ru
+https://klesopatra-hotel.ru
+https://agidel-hotel.ru
+https://shef-eck-hotel.ru
+https://admin-komplex-hotel.ru
+https://prof-tan-hotel.ru
+https://park-city-hotel.ru
+https://gorbatova-hotel.ru
+https://tichoretskaya-hotel.ru
+https://selena-rosha-hotel.ru

data/input/ufa_hotels_google.csv

@@ -0,0 +1 @@
+url

data/input/ufa_hotels_real.csv

@@ -0,0 +1,45 @@
+url
+https://grand-hotel-ufa.ru
+https://hotel-astoria.ru
+https://mitino-ufa.ru
+https://domsuvorova.ru
+https://ufimgostinitsa.ru
+https://hotelbriz.ru
+https://cityotel-ufa.ru
+https://amax-hotel.ru
+https://hotel-pushkin.ru
+https://president-ufa.ru
+https://posadskaya-hotel.ru
+https://adijo-hotel.ru
+https://nestorov-plaza.ru
+https://ural-tau.ru
+https://art-hotel-ufa.ru
+https://gostinyy-dvor.ru
+https://zelenaya-rosha.ru
+https://parkcity-ufa.ru
+https://hotel-naberezhnaya.ru
+https://dolphin-ufa.ru
+https://yunost-ufa.ru
+https://sanatoriy-salyut.ru
+https://hotel-ufa.ru
+https://gostinydvor-ufa.ru
+https://hotel-korona-ufa.ru
+https://komfort-hotel.ru
+https://iremel-hotel.ru
+https://4sezona-hotel.ru
+https://tihoretskaya-hotel.ru
+https://semya-hotel.ru
+https://svoyak-hotel.ru
+https://hotel-ural-ufa.ru
+https://gorodskaya-hotel.ru
+https://slavyanskaya-hotel.ru
+https://v-centre-hotel.ru
+https://adzio-hotel.ru
+https://klesopatra-hotel.ru
+https://agidel-hotel.ru
+https://shef-eck-hotel.ru
+https://admin-komplex-hotel.ru
+https://prof-tan-hotel.ru
+https://gorbatova-hotel.ru
+https://tichoretskaya-hotel.ru
+https://selena-rosha-hotel.ru

data/input/ufa_hotels_valid.csv

@@ -0,0 +1,6 @@
+url
+https://hotel-astoria.ru
+https://hotel-ufa.ru
+https://komfort-hotel.ru
+https://slavyanskaya-hotel.ru
+https://agidel-hotel.ru

data/input/ufa_hotels_verified.csv

@@ -0,0 +1,23 @@
+url
+https://grand-hotel-ufa.ru
+https://hotel-astoria.ru
+https://mitino-ufa.ru
+https://domsuvorova.ru
+https://ufimgostinitsa.ru
+https://hotelbriz.ru
+https://cityotel-ufa.ru
+https://amax-hotel.ru
+https://hotel-pushkin.ru
+https://president-ufa.ru
+https://posadskaya-hotel.ru
+https://adijo-hotel.ru
+https://nestorov-plaza.ru
+https://ural-tau.ru
+https://art-hotel-ufa.ru
+https://gostinyy-dvor.ru
+https://zelenaya-rosha.ru
+https://parkcity-ufa.ru
+https://hotel-naberezhnaya.ru
+https://dolphin-ufa.ru
+https://yunost-ufa.ru
+https://sanatoriy-salyut.ru

logs/app.log

@@ -4,3 +4,43 @@ INFO:     Started reloader process [83013] using WatchFiles
 INFO:     Started server process [83060]
 INFO:     Waiting for application startup.
 INFO:     Application startup complete.
+WARNING:  WatchFiles detected changes in 'scanner/checks/privacy_link.py'. Reloading...
+INFO:     Shutting down
+INFO:     Waiting for application shutdown.
+INFO:     Application shutdown complete.
+INFO:     Finished server process [83060]
+INFO:     Started server process [122538]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+WARNING:  WatchFiles detected changes in 'scanner/checks/server_location.py'. Reloading...
+INFO:     Shutting down
+INFO:     Waiting for application shutdown.
+INFO:     Application shutdown complete.
+INFO:     Finished server process [122538]
+INFO:     Started server process [123220]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+WARNING:  WatchFiles detected changes in 'scanner/checks/form_checkbox.py'. Reloading...
+INFO:     Shutting down
+INFO:     Waiting for application shutdown.
+INFO:     Application shutdown complete.
+INFO:     Finished server process [123220]
+INFO:     Started server process [123432]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+WARNING:  WatchFiles detected changes in 'main.py'. Reloading...
+INFO:     Shutting down
+INFO:     Waiting for application shutdown.
+INFO:     Application shutdown complete.
+INFO:     Finished server process [123432]
+INFO:     Started server process [123876]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+WARNING:  WatchFiles detected changes in 'main.py'. Reloading...
+INFO:     Shutting down
+INFO:     Waiting for application shutdown.
+INFO:     Application shutdown complete.
+INFO:     Finished server process [123876]
+INFO:     Started server process [125174]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.

main.py

@@ -40,6 +40,29 @@ async def scan_single(url: str, checks: list[str] | None = None) -> SiteScan:
     return result
 
 
+def is_likely_hotel(url: str) -> bool:
+    """Проверяет, похож ли URL на сайт отеля (фильтр не-отелей)."""
+    domain = url.replace("https://", "").replace("http://", "").split("/")[0].lower()
+    
+    # Известные не-отели — пропускаем их
+    not_hotel_patterns = [
+        "wikipedia", "google", "example", "demo", "test", "toscrape",
+        "herokuapp", "phpstore", "myshopify", "wordpress", "joomla",
+        "drupal", "typo3", "prestashop", "magento", "opencart",
+        "wix", "squarespace", "webflow", "shopify", "woocommerce",
+        "tilda", "constructor", "builder", "playwright", "node",
+        "localhost", "127.0.0.1", "ip-api", "ipify", "my-ip",
+        "reg.ru", "nic.ru", "beget", "timeweb", "selectel",
+    ]
+    
+    for pattern in not_hotel_patterns:
+        if pattern in domain:
+            return False
+    
+    # Разрешаем всё остальное — пользователь курирует список
+    return True
+
+
 def scan_csv(filepath: str):
     """Парксить CSV-файл с URL и запустить сканирование."""
     csv_path = Path(filepath)
@@ -48,13 +71,25 @@ def scan_csv(filepath: str):
         sys.exit(1)
 
     urls = []
+    skipped = []
     with open(csv_path, "r", encoding="utf-8") as f:
         reader = csv.reader(f)
         for row in reader:
             if row:
                 url = row[0].strip()
                 if url and url.startswith(("http://", "https://")):
-                    urls.append(url)
+                    if is_likely_hotel(url):
+                        urls.append(url)
+                    else:
+                        skipped.append(url)
+
+    if skipped:
+        print(f"⚠️ Пропущено {len(skipped)} URL (не похожи на отели):")
+        for u in skipped[:5]:
+            print(f"    {u}")
+        if len(skipped) > 5:
+            print(f"    ...и ещё {len(skipped) - 5}")
+        print()
 
     if not urls:
         print("❌ В CSV не найдено валидных URL")

scanner/checks/form_checkbox.py

@@ -97,7 +97,7 @@ class FormCheckboxCheck(BaseCheck):
                     details={
                         "total_forms": forms_needing_consent,
                         "without_consent": forms_without_consent,
-                        "sample_texts": [f["checkboxes"][0] for f in forms_without_consent[:3]],
+                        "sample_texts": [f["checkboxes"][0] if f["checkboxes"] else "без чекбоксов" for f in forms_without_consent[:3]],
                     },
                 )
 

scanner/checks/privacy_link.py

@@ -63,6 +63,22 @@ class PrivacyLinkCheck(BaseCheck):
             # Проверяем, что ссылка ведет на рабочую страницу
             policy_url = found_links[0]["url"]
             try:
+                # Проверяем, не является ли URL документом (PDF, DOCX, XLSX)
+                doc_extensions = ['.pdf', '.docx', '.doc', '.xlsx', '.xls', '.rtf', '.txt']
+                is_document = any(policy_url.lower().endswith(ext) for ext in doc_extensions)
+                
+                if is_document:
+                    # Документ для скачивания — это допустимо для политики
+                    return self._pass_result(
+                        message=f"Найдена ссылка на Политику конфиденциальности ({policy_url})",
+                        details={
+                            "policy_url": policy_url,
+                            "link_text": found_links[0]["text"],
+                            "is_document": True,
+                        },
+                    )
+
+                # Для HTML-страниц — проверяем доступность
                 await page.goto(policy_url, timeout=5000)
                 policy_text = await page.evaluate("() => document.body.innerText")
                 await page.goto(url, timeout=5000)  # Возвращаемся на главную

scanner/checks/server_location.py

@@ -5,6 +5,9 @@
 """
 from __future__ import annotations
 
+import socket
+from urllib.parse import urlparse
+
 from playwright.async_api import Page
 
 from scanner.checks.base import BaseCheck
@@ -20,65 +23,43 @@ class ServerLocationCheck(BaseCheck):
     risk_on_fail = RiskLevel.CRITICAL
     default_weight = 25
 
-    # Популярные иностранные провайдеры (можно расширять)
+    # Популярные русские DNS-серверы/хостеры
     RUSSIAN_HOSTERS = [
         "yandex", "mail.ru", "vk", "selectel", "timeweb", "beget",
         "reg.ru", "nic.ru", "rtcdn", "astra", "sbercloud",
         "cloud4russia", "гостех", "ростелеком", "мтс", "билайн",
-        "мегафон", "вымпелком", "ростелеком", "netology",
+        "мегафон", "вымпелком", "netology",
+        "yandex.net", "mail.ru", "vk.com", "selectel.cloud",
     ]
 
     async def check(self, page: Page, url: str) -> CheckResult:
         try:
-            # Получаем IP-адрес сервера через JavaScript
+            # Извлекаем домен из URL
-            ip_info = await page.evaluate("""() => {
+            parsed = urlparse(url)
-                return new Promise((resolve) => {
+            hostname = parsed.hostname or ""
-                    // Пробуем определить IP через несколько методов
+            
-                    const methods = [
+            if not hostname:
-                        () => fetch('https://api.ipify.org?format=json')
+                return self._skip_result(message="Не удалось извлечь домен из URL.")
-                            .then(r => r.json()).then(d => ({ip: d.ip, method: 'ipify'})),
-                        () => fetch('https://api.my-ip.io/v2/json')
-                            .then(r => r.json()).then(d => ({ip: d.ip, method: 'my-ip-io'})),
-                    ];
 
-                    // Пробуем каждый метод
+            # Получаем IP через DNS
-                    (async () => {
+            try:
-                        for (const method of methods) {
+                ip = socket.gethostbyname(hostname)
-                            try {
+            except socket.gaierror:
-                                const result = await method();
+                return self._skip_result(message=f"Не удалось разрешить DNS для {hostname}.")
-                                resolve(result);
-                                return;
-                            } catch (e) {}
-                        }
-                        resolve(null);
-                    })();
-                });
-            }""")
 
-            if not ip_info:
+            # Определяем страну по IP
-                return self._skip_result(
-                    message="Не удалось определить IP-адрес сервера.",
-                )
-
-            ip = ip_info.get("ip", "")
-            method = ip_info.get("method", "")
-
-            # Определяем страну по IP (используем публичный API)
             country = await self._get_country(ip)
 
             if not country:
-                return self._skip_result(
+                return self._skip_result(message=f"Не удалось определить страну для IP {ip}.")
-                    message=f"Не удалось определить страну для IP {ip}.",
-                )
 
             # Проверяем, Россия ли это
-            if country.lower() == "ru" or "Россия" in country:
+            if country.lower() == "ru" or "Россия" in country or "russia" in country.lower():
                 return self._pass_result(
                     message=f"Сервер находится в России (IP: {ip}, Страна: {country})",
                     details={
                         "ip": ip,
                         "country": country,
-                        "method": method,
                     },
                 )
 
@@ -91,7 +72,6 @@ class ServerLocationCheck(BaseCheck):
                 details={
                     "ip": ip,
                     "country": country,
-                    "method": method,
                 },
             )