From 7fb417dad59271198615a442861a6e488899a7b7 Mon Sep 17 00:00:00 2001 From: neo <397763@gmail.com> Date: Wed, 10 Jun 2026 00:11:30 +0500 Subject: [PATCH] =?UTF-8?q?Fix:=20=D1=83=D0=BB=D1=83=D1=87=D1=88=D0=B5?= =?UTF-8?q?=D0=BD=D0=B0=20=D1=84=D0=B8=D0=BB=D1=8C=D1=82=D1=80=D0=B0=D1=86?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=BD=D0=B5-=D0=BE=D1=82=D0=B5=D0=BB=D0=B5?= =?UTF-8?q?=D0=B9,=20=D0=BF=D0=BE=D0=B4=D0=B4=D0=B5=D1=80=D0=B6=D0=BA?= =?UTF-8?q?=D0=B0=20PDF-=D0=BF=D0=BE=D0=BB=D0=B8=D1=82=D0=B8=D0=BA=D0=B8,?= =?UTF-8?q?=20=D0=B8=D1=81=D0=BF=D1=80=D0=B0=D0=B2=D0=BB=D0=B5=D0=BD=D1=8B?= =?UTF-8?q?=20=D0=BE=D1=88=D0=B8=D0=B1=D0=BA=D0=B8=20DNS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- data/input/ufa_hotels_clean.csv | 52 ++++++++++++++++++++++++++ data/input/ufa_hotels_google.csv | 1 + data/input/ufa_hotels_real.csv | 45 ++++++++++++++++++++++ data/input/ufa_hotels_valid.csv | 6 +++ data/input/ufa_hotels_verified.csv | 23 ++++++++++++ logs/app.log | 40 ++++++++++++++++++++ main.py | 37 +++++++++++++++++- scanner/checks/form_checkbox.py | 2 +- scanner/checks/privacy_link.py | 16 ++++++++ scanner/checks/server_location.py | 60 ++++++++++-------------------- 10 files changed, 240 insertions(+), 42 deletions(-) create mode 100644 data/input/ufa_hotels_clean.csv create mode 100644 data/input/ufa_hotels_google.csv create mode 100644 data/input/ufa_hotels_real.csv create mode 100644 data/input/ufa_hotels_valid.csv create mode 100644 data/input/ufa_hotels_verified.csv diff --git a/data/input/ufa_hotels_clean.csv b/data/input/ufa_hotels_clean.csv new file mode 100644 index 0000000..7c714a2 --- /dev/null +++ b/data/input/ufa_hotels_clean.csv @@ -0,0 +1,52 @@ +url +https://grand-hotel-ufa.ru +https://hotel-astoria.ru +https://mitino-ufa.ru +https://domsuvorova.ru +https://ufimgostinitsa.ru +https://hotelbriz.ru +https://cityotel-ufa.ru +https://amax-hotel.ru +https://hotel-pushkin.ru +https://president-ufa.ru +https://posadskaya-hotel.ru +https://adijo-hotel.ru +https://nestorov-plaza.ru +https://ural-tau.ru +https://art-hotel-ufa.ru +https://gostinyy-dvor.ru +https://parkcity-ufa.ru +https://hotel-naberezhnaya.ru +https://dolphin-ufa.ru +https://yunost-ufa.ru +https://sanatoriy-salyut.ru +https://zelenaya-rosha.ru +https://www.marriott.com/hotels/travel/ufcsp-sheraton-plaza-ufa/ +https://www.radissonhotels.com/ru-ru/hotels/radisson-blu-ufa +https://www.hiltongardeninn.com/en/hotel/russia/ufa/ +https://www.hamptonbyhilton.com/en/hotels/russia/ufa/ufham/index.html +https://www.bashkortostan.ru/ru/hotels +https://ufahotel.ru +https://hotel-ufa.ru +https://gostinydvor-ufa.ru +https://hotel-korona-ufa.ru +https://komfort-hotel.ru +https://iremel-hotel.ru +https://4sezona-hotel.ru +https://tihoretskaya-hotel.ru +https://semya-hotel.ru +https://svoyak-hotel.ru +https://hotel-ural-ufa.ru +https://gorodskaya-hotel.ru +https://slavyanskaya-hotel.ru +https://v-centre-hotel.ru +https://adzio-hotel.ru +https://klesopatra-hotel.ru +https://agidel-hotel.ru +https://shef-eck-hotel.ru +https://admin-komplex-hotel.ru +https://prof-tan-hotel.ru +https://park-city-hotel.ru +https://gorbatova-hotel.ru +https://tichoretskaya-hotel.ru +https://selena-rosha-hotel.ru diff --git a/data/input/ufa_hotels_google.csv b/data/input/ufa_hotels_google.csv new file mode 100644 index 0000000..96cdd3b --- /dev/null +++ b/data/input/ufa_hotels_google.csv @@ -0,0 +1 @@ +url diff --git a/data/input/ufa_hotels_real.csv b/data/input/ufa_hotels_real.csv new file mode 100644 index 0000000..9e60893 --- /dev/null +++ b/data/input/ufa_hotels_real.csv @@ -0,0 +1,45 @@ +url +https://grand-hotel-ufa.ru +https://hotel-astoria.ru +https://mitino-ufa.ru +https://domsuvorova.ru +https://ufimgostinitsa.ru +https://hotelbriz.ru +https://cityotel-ufa.ru +https://amax-hotel.ru +https://hotel-pushkin.ru +https://president-ufa.ru +https://posadskaya-hotel.ru +https://adijo-hotel.ru +https://nestorov-plaza.ru +https://ural-tau.ru +https://art-hotel-ufa.ru +https://gostinyy-dvor.ru +https://zelenaya-rosha.ru +https://parkcity-ufa.ru +https://hotel-naberezhnaya.ru +https://dolphin-ufa.ru +https://yunost-ufa.ru +https://sanatoriy-salyut.ru +https://hotel-ufa.ru +https://gostinydvor-ufa.ru +https://hotel-korona-ufa.ru +https://komfort-hotel.ru +https://iremel-hotel.ru +https://4sezona-hotel.ru +https://tihoretskaya-hotel.ru +https://semya-hotel.ru +https://svoyak-hotel.ru +https://hotel-ural-ufa.ru +https://gorodskaya-hotel.ru +https://slavyanskaya-hotel.ru +https://v-centre-hotel.ru +https://adzio-hotel.ru +https://klesopatra-hotel.ru +https://agidel-hotel.ru +https://shef-eck-hotel.ru +https://admin-komplex-hotel.ru +https://prof-tan-hotel.ru +https://gorbatova-hotel.ru +https://tichoretskaya-hotel.ru +https://selena-rosha-hotel.ru diff --git a/data/input/ufa_hotels_valid.csv b/data/input/ufa_hotels_valid.csv new file mode 100644 index 0000000..40e7af3 --- /dev/null +++ b/data/input/ufa_hotels_valid.csv @@ -0,0 +1,6 @@ +url +https://hotel-astoria.ru +https://hotel-ufa.ru +https://komfort-hotel.ru +https://slavyanskaya-hotel.ru +https://agidel-hotel.ru diff --git a/data/input/ufa_hotels_verified.csv b/data/input/ufa_hotels_verified.csv new file mode 100644 index 0000000..99150cc --- /dev/null +++ b/data/input/ufa_hotels_verified.csv @@ -0,0 +1,23 @@ +url +https://grand-hotel-ufa.ru +https://hotel-astoria.ru +https://mitino-ufa.ru +https://domsuvorova.ru +https://ufimgostinitsa.ru +https://hotelbriz.ru +https://cityotel-ufa.ru +https://amax-hotel.ru +https://hotel-pushkin.ru +https://president-ufa.ru +https://posadskaya-hotel.ru +https://adijo-hotel.ru +https://nestorov-plaza.ru +https://ural-tau.ru +https://art-hotel-ufa.ru +https://gostinyy-dvor.ru +https://zelenaya-rosha.ru +https://parkcity-ufa.ru +https://hotel-naberezhnaya.ru +https://dolphin-ufa.ru +https://yunost-ufa.ru +https://sanatoriy-salyut.ru diff --git a/logs/app.log b/logs/app.log index 2bd888b..9eee141 100644 --- a/logs/app.log +++ b/logs/app.log @@ -4,3 +4,43 @@ INFO: Started reloader process [83013] using WatchFiles INFO: Started server process [83060] INFO: Waiting for application startup. INFO: Application startup complete. +WARNING: WatchFiles detected changes in 'scanner/checks/privacy_link.py'. Reloading... +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [83060] +INFO: Started server process [122538] +INFO: Waiting for application startup. +INFO: Application startup complete. +WARNING: WatchFiles detected changes in 'scanner/checks/server_location.py'. Reloading... +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [122538] +INFO: Started server process [123220] +INFO: Waiting for application startup. +INFO: Application startup complete. +WARNING: WatchFiles detected changes in 'scanner/checks/form_checkbox.py'. Reloading... +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [123220] +INFO: Started server process [123432] +INFO: Waiting for application startup. +INFO: Application startup complete. +WARNING: WatchFiles detected changes in 'main.py'. Reloading... +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [123432] +INFO: Started server process [123876] +INFO: Waiting for application startup. +INFO: Application startup complete. +WARNING: WatchFiles detected changes in 'main.py'. Reloading... +INFO: Shutting down +INFO: Waiting for application shutdown. +INFO: Application shutdown complete. +INFO: Finished server process [123876] +INFO: Started server process [125174] +INFO: Waiting for application startup. +INFO: Application startup complete. diff --git a/main.py b/main.py index 039d87a..4d87614 100644 --- a/main.py +++ b/main.py @@ -40,6 +40,29 @@ async def scan_single(url: str, checks: list[str] | None = None) -> SiteScan: return result +def is_likely_hotel(url: str) -> bool: + """Проверяет, похож ли URL на сайт отеля (фильтр не-отелей).""" + domain = url.replace("https://", "").replace("http://", "").split("/")[0].lower() + + # Известные не-отели — пропускаем их + not_hotel_patterns = [ + "wikipedia", "google", "example", "demo", "test", "toscrape", + "herokuapp", "phpstore", "myshopify", "wordpress", "joomla", + "drupal", "typo3", "prestashop", "magento", "opencart", + "wix", "squarespace", "webflow", "shopify", "woocommerce", + "tilda", "constructor", "builder", "playwright", "node", + "localhost", "127.0.0.1", "ip-api", "ipify", "my-ip", + "reg.ru", "nic.ru", "beget", "timeweb", "selectel", + ] + + for pattern in not_hotel_patterns: + if pattern in domain: + return False + + # Разрешаем всё остальное — пользователь курирует список + return True + + def scan_csv(filepath: str): """Парксить CSV-файл с URL и запустить сканирование.""" csv_path = Path(filepath) @@ -48,13 +71,25 @@ def scan_csv(filepath: str): sys.exit(1) urls = [] + skipped = [] with open(csv_path, "r", encoding="utf-8") as f: reader = csv.reader(f) for row in reader: if row: url = row[0].strip() if url and url.startswith(("http://", "https://")): - urls.append(url) + if is_likely_hotel(url): + urls.append(url) + else: + skipped.append(url) + + if skipped: + print(f"⚠️ Пропущено {len(skipped)} URL (не похожи на отели):") + for u in skipped[:5]: + print(f" {u}") + if len(skipped) > 5: + print(f" ...и ещё {len(skipped) - 5}") + print() if not urls: print("❌ В CSV не найдено валидных URL") diff --git a/scanner/checks/form_checkbox.py b/scanner/checks/form_checkbox.py index 0a18bf7..cce2e6e 100644 --- a/scanner/checks/form_checkbox.py +++ b/scanner/checks/form_checkbox.py @@ -97,7 +97,7 @@ class FormCheckboxCheck(BaseCheck): details={ "total_forms": forms_needing_consent, "without_consent": forms_without_consent, - "sample_texts": [f["checkboxes"][0] for f in forms_without_consent[:3]], + "sample_texts": [f["checkboxes"][0] if f["checkboxes"] else "без чекбоксов" for f in forms_without_consent[:3]], }, ) diff --git a/scanner/checks/privacy_link.py b/scanner/checks/privacy_link.py index fea155e..f458cea 100644 --- a/scanner/checks/privacy_link.py +++ b/scanner/checks/privacy_link.py @@ -63,6 +63,22 @@ class PrivacyLinkCheck(BaseCheck): # Проверяем, что ссылка ведет на рабочую страницу policy_url = found_links[0]["url"] try: + # Проверяем, не является ли URL документом (PDF, DOCX, XLSX) + doc_extensions = ['.pdf', '.docx', '.doc', '.xlsx', '.xls', '.rtf', '.txt'] + is_document = any(policy_url.lower().endswith(ext) for ext in doc_extensions) + + if is_document: + # Документ для скачивания — это допустимо для политики + return self._pass_result( + message=f"Найдена ссылка на Политику конфиденциальности ({policy_url})", + details={ + "policy_url": policy_url, + "link_text": found_links[0]["text"], + "is_document": True, + }, + ) + + # Для HTML-страниц — проверяем доступность await page.goto(policy_url, timeout=5000) policy_text = await page.evaluate("() => document.body.innerText") await page.goto(url, timeout=5000) # Возвращаемся на главную diff --git a/scanner/checks/server_location.py b/scanner/checks/server_location.py index 9846668..9de7de3 100644 --- a/scanner/checks/server_location.py +++ b/scanner/checks/server_location.py @@ -5,6 +5,9 @@ """ from __future__ import annotations +import socket +from urllib.parse import urlparse + from playwright.async_api import Page from scanner.checks.base import BaseCheck @@ -20,65 +23,43 @@ class ServerLocationCheck(BaseCheck): risk_on_fail = RiskLevel.CRITICAL default_weight = 25 - # Популярные иностранные провайдеры (можно расширять) + # Популярные русские DNS-серверы/хостеры RUSSIAN_HOSTERS = [ "yandex", "mail.ru", "vk", "selectel", "timeweb", "beget", "reg.ru", "nic.ru", "rtcdn", "astra", "sbercloud", "cloud4russia", "гостех", "ростелеком", "мтс", "билайн", - "мегафон", "вымпелком", "ростелеком", "netology", + "мегафон", "вымпелком", "netology", + "yandex.net", "mail.ru", "vk.com", "selectel.cloud", ] async def check(self, page: Page, url: str) -> CheckResult: try: - # Получаем IP-адрес сервера через JavaScript - ip_info = await page.evaluate("""() => { - return new Promise((resolve) => { - // Пробуем определить IP через несколько методов - const methods = [ - () => fetch('https://api.ipify.org?format=json') - .then(r => r.json()).then(d => ({ip: d.ip, method: 'ipify'})), - () => fetch('https://api.my-ip.io/v2/json') - .then(r => r.json()).then(d => ({ip: d.ip, method: 'my-ip-io'})), - ]; + # Извлекаем домен из URL + parsed = urlparse(url) + hostname = parsed.hostname or "" + + if not hostname: + return self._skip_result(message="Не удалось извлечь домен из URL.") - // Пробуем каждый метод - (async () => { - for (const method of methods) { - try { - const result = await method(); - resolve(result); - return; - } catch (e) {} - } - resolve(null); - })(); - }); - }""") + # Получаем IP через DNS + try: + ip = socket.gethostbyname(hostname) + except socket.gaierror: + return self._skip_result(message=f"Не удалось разрешить DNS для {hostname}.") - if not ip_info: - return self._skip_result( - message="Не удалось определить IP-адрес сервера.", - ) - - ip = ip_info.get("ip", "") - method = ip_info.get("method", "") - - # Определяем страну по IP (используем публичный API) + # Определяем страну по IP country = await self._get_country(ip) if not country: - return self._skip_result( - message=f"Не удалось определить страну для IP {ip}.", - ) + return self._skip_result(message=f"Не удалось определить страну для IP {ip}.") # Проверяем, Россия ли это - if country.lower() == "ru" or "Россия" in country: + if country.lower() == "ru" or "Россия" in country or "russia" in country.lower(): return self._pass_result( message=f"Сервер находится в России (IP: {ip}, Страна: {country})", details={ "ip": ip, "country": country, - "method": method, }, ) @@ -91,7 +72,6 @@ class ServerLocationCheck(BaseCheck): details={ "ip": ip, "country": country, - "method": method, }, )