Fix: улучшена фильтрация не-отелей, поддержка PDF-политики, исправлены ошибки DNS
This commit is contained in:
neo committed
1 parent
928637ad25
commit
7fb417dad5
10 files changed
+240
-42
No files matched your search
@@ -0,0 +1,52 @@
|
||||
url
|
||||
https://grand-hotel-ufa.ru
|
||||
https://hotel-astoria.ru
|
||||
https://mitino-ufa.ru
|
||||
https://domsuvorova.ru
|
||||
https://ufimgostinitsa.ru
|
||||
https://hotelbriz.ru
|
||||
https://cityotel-ufa.ru
|
||||
https://amax-hotel.ru
|
||||
https://hotel-pushkin.ru
|
||||
https://president-ufa.ru
|
||||
https://posadskaya-hotel.ru
|
||||
https://adijo-hotel.ru
|
||||
https://nestorov-plaza.ru
|
||||
https://ural-tau.ru
|
||||
https://art-hotel-ufa.ru
|
||||
https://gostinyy-dvor.ru
|
||||
https://parkcity-ufa.ru
|
||||
https://hotel-naberezhnaya.ru
|
||||
https://dolphin-ufa.ru
|
||||
https://yunost-ufa.ru
|
||||
https://sanatoriy-salyut.ru
|
||||
https://zelenaya-rosha.ru
|
||||
https://www.marriott.com/hotels/travel/ufcsp-sheraton-plaza-ufa/
|
||||
https://www.radissonhotels.com/ru-ru/hotels/radisson-blu-ufa
|
||||
https://www.hiltongardeninn.com/en/hotel/russia/ufa/
|
||||
https://www.hamptonbyhilton.com/en/hotels/russia/ufa/ufham/index.html
|
||||
https://www.bashkortostan.ru/ru/hotels
|
||||
https://ufahotel.ru
|
||||
https://hotel-ufa.ru
|
||||
https://gostinydvor-ufa.ru
|
||||
https://hotel-korona-ufa.ru
|
||||
https://komfort-hotel.ru
|
||||
https://iremel-hotel.ru
|
||||
https://4sezona-hotel.ru
|
||||
https://tihoretskaya-hotel.ru
|
||||
https://semya-hotel.ru
|
||||
https://svoyak-hotel.ru
|
||||
https://hotel-ural-ufa.ru
|
||||
https://gorodskaya-hotel.ru
|
||||
https://slavyanskaya-hotel.ru
|
||||
https://v-centre-hotel.ru
|
||||
https://adzio-hotel.ru
|
||||
https://klesopatra-hotel.ru
|
||||
https://agidel-hotel.ru
|
||||
https://shef-eck-hotel.ru
|
||||
https://admin-komplex-hotel.ru
|
||||
https://prof-tan-hotel.ru
|
||||
https://park-city-hotel.ru
|
||||
https://gorbatova-hotel.ru
|
||||
https://tichoretskaya-hotel.ru
|
||||
https://selena-rosha-hotel.ru
|
||||
|
@@ -0,0 +1 @@
|
||||
url
|
||||
|
@@ -0,0 +1,45 @@
|
||||
url
|
||||
https://grand-hotel-ufa.ru
|
||||
https://hotel-astoria.ru
|
||||
https://mitino-ufa.ru
|
||||
https://domsuvorova.ru
|
||||
https://ufimgostinitsa.ru
|
||||
https://hotelbriz.ru
|
||||
https://cityotel-ufa.ru
|
||||
https://amax-hotel.ru
|
||||
https://hotel-pushkin.ru
|
||||
https://president-ufa.ru
|
||||
https://posadskaya-hotel.ru
|
||||
https://adijo-hotel.ru
|
||||
https://nestorov-plaza.ru
|
||||
https://ural-tau.ru
|
||||
https://art-hotel-ufa.ru
|
||||
https://gostinyy-dvor.ru
|
||||
https://zelenaya-rosha.ru
|
||||
https://parkcity-ufa.ru
|
||||
https://hotel-naberezhnaya.ru
|
||||
https://dolphin-ufa.ru
|
||||
https://yunost-ufa.ru
|
||||
https://sanatoriy-salyut.ru
|
||||
https://hotel-ufa.ru
|
||||
https://gostinydvor-ufa.ru
|
||||
https://hotel-korona-ufa.ru
|
||||
https://komfort-hotel.ru
|
||||
https://iremel-hotel.ru
|
||||
https://4sezona-hotel.ru
|
||||
https://tihoretskaya-hotel.ru
|
||||
https://semya-hotel.ru
|
||||
https://svoyak-hotel.ru
|
||||
https://hotel-ural-ufa.ru
|
||||
https://gorodskaya-hotel.ru
|
||||
https://slavyanskaya-hotel.ru
|
||||
https://v-centre-hotel.ru
|
||||
https://adzio-hotel.ru
|
||||
https://klesopatra-hotel.ru
|
||||
https://agidel-hotel.ru
|
||||
https://shef-eck-hotel.ru
|
||||
https://admin-komplex-hotel.ru
|
||||
https://prof-tan-hotel.ru
|
||||
https://gorbatova-hotel.ru
|
||||
https://tichoretskaya-hotel.ru
|
||||
https://selena-rosha-hotel.ru
|
||||
|
@@ -0,0 +1,6 @@
|
||||
url
|
||||
https://hotel-astoria.ru
|
||||
https://hotel-ufa.ru
|
||||
https://komfort-hotel.ru
|
||||
https://slavyanskaya-hotel.ru
|
||||
https://agidel-hotel.ru
|
||||
|
@@ -0,0 +1,23 @@
|
||||
url
|
||||
https://grand-hotel-ufa.ru
|
||||
https://hotel-astoria.ru
|
||||
https://mitino-ufa.ru
|
||||
https://domsuvorova.ru
|
||||
https://ufimgostinitsa.ru
|
||||
https://hotelbriz.ru
|
||||
https://cityotel-ufa.ru
|
||||
https://amax-hotel.ru
|
||||
https://hotel-pushkin.ru
|
||||
https://president-ufa.ru
|
||||
https://posadskaya-hotel.ru
|
||||
https://adijo-hotel.ru
|
||||
https://nestorov-plaza.ru
|
||||
https://ural-tau.ru
|
||||
https://art-hotel-ufa.ru
|
||||
https://gostinyy-dvor.ru
|
||||
https://zelenaya-rosha.ru
|
||||
https://parkcity-ufa.ru
|
||||
https://hotel-naberezhnaya.ru
|
||||
https://dolphin-ufa.ru
|
||||
https://yunost-ufa.ru
|
||||
https://sanatoriy-salyut.ru
|
||||
|
@@ -4,3 +4,43 @@ INFO: Started reloader process [83013] using WatchFiles
|
||||
INFO: Started server process [83060]
|
||||
INFO: Waiting for application startup.
|
||||
INFO: Application startup complete.
|
||||
WARNING: WatchFiles detected changes in 'scanner/checks/privacy_link.py'. Reloading...
|
||||
INFO: Shutting down
|
||||
INFO: Waiting for application shutdown.
|
||||
INFO: Application shutdown complete.
|
||||
INFO: Finished server process [83060]
|
||||
INFO: Started server process [122538]
|
||||
INFO: Waiting for application startup.
|
||||
INFO: Application startup complete.
|
||||
WARNING: WatchFiles detected changes in 'scanner/checks/server_location.py'. Reloading...
|
||||
INFO: Shutting down
|
||||
INFO: Waiting for application shutdown.
|
||||
INFO: Application shutdown complete.
|
||||
INFO: Finished server process [122538]
|
||||
INFO: Started server process [123220]
|
||||
INFO: Waiting for application startup.
|
||||
INFO: Application startup complete.
|
||||
WARNING: WatchFiles detected changes in 'scanner/checks/form_checkbox.py'. Reloading...
|
||||
INFO: Shutting down
|
||||
INFO: Waiting for application shutdown.
|
||||
INFO: Application shutdown complete.
|
||||
INFO: Finished server process [123220]
|
||||
INFO: Started server process [123432]
|
||||
INFO: Waiting for application startup.
|
||||
INFO: Application startup complete.
|
||||
WARNING: WatchFiles detected changes in 'main.py'. Reloading...
|
||||
INFO: Shutting down
|
||||
INFO: Waiting for application shutdown.
|
||||
INFO: Application shutdown complete.
|
||||
INFO: Finished server process [123432]
|
||||
INFO: Started server process [123876]
|
||||
INFO: Waiting for application startup.
|
||||
INFO: Application startup complete.
|
||||
WARNING: WatchFiles detected changes in 'main.py'. Reloading...
|
||||
INFO: Shutting down
|
||||
INFO: Waiting for application shutdown.
|
||||
INFO: Application shutdown complete.
|
||||
INFO: Finished server process [123876]
|
||||
INFO: Started server process [125174]
|
||||
INFO: Waiting for application startup.
|
||||
INFO: Application startup complete.
|
||||
@@ -40,6 +40,29 @@ async def scan_single(url: str, checks: list[str] | None = None) -> SiteScan:
|
||||
return result
|
||||
|
||||
|
||||
def is_likely_hotel(url: str) -> bool:
|
||||
"""Проверяет, похож ли URL на сайт отеля (фильтр не-отелей)."""
|
||||
domain = url.replace("https://", "").replace("http://", "").split("/")[0].lower()
|
||||
|
||||
# Известные не-отели — пропускаем их
|
||||
not_hotel_patterns = [
|
||||
"wikipedia", "google", "example", "demo", "test", "toscrape",
|
||||
"herokuapp", "phpstore", "myshopify", "wordpress", "joomla",
|
||||
"drupal", "typo3", "prestashop", "magento", "opencart",
|
||||
"wix", "squarespace", "webflow", "shopify", "woocommerce",
|
||||
"tilda", "constructor", "builder", "playwright", "node",
|
||||
"localhost", "127.0.0.1", "ip-api", "ipify", "my-ip",
|
||||
"reg.ru", "nic.ru", "beget", "timeweb", "selectel",
|
||||
]
|
||||
|
||||
for pattern in not_hotel_patterns:
|
||||
if pattern in domain:
|
||||
return False
|
||||
|
||||
# Разрешаем всё остальное — пользователь курирует список
|
||||
return True
|
||||
|
||||
|
||||
def scan_csv(filepath: str):
|
||||
"""Парксить CSV-файл с URL и запустить сканирование."""
|
||||
csv_path = Path(filepath)
|
||||
@@ -48,13 +71,25 @@ def scan_csv(filepath: str):
|
||||
sys.exit(1)
|
||||
|
||||
urls = []
|
||||
skipped = []
|
||||
with open(csv_path, "r", encoding="utf-8") as f:
|
||||
reader = csv.reader(f)
|
||||
for row in reader:
|
||||
if row:
|
||||
url = row[0].strip()
|
||||
if url and url.startswith(("http://", "https://")):
|
||||
urls.append(url)
|
||||
if is_likely_hotel(url):
|
||||
urls.append(url)
|
||||
else:
|
||||
skipped.append(url)
|
||||
|
||||
if skipped:
|
||||
print(f"⚠️ Пропущено {len(skipped)} URL (не похожи на отели):")
|
||||
for u in skipped[:5]:
|
||||
print(f" {u}")
|
||||
if len(skipped) > 5:
|
||||
print(f" ...и ещё {len(skipped) - 5}")
|
||||
print()
|
||||
|
||||
if not urls:
|
||||
print("❌ В CSV не найдено валидных URL")
|
||||
|
||||
@@ -97,7 +97,7 @@ class FormCheckboxCheck(BaseCheck):
|
||||
details={
|
||||
"total_forms": forms_needing_consent,
|
||||
"without_consent": forms_without_consent,
|
||||
"sample_texts": [f["checkboxes"][0] for f in forms_without_consent[:3]],
|
||||
"sample_texts": [f["checkboxes"][0] if f["checkboxes"] else "без чекбоксов" for f in forms_without_consent[:3]],
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@@ -63,6 +63,22 @@ class PrivacyLinkCheck(BaseCheck):
|
||||
# Проверяем, что ссылка ведет на рабочую страницу
|
||||
policy_url = found_links[0]["url"]
|
||||
try:
|
||||
# Проверяем, не является ли URL документом (PDF, DOCX, XLSX)
|
||||
doc_extensions = ['.pdf', '.docx', '.doc', '.xlsx', '.xls', '.rtf', '.txt']
|
||||
is_document = any(policy_url.lower().endswith(ext) for ext in doc_extensions)
|
||||
|
||||
if is_document:
|
||||
# Документ для скачивания — это допустимо для политики
|
||||
return self._pass_result(
|
||||
message=f"Найдена ссылка на Политику конфиденциальности ({policy_url})",
|
||||
details={
|
||||
"policy_url": policy_url,
|
||||
"link_text": found_links[0]["text"],
|
||||
"is_document": True,
|
||||
},
|
||||
)
|
||||
|
||||
# Для HTML-страниц — проверяем доступность
|
||||
await page.goto(policy_url, timeout=5000)
|
||||
policy_text = await page.evaluate("() => document.body.innerText")
|
||||
await page.goto(url, timeout=5000) # Возвращаемся на главную
|
||||
|
||||
@@ -5,6 +5,9 @@
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import socket
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from playwright.async_api import Page
|
||||
|
||||
from scanner.checks.base import BaseCheck
|
||||
@@ -20,65 +23,43 @@ class ServerLocationCheck(BaseCheck):
|
||||
risk_on_fail = RiskLevel.CRITICAL
|
||||
default_weight = 25
|
||||
|
||||
# Популярные иностранные провайдеры (можно расширять)
|
||||
# Популярные русские DNS-серверы/хостеры
|
||||
RUSSIAN_HOSTERS = [
|
||||
"yandex", "mail.ru", "vk", "selectel", "timeweb", "beget",
|
||||
"reg.ru", "nic.ru", "rtcdn", "astra", "sbercloud",
|
||||
"cloud4russia", "гостех", "ростелеком", "мтс", "билайн",
|
||||
"мегафон", "вымпелком", "ростелеком", "netology",
|
||||
"мегафон", "вымпелком", "netology",
|
||||
"yandex.net", "mail.ru", "vk.com", "selectel.cloud",
|
||||
]
|
||||
|
||||
async def check(self, page: Page, url: str) -> CheckResult:
|
||||
try:
|
||||
# Получаем IP-адрес сервера через JavaScript
|
||||
ip_info = await page.evaluate("""() => {
|
||||
return new Promise((resolve) => {
|
||||
// Пробуем определить IP через несколько методов
|
||||
const methods = [
|
||||
() => fetch('https://api.ipify.org?format=json')
|
||||
.then(r => r.json()).then(d => ({ip: d.ip, method: 'ipify'})),
|
||||
() => fetch('https://api.my-ip.io/v2/json')
|
||||
.then(r => r.json()).then(d => ({ip: d.ip, method: 'my-ip-io'})),
|
||||
];
|
||||
# Извлекаем домен из URL
|
||||
parsed = urlparse(url)
|
||||
hostname = parsed.hostname or ""
|
||||
|
||||
if not hostname:
|
||||
return self._skip_result(message="Не удалось извлечь домен из URL.")
|
||||
|
||||
// Пробуем каждый метод
|
||||
(async () => {
|
||||
for (const method of methods) {
|
||||
try {
|
||||
const result = await method();
|
||||
resolve(result);
|
||||
return;
|
||||
} catch (e) {}
|
||||
}
|
||||
resolve(null);
|
||||
})();
|
||||
});
|
||||
}""")
|
||||
# Получаем IP через DNS
|
||||
try:
|
||||
ip = socket.gethostbyname(hostname)
|
||||
except socket.gaierror:
|
||||
return self._skip_result(message=f"Не удалось разрешить DNS для {hostname}.")
|
||||
|
||||
if not ip_info:
|
||||
return self._skip_result(
|
||||
message="Не удалось определить IP-адрес сервера.",
|
||||
)
|
||||
|
||||
ip = ip_info.get("ip", "")
|
||||
method = ip_info.get("method", "")
|
||||
|
||||
# Определяем страну по IP (используем публичный API)
|
||||
# Определяем страну по IP
|
||||
country = await self._get_country(ip)
|
||||
|
||||
if not country:
|
||||
return self._skip_result(
|
||||
message=f"Не удалось определить страну для IP {ip}.",
|
||||
)
|
||||
return self._skip_result(message=f"Не удалось определить страну для IP {ip}.")
|
||||
|
||||
# Проверяем, Россия ли это
|
||||
if country.lower() == "ru" or "Россия" in country:
|
||||
if country.lower() == "ru" or "Россия" in country or "russia" in country.lower():
|
||||
return self._pass_result(
|
||||
message=f"Сервер находится в России (IP: {ip}, Страна: {country})",
|
||||
details={
|
||||
"ip": ip,
|
||||
"country": country,
|
||||
"method": method,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -91,7 +72,6 @@ class ServerLocationCheck(BaseCheck):
|
||||
details={
|
||||
"ip": ip,
|
||||
"country": country,
|
||||
"method": method,
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
Reference in new issue
Block a user