384 lines
17 KiB
Python
384 lines
17 KiB
Python
import json
|
||
import os
|
||
from dataclasses import dataclass, field
|
||
from pathlib import Path
|
||
|
||
from dotenv import load_dotenv
|
||
|
||
load_dotenv()
|
||
|
||
|
||
TRUE_VALUES = {"1", "true", "yes", "on"}
|
||
|
||
|
||
# Хелперы для чтения env-переменных с приведением типов
|
||
|
||
def _env_str(name: str, default: str) -> str:
|
||
value = os.getenv(name)
|
||
return value if value is not None else default
|
||
|
||
|
||
def _env_optional_str(name: str) -> str | None:
|
||
value = os.getenv(name)
|
||
if value is None:
|
||
return None
|
||
value = value.strip()
|
||
return value or None
|
||
|
||
|
||
def _env_path_str(name: str) -> str | None:
|
||
value = _env_optional_str(name)
|
||
if value is None:
|
||
return None
|
||
return str(Path(value).expanduser())
|
||
|
||
|
||
def _env_bool(name: str, default: bool) -> bool:
|
||
fallback = "true" if default else "false"
|
||
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
|
||
|
||
|
||
def _env_int(name: str, default: int) -> int:
|
||
return int(_env_str(name, str(default)).strip())
|
||
|
||
|
||
def _env_float(name: str, default: float) -> float:
|
||
return float(_env_str(name, str(default)).strip())
|
||
|
||
|
||
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
|
||
|
||
@dataclass(slots=True)
|
||
class FingerprintConfig:
|
||
user_agent: str = (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/135.0.0.0 Safari/537.36"
|
||
)
|
||
viewport_presets: tuple[dict[str, int], ...] = (
|
||
{"width": 1920, "height": 1080},
|
||
{"width": 1600, "height": 900},
|
||
{"width": 1536, "height": 864},
|
||
{"width": 1440, "height": 900},
|
||
{"width": 1366, "height": 768},
|
||
)
|
||
timezone_candidates: tuple[str, ...] = (
|
||
"America/New_York",
|
||
"America/Chicago",
|
||
"America/Los_Angeles",
|
||
)
|
||
locale: str = "en-US"
|
||
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
|
||
|
||
|
||
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
|
||
|
||
@dataclass(slots=True)
|
||
class CaptureConfig:
|
||
capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
|
||
max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
|
||
max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
|
||
|
||
|
||
# Конфиг пауз между действиями (имитация человека)
|
||
|
||
@dataclass(slots=True)
|
||
class HumanPaceConfig:
|
||
enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
|
||
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
|
||
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
|
||
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
|
||
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
|
||
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
|
||
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
|
||
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
|
||
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
|
||
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
|
||
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
|
||
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
|
||
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
|
||
|
||
|
||
# Конфиг сбора листинга (URL, лимиты страниц и машин)
|
||
|
||
@dataclass(slots=True)
|
||
class ListingConfig:
|
||
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
|
||
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
|
||
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
|
||
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
|
||
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
|
||
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
|
||
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
|
||
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
|
||
early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
|
||
# Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
|
||
# JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка
|
||
# или "runtime" для построения по runtime_config.filters.brands.
|
||
# Пустая строка = без сегментации (backward compatible).
|
||
listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
|
||
fast_segment_year_splits: bool = _env_bool("IAAI_FAST_SEGMENT_YEAR_SPLITS", True)
|
||
|
||
|
||
# Список брендов IAAI для автоматической сегментации.
|
||
# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
|
||
IAAI_DEFAULT_MAKES: tuple[str, ...] = (
|
||
"TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
|
||
"KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
|
||
"VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
|
||
"RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
|
||
"MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
|
||
"MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
|
||
"SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
|
||
"OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
|
||
"RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
|
||
"BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
|
||
"MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
|
||
)
|
||
|
||
# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
|
||
IAAI_PAGINATION_CEILING = 22_600
|
||
|
||
# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
|
||
_LARGE_MAKES: frozenset[str] = frozenset({
|
||
"TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
|
||
"KIA", "DODGE", "JEEP",
|
||
})
|
||
_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
|
||
(1900, 2012),
|
||
(2013, 2019),
|
||
(2020, 2027),
|
||
)
|
||
|
||
|
||
def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
|
||
"""Строит сегменты по переданному списку брендов.
|
||
|
||
Крупные бренды режутся по годовым диапазонам так же, как в ``auto``.
|
||
"""
|
||
segments: list[dict[str, str | int | None]] = []
|
||
seen: set[str] = set()
|
||
for raw_make in makes:
|
||
make = str(raw_make or "").strip().upper()
|
||
if not make or make in seen:
|
||
continue
|
||
seen.add(make)
|
||
if make in _LARGE_MAKES:
|
||
for yr_min, yr_max in _YEAR_SPLITS:
|
||
segments.append({"make": make, "year_min": yr_min, "year_max": yr_max})
|
||
else:
|
||
segments.append({"make": make, "year_min": None, "year_max": None})
|
||
return segments
|
||
|
||
|
||
def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
|
||
"""Строит HTTP-first сегменты без UI-фильтров годов.
|
||
|
||
Это ближе к iaai-fast: один бренд = один hidden-payload HTTP обход.
|
||
Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля.
|
||
"""
|
||
segments: list[dict[str, str | int | None]] = []
|
||
seen: set[str] = set()
|
||
for raw_make in makes:
|
||
make = str(raw_make or "").strip().upper()
|
||
if not make or make in seen:
|
||
continue
|
||
seen.add(make)
|
||
segments.append({"make": make, "year_min": None, "year_max": None})
|
||
return segments
|
||
|
||
|
||
def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
|
||
"""Парсит IAAI_LISTING_SEGMENTS в список сегментов.
|
||
|
||
Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
|
||
Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
|
||
Крупные бренды автоматически разбиваются по диапазонам годов.
|
||
"""
|
||
raw = raw.strip()
|
||
if not raw:
|
||
return []
|
||
if raw.lower() == "auto":
|
||
return build_listing_segments_for_makes(list(IAAI_DEFAULT_MAKES))
|
||
try:
|
||
data = json.loads(raw)
|
||
except (json.JSONDecodeError, ValueError):
|
||
return []
|
||
if not isinstance(data, list):
|
||
return []
|
||
segments = []
|
||
for item in data:
|
||
if isinstance(item, str):
|
||
segments.append({"make": item.upper(), "year_min": None, "year_max": None})
|
||
elif isinstance(item, dict):
|
||
segments.append({
|
||
"make": str(item.get("make") or "").upper() or None,
|
||
"year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
|
||
"year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
|
||
})
|
||
return segments
|
||
|
||
|
||
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
|
||
|
||
@dataclass(slots=True)
|
||
class DatabaseConfig:
|
||
url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
|
||
echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
|
||
pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
|
||
max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
|
||
pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
|
||
auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
|
||
|
||
|
||
# --- Конфиг Redis (URL для Celery broker) ---
|
||
|
||
@dataclass(slots=True)
|
||
class RedisConfig:
|
||
url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
|
||
socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
|
||
socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
|
||
health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
|
||
|
||
|
||
# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
|
||
|
||
@dataclass(slots=True)
|
||
class DiscoveryConfig:
|
||
mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
|
||
hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
|
||
hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
|
||
always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", False)
|
||
|
||
|
||
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
|
||
|
||
@dataclass(slots=True)
|
||
class CeleryConfig:
|
||
broker_url: str = _env_str("CELERY_BROKER_URL", "")
|
||
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
|
||
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
|
||
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
|
||
task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
|
||
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
|
||
worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
|
||
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
|
||
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
|
||
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
|
||
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
|
||
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
|
||
parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
|
||
fetch_concurrency: int = _env_int("IAAI_FETCH_CONCURRENCY", _env_int("IAAI_PARALLEL_TABS", 8))
|
||
parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
|
||
block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
|
||
|
||
|
||
@dataclass(slots=True)
|
||
class ScrapingProfileConfig:
|
||
name: str = _env_str("IAAI_SCRAPING_PROFILE", _env_str("IAAI_PROFILE", "stable")).strip().lower()
|
||
http_first: bool = _env_bool("IAAI_HTTP_FIRST", True)
|
||
browser_fallback_enabled: bool = _env_bool("IAAI_BROWSER_FALLBACK_ENABLED", True)
|
||
anonymous_bootstrap_enabled: bool = _env_bool("IAAI_ANONYMOUS_BOOTSTRAP_ENABLED", True)
|
||
challenge_refresh_enabled: bool = _env_bool("IAAI_CHALLENGE_REFRESH_ENABLED", True)
|
||
challenge_refresh_attempts: int = _env_int("IAAI_CHALLENGE_REFRESH_ATTEMPTS", 3)
|
||
listing_post_attempts: int = _env_int("IAAI_LISTING_POST_ATTEMPTS", 4)
|
||
request_jitter_max_s: float = _env_float("IAAI_REQUEST_JITTER_MAX_S", 0.15)
|
||
detail_retries: int | None = _env_int("IAAI_DETAIL_RETRIES", -1)
|
||
listing_retries: int | None = _env_int("IAAI_LISTING_RETRIES", -1)
|
||
|
||
def __post_init__(self) -> None:
|
||
if self.name == "fast":
|
||
if "IAAI_BROWSER_FALLBACK_ENABLED" not in os.environ:
|
||
self.browser_fallback_enabled = False
|
||
if "IAAI_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
|
||
self.anonymous_bootstrap_enabled = False
|
||
if "IAAI_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
|
||
self.challenge_refresh_attempts = 1
|
||
if "IAAI_LISTING_POST_ATTEMPTS" not in os.environ:
|
||
self.listing_post_attempts = 2
|
||
if "IAAI_REQUEST_JITTER_MAX_S" not in os.environ:
|
||
self.request_jitter_max_s = 0.0
|
||
if "IAAI_DETAIL_RETRIES" not in os.environ:
|
||
self.detail_retries = 1
|
||
if "IAAI_LISTING_RETRIES" not in os.environ:
|
||
self.listing_retries = 1
|
||
else:
|
||
if self.detail_retries is not None and self.detail_retries < 0:
|
||
self.detail_retries = None
|
||
if self.listing_retries is not None and self.listing_retries < 0:
|
||
self.listing_retries = None
|
||
|
||
|
||
# --- Конфиг прокси (server, username, password) ---
|
||
|
||
@dataclass(slots=True)
|
||
class ProxyConfig:
|
||
server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
|
||
username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
|
||
password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
|
||
|
||
@property
|
||
def enabled(self) -> bool:
|
||
return bool(self.server)
|
||
|
||
def to_playwright_dict(self) -> dict[str, str] | None:
|
||
if not self.server:
|
||
return None
|
||
result: dict[str, str] = {"server": self.server}
|
||
if self.username:
|
||
result["username"] = self.username
|
||
if self.password:
|
||
result["password"] = self.password
|
||
return result
|
||
|
||
|
||
# Главный объект настроек: собирает все блоки конфигурации
|
||
|
||
@dataclass(slots=True)
|
||
class Settings:
|
||
home_url: str = "https://www.iaai.com/"
|
||
default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
|
||
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
|
||
fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 15000)
|
||
fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
|
||
fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
|
||
max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
|
||
retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
|
||
retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
|
||
retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
|
||
headless: bool = _env_bool("IAAI_HEADLESS", True)
|
||
browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
|
||
log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
|
||
log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
|
||
enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
|
||
sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
|
||
raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
|
||
tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
|
||
runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
|
||
scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
|
||
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
|
||
capture: CaptureConfig = field(default_factory=CaptureConfig)
|
||
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
|
||
listing: ListingConfig = field(default_factory=ListingConfig)
|
||
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
||
redis: RedisConfig = field(default_factory=RedisConfig)
|
||
celery: CeleryConfig = field(default_factory=CeleryConfig)
|
||
proxy: ProxyConfig = field(default_factory=ProxyConfig)
|
||
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
|
||
scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
|
||
|
||
@property
|
||
def parallel_tabs(self) -> int:
|
||
return self.celery.parallel_tabs
|
||
|
||
@property
|
||
def fetch_concurrency(self) -> int:
|
||
return self.celery.fetch_concurrency
|
||
|
||
@property
|
||
def block_resources(self) -> bool:
|
||
return self.celery.block_resources
|
||
|
||
# Глобальный синглтон — используется по умолчанию во всех модулях.
|
||
settings = Settings()
|