qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix car mapping

Browse Source
This commit is contained in:
qananasikq
2026-07-01 13:49:30 +03:00
commit 3cd7e785da
68 changed files with 14973 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

315
dubizzle_scraper/core/config.py Normal file
View File

@@ -0,0 +1,315 @@
import json
import os
from dataclasses import dataclass, field
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
TRUE_VALUES = {"1", "true", "yes", "on"}
def _resolve_env_value(name: str) -> str | None:
"""Возвращает значение env с поддержкой legacy-префиксов DUBIZZLE_/IAAI_."""
value = os.getenv(name)
if value is not None:
return value
if name.startswith("DUBIZZLE_"):
return os.getenv("IAAI_" + name[len("DUBIZZLE_"):])
return None
# Хелперы для чтения env-переменных с приведением типов
def _env_str(name: str, default: str) -> str:
value = _resolve_env_value(name)
return value if value is not None else default
def _env_optional_str(name: str) -> str | None:
value = _resolve_env_value(name)
if value is None:
return None
value = value.strip()
return value or None
def _env_path_str(name: str) -> str | None:
value = _env_optional_str(name)
if value is None:
return None
return str(Path(value).expanduser())
def _env_bool(name: str, default: bool) -> bool:
fallback = "true" if default else "false"
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
def _env_int(name: str, default: int) -> int:
return int(_env_str(name, str(default)).strip())
def _env_float(name: str, default: float) -> float:
return float(_env_str(name, str(default)).strip())
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
@dataclass(slots=True)
class FingerprintConfig:
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/135.0.0.0 Safari/537.36"
)
viewport_presets: tuple[dict[str, int], ...] = (
{"width": 1920, "height": 1080},
{"width": 1600, "height": 900},
{"width": 1536, "height": 864},
{"width": 1440, "height": 900},
{"width": 1366, "height": 768},
)
timezone_candidates: tuple[str, ...] = (
"America/New_York",
"America/Chicago",
"America/Los_Angeles",
)
locale: str = "en-US"
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
@dataclass(slots=True)
class CaptureConfig:
capture_same_origin_only: bool = _env_bool("DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY", True)
max_requests: int = _env_int("DUBIZZLE_MAX_CAPTURED_REQUESTS", 40)
max_json_responses: int = _env_int("DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES", 30)
# Конфиг пауз между действиями (имитация человека)
@dataclass(slots=True)
class HumanPaceConfig:
enabled: bool = _env_bool("DUBIZZLE_HUMAN_PACE_ENABLED", True)
after_listing_open_min_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MIN_S", 0.5)
after_listing_open_max_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MAX_S", 1.2)
after_filter_action_min_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MIN_S", 0.5)
after_filter_action_max_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MAX_S", 1.2)
before_vehicle_open_min_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
before_vehicle_open_max_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
after_vehicle_open_min_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
after_vehicle_open_max_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
between_vehicles_min_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MIN_S", 0.05)
between_vehicles_max_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MAX_S", 0.15)
after_page_change_min_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S", 0.8)
after_page_change_max_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S", 1.8)
# Конфиг сбора листинга (URL, лимиты страниц и машин)
@dataclass(slots=True)
class ListingConfig:
cars_url: str = _env_str("DUBIZZLE_CARS_LISTING_URL", "https://dubai.dubizzle.com/motors/used-cars/")
max_pages_per_run: int = _env_int("DUBIZZLE_MAX_PAGES_PER_RUN", 9999)
max_vehicles_per_run: int = _env_int("DUBIZZLE_MAX_VEHICLES_PER_RUN", 50000)
page_link_limit: int = _env_int("DUBIZZLE_PAGE_LINK_LIMIT", 500)
include_pagination: bool = _env_bool("DUBIZZLE_INCLUDE_PAGINATION", True)
collect_current_page_only: bool = _env_bool("DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY", False)
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
early_stop_threshold: float = _env_float("DUBIZZLE_EARLY_STOP_THRESHOLD", 0.8)
# Сегментация листинга по брендам для обхода лимита пагинации DUBIZZLE (~22 600 машин).
# JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
# Пустая строка = без сегментации (backward compatible).
listing_segments_json: str = _env_str("DUBIZZLE_LISTING_SEGMENTS", "")
# Пагинационный потолок одного Algolia-запроса: ~100 страниц × 100 = 10 000 результатов.
DUBIZZLE_PAGINATION_CEILING = 10_000
# Авто-сегментация по году. Эти диапазоны подобраны так, чтобы каждый сегмент
# оставался ниже лимита Algolia и собирался быстрее, чем старая make/year схема.
_AUTO_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
(1900, 2012),
(2013, 2015),
(2016, 2017),
(2018, 2019),
(2020, 2021),
(2022, 2023),
(2024, 2025),
(2026, 2027),
)
def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
"""Парсит DUBIZZLE_LISTING_SEGMENTS в список сегментов.
Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
Специальное значение ``"auto"`` генерирует быстрые year-only сегменты,
которые гарантированно проходят через Algolia без упора в лимит ~10k.
"""
raw = raw.strip()
if not raw:
return []
if raw.lower() == "auto":
return [
{"make": None, "year_min": yr_min, "year_max": yr_max}
for yr_min, yr_max in _AUTO_YEAR_SPLITS
]
try:
data = json.loads(raw)
except (json.JSONDecodeError, ValueError):
return []
if not isinstance(data, list):
return []
segments = []
for item in data:
if isinstance(item, str):
segments.append({"make": item.upper(), "year_min": None, "year_max": None})
elif isinstance(item, dict):
segments.append({
"make": str(item.get("make") or "").upper() or None,
"year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
"year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
})
return segments
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
@dataclass(slots=True)
class DatabaseConfig:
url: str = _env_str("DUBIZZLE_DATABASE_URL", "postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper")
echo: bool = _env_bool("DUBIZZLE_DATABASE_ECHO", False)
pool_size: int = _env_int("DUBIZZLE_DATABASE_POOL_SIZE", 5)
max_overflow: int = _env_int("DUBIZZLE_DATABASE_MAX_OVERFLOW", 10)
pool_recycle_seconds: int = _env_int("DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
auto_create_tables: bool = _env_bool("DUBIZZLE_DATABASE_AUTO_CREATE_TABLES", False)
# --- Конфиг Redis (URL для Celery broker) ---
@dataclass(slots=True)
class RedisConfig:
url: str = _env_str("DUBIZZLE_REDIS_URL", "redis://localhost:6379/0")
socket_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
socket_connect_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
health_check_interval_seconds: int = _env_int("DUBIZZLE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
@dataclass(slots=True)
class DiscoveryConfig:
mode: str = _env_str("DUBIZZLE_DISCOVERY_MODE", "algolia")
hourly_mode: str = _env_str("DUBIZZLE_HOURLY_MODE", "rolling_refresh")
hourly_refresh_batch_size: int = _env_int("DUBIZZLE_HOURLY_REFRESH_BATCH_SIZE", 500)
always_full_scan: bool = _env_bool("DUBIZZLE_ALWAYS_FULL_SCAN", False)
sitemap_fallback_on_empty: bool = _env_bool("DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY", False)
@dataclass(slots=True)
class AlgoliaConfig:
application_id: str = _env_str("DUBIZZLE_ALGOLIA_APPLICATION_ID", "WD0PTZ13ZS")
api_key: str = _env_str(
"DUBIZZLE_ALGOLIA_API_KEY",
"cef139620248f1bc328a00fddc7107a6",
)
index_name: str = _env_str("DUBIZZLE_ALGOLIA_INDEX_NAME", "motors.com")
category_slug: str = _env_str("DUBIZZLE_ALGOLIA_CATEGORY_SLUG", "motors/used-cars")
base_url: str = _env_str("DUBIZZLE_ALGOLIA_BASE_URL", "https://WD0PTZ13ZS-dsn.algolia.net")
hits_per_page: int = _env_int("DUBIZZLE_ALGOLIA_HITS_PER_PAGE", 20)
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@dataclass(slots=True)
class CeleryConfig:
broker_url: str = _env_str("CELERY_BROKER_URL", "")
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("DUBIZZLE_PARALLEL_TABS", 8)
parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
block_resources: bool = _env_bool("DUBIZZLE_BLOCK_RESOURCES", True)
# --- Конфиг прокси (server, username, password) ---
@dataclass(slots=True)
class ProxyConfig:
server: str | None = _env_optional_str("DUBIZZLE_PROXY_SERVER")
username: str | None = _env_optional_str("DUBIZZLE_PROXY_USERNAME")
password: str | None = _env_optional_str("DUBIZZLE_PROXY_PASSWORD")
@property
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
if self.username:
result["username"] = self.username
if self.password:
result["password"] = self.password
return result
# Главный объект настроек: собирает все блоки конфигурации
@dataclass(slots=True)
class Settings:
home_url: str = "https://www.dubizzle.com/"
default_timeout_ms: int = _env_int("DUBIZZLE_TIMEOUT_MS", 45000)
network_settle_ms: int = _env_int("DUBIZZLE_NETWORK_SETTLE_MS", 400)
fast_path_timeout_ms: int = _env_int("DUBIZZLE_FAST_PATH_TIMEOUT_MS", 5000)
fast_path_max_attempts: int = _env_int("DUBIZZLE_FAST_PATH_MAX_ATTEMPTS", 1)
fallback_navigation_timeout_ms: int = _env_int("DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS", 15000)
max_retries: int = _env_int("DUBIZZLE_MAX_RETRIES", 3)
retry_delay_seconds: float = _env_float("DUBIZZLE_RETRY_DELAY_SECONDS", 2.5)
retry_backoff_multiplier: float = _env_float("DUBIZZLE_RETRY_BACKOFF_MULTIPLIER", 2.0)
retry_jitter_seconds: float = _env_float("DUBIZZLE_RETRY_JITTER_SECONDS", 0.25)
headless: bool = _env_bool("DUBIZZLE_HEADLESS", True)
browser_engine: str = _env_str("DUBIZZLE_BROWSER_ENGINE", "auto")
log_level: str = _env_str("DUBIZZLE_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("DUBIZZLE_LOG_FILE")
enable_trace_id_logs: bool = _env_bool("DUBIZZLE_ENABLE_TRACE_ID_LOGS", True)
sync_only_new: bool = _env_bool("DUBIZZLE_SYNC_ONLY_NEW", False)
raw_output_json: str | None = _env_optional_str("DUBIZZLE_RAW_OUTPUT_JSON")
tokens_file: str | None = _env_path_str("DUBIZZLE_TOKENS_FILE")
runtime_config_file: str | None = _env_path_str("DUBIZZLE_RUNTIME_CONFIG_FILE")
scheduler_interval_minutes: int = _env_int("DUBIZZLE_SCHEDULER_INTERVAL_MINUTES", 60)
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
capture: CaptureConfig = field(default_factory=CaptureConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
algolia: AlgoliaConfig = field(default_factory=AlgoliaConfig)
@property
def parallel_tabs(self) -> int:
return self.celery.parallel_tabs
@property
def block_resources(self) -> bool:
return self.celery.block_resources
# Глобальный синглтон — используется по умолчанию во всех модулях.
settings = Settings()