initial commit

encar_scraper/core/__init__.py

@@ -0,0 +1 @@
+__all__: list[str] = []

encar_scraper/core/config.py

@@ -0,0 +1,106 @@
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = os.getenv(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = os.getenv(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent для HTTP запросов) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("ENCAR_DATABASE_URL", "postgresql+psycopg2://encar:encar@localhost:5434/encar_db")
+    echo: bool = _env_bool("ENCAR_DATABASE_ECHO", False)
+    pool_size: int = _env_int("ENCAR_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("ENCAR_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("ENCAR_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("ENCAR_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("ENCAR_REDIS_URL", "redis://localhost:6380/1")
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 14400)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 14520)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 50)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    # Encar settings
+    encar_beat_interval_minutes: int = _env_int("ENCAR_BEAT_INTERVAL_MINUTES", 60)
+    encar_beat_limit: int | None = _env_int("ENCAR_BEAT_LIMIT", 0) or None
+
+
+# --- Главный объект настроек: собирает все блоки конфигурации ---
+
+@dataclass(slots=True)
+class Settings:
+    log_level: str = _env_str("ENCAR_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("ENCAR_LOG_FILE")
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()

encar_scraper/core/exceptions.py

@@ -0,0 +1,2 @@
+class ScraperError(Exception):
+    """Базовое исключение скрапера."""

encar_scraper/core/logs.py

@@ -0,0 +1,32 @@
+import logging
+import sys
+from contextvars import ContextVar
+
+# Храним trace_id текущего потока/корутины.
+TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
+
+
+class TraceIdFilter(logging.Filter):
+    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
+    def filter(self, record: logging.LogRecord) -> bool:
+        record.trace_id = TRACE_ID.get()
+        return True
+
+
+def set_trace_id(trace_id: str) -> None:
+    TRACE_ID.set(trace_id)
+
+
+def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
+    if log_file:
+        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
+    trace_filter = TraceIdFilter()
+    for handler in handlers:
+        handler.addFilter(trace_filter)
+    logging.basicConfig(
+        level=getattr(logging, level.upper(), logging.INFO),
+        format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s",
+        handlers=handlers,
+        force=True,
+    )

encar_scraper/core/runtime_config.py

@@ -0,0 +1,97 @@
+"""Загрузчик runtime_config.json для управления фильтрами синхронизации."""
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+logger = logging.getLogger("encar_scraper.runtime_config")
+
+DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent.parent / "runtime_config.json"
+
+
+@dataclass
+class SyncConfig:
+    name: str | None = None
+    lane: str = "encar_cars"
+    only_new: bool = False
+    limit: int | None = None
+    probe_all_photos: bool = False
+
+
+@dataclass
+class FiltersConfig:
+    price_min: int | None = None
+    price_max: int | None = None
+    mileage_min: int | None = None
+    mileage_max: int | None = None
+    brands: list[str] = field(default_factory=list)
+    models: list[str] = field(default_factory=list)
+    years: list[int] = field(default_factory=list)
+    body_types: list[str] = field(default_factory=list)
+    colors: list[str] = field(default_factory=list)
+    drives: list[str] = field(default_factory=list)
+    gearboxes: list[str] = field(default_factory=list)
+    exclude_brands: list[str] = field(default_factory=list)
+    exclude_models: list[str] = field(default_factory=list)
+    exclude_years: list[int] = field(default_factory=list)
+    exclude_body_types: list[str] = field(default_factory=list)
+
+
+@dataclass
+class RuntimeConfig:
+    sync: SyncConfig = field(default_factory=SyncConfig)
+    filters: FiltersConfig = field(default_factory=FiltersConfig)
+
+
+def load_runtime_config(path: str | Path | None = None) -> RuntimeConfig:
+    """Загружает runtime_config.json. Если файл не найден — возвращает дефолт."""
+    config_path = Path(path) if path else DEFAULT_CONFIG_PATH
+    if not config_path.exists():
+        logger.info("runtime_config.json not found at %s, using defaults", config_path)
+        return RuntimeConfig()
+
+    try:
+        raw: dict[str, Any] = json.loads(config_path.read_text(encoding="utf-8"))
+    except Exception as exc:
+        logger.warning("Failed to parse runtime_config.json: %s, using defaults", exc)
+        return RuntimeConfig()
+
+    sync_raw = raw.get("sync") or {}
+    filters_raw = raw.get("filters") or {}
+    price_raw = filters_raw.get("price") or {}
+    mileage_raw = filters_raw.get("mileage") or {}
+
+    sync = SyncConfig(
+        name=sync_raw.get("name"),
+        lane=sync_raw.get("lane", "encar_cars"),
+        only_new=bool(sync_raw.get("only_new", False)),
+        limit=sync_raw.get("limit"),
+        probe_all_photos=bool(sync_raw.get("probe_all_photos", False)),
+    )
+
+    filters = FiltersConfig(
+        price_min=price_raw.get("min"),
+        price_max=price_raw.get("max"),
+        mileage_min=mileage_raw.get("min"),
+        mileage_max=mileage_raw.get("max"),
+        brands=[b.lower() for b in (filters_raw.get("brands") or [])],
+        models=[m.lower() for m in (filters_raw.get("models") or [])],
+        years=filters_raw.get("years") or [],
+        body_types=[b.lower() for b in (filters_raw.get("body_types") or [])],
+        colors=[c.lower() for c in (filters_raw.get("colors") or [])],
+        drives=[d.lower() for d in (filters_raw.get("drives") or [])],
+        gearboxes=[g.lower() for g in (filters_raw.get("gearboxes") or [])],
+        exclude_brands=[b.lower() for b in (filters_raw.get("exclude_brands") or [])],
+        exclude_models=[m.lower() for m in (filters_raw.get("exclude_models") or [])],
+        exclude_years=filters_raw.get("exclude_years") or [],
+        exclude_body_types=[b.lower() for b in (filters_raw.get("exclude_body_types") or [])],
+    )
+
+    logger.info(
+        "runtime_config loaded: lane=%s, limit=%s, brands=%d, exclude_brands=%d",
+        sync.lane, sync.limit, len(filters.brands), len(filters.exclude_brands),
+    )
+    return RuntimeConfig(sync=sync, filters=filters)

encar_scraper/core/utils.py

@@ -0,0 +1,72 @@
+import json
+import re
+from pathlib import Path
+from typing import Any, Callable, Iterable
+
+
+def save_to_json(data: Any, filename: str | Path) -> None:
+    path = Path(filename)
+    path.parent.mkdir(parents=True, exist_ok=True)
+    path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def first_non_empty(values: Iterable[Any]) -> Any | None:
+    for value in values:
+        if value not in (None, "", [], {}, ()):
+            return value
+    return None
+
+
+# Регулярные выражения для VIN, lot и price.
+VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
+LOT_RE = re.compile(r"\b(\d{7,10})\b")
+PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
+
+
+def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
+    found = []
+    if _depth >= max_depth:
+        return found
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if key.lower() in target_keys:
+                found.append(value)
+            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    elif isinstance(obj, list):
+        for item in obj:
+            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    return found
+
+
+def deep_find_all_keys(
+    payloads: list,
+    field_map: dict[str, set[str]],
+    max_depth: int = 64,
+) -> dict[str, list]:
+    """Извлекает все нужные поля за один проход по JSON."""
+    # Готовим обратную карту: нормализованный ключ -> имя поля.
+    reverse: dict[str, str] = {}
+    for field_name, keys in field_map.items():
+        for k in keys:
+            reverse[k.lower()] = field_name
+
+    result: dict[str, list] = {f: [] for f in field_map}
+
+    def _recurse(obj: Any, depth: int) -> None:
+        if depth >= max_depth:
+            return
+        if isinstance(obj, dict):
+            for k, v in obj.items():
+                field = reverse.get(k.lower())
+                if field is not None:
+                    result[field].append(v)
+                _recurse(v, depth + 1)
+        elif isinstance(obj, list):
+            for item in obj:
+                _recurse(item, depth + 1)
+
+    for payload in payloads:
+        _recurse(payload, 0)
+
+    return result