diff --git a/.env.example b/.env.example index 9fa711f..35d5a3e 100644 --- a/.env.example +++ b/.env.example @@ -1,66 +1,28 @@ -IAAI_HEADLESS=true -IAAI_LOG_LEVEL=INFO -# IAAI_LOG_FILE=iaai_scraper.log +# ===== Encar Scraper Config ===== -# Network capture settings. -IAAI_CAPTURE_SAME_ORIGIN_ONLY=true -IAAI_MAX_CAPTURED_REQUESTS=40 -IAAI_MAX_CAPTURED_JSON_RESPONSES=30 +# Логирование +ENCAR_LOG_LEVEL=INFO -# Sequential listing-first mode. -IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars -IAAI_MAX_PAGES_PER_RUN=10 -IAAI_MAX_VEHICLES_PER_RUN=50000 -IAAI_PAGE_LINK_LIMIT=500 -IAAI_INCLUDE_PAGINATION=true -IAAI_COLLECT_CURRENT_PAGE_ONLY=false +# База данных PostgreSQL +ENCAR_DATABASE_URL=postgresql+psycopg2://encar:encar@postgres:5432/encar_db +ENCAR_DATABASE_ECHO=false +ENCAR_DATABASE_POOL_SIZE=5 +ENCAR_DATABASE_MAX_OVERFLOW=10 +ENCAR_DATABASE_POOL_RECYCLE_SECONDS=1800 -# Паузы (отключены для максимальной скорости; включи на VPS если попадаешь под блокировки). -IAAI_HUMAN_PACE_ENABLED=false -IAAI_AFTER_LISTING_OPEN_MIN_S=0.2 -IAAI_AFTER_LISTING_OPEN_MAX_S=0.5 -IAAI_AFTER_FILTER_ACTION_MIN_S=0.2 -IAAI_AFTER_FILTER_ACTION_MAX_S=0.5 -IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02 -IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08 -IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02 -IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08 -IAAI_BETWEEN_VEHICLES_MIN_S=0.02 -IAAI_BETWEEN_VEHICLES_MAX_S=0.08 -IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2 -IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5 +# Redis +ENCAR_REDIS_URL=redis://redis:6379/1 -# Sync settings -IAAI_SYNC_ONLY_NEW=true -IAAI_TOKENS_FILE=/data/tokens.json -IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json +# Celery +CELERY_BROKER_URL=redis://redis:6379/1 +CELERY_RESULT_BACKEND=redis://redis:6379/1 +CELERY_TASK_SOFT_TIME_LIMIT=14400 +CELERY_TASK_TIME_LIMIT=14520 +CELERY_BROKER_VISIBILITY_TIMEOUT=15000 +CELERY_WORKER_CONCURRENCY=4 +CELERY_WORKER_MAX_TASKS_PER_CHILD=50 -# Retry / backoff -IAAI_RETRY_DELAY_SECONDS=2.5 -IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 -IAAI_RETRY_JITTER_SECONDS=0.25 - -# Proxy (HTTP/HTTPS preferred for Playwright) -# Chromium does not support SOCKS5 proxy authentication directly. -# Use residential or mobile USA proxy. -# IAAI_PROXY_SERVER=http://proxy.example.com:8080 -# IAAI_PROXY_USERNAME= -# IAAI_PROXY_PASSWORD= - -# Database (PostgreSQL). -IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper -IAAI_DATABASE_ECHO=false -IAAI_DATABASE_POOL_SIZE=10 -IAAI_DATABASE_MAX_OVERFLOW=20 - -# ─── Redis (Celery broker) ───────────────────────────────── -IAAI_REDIS_URL=redis://redis:6379/0 - -# ─── Celery ──────────────────────────────────────────────── -CELERY_BROKER_URL=redis://redis:6379/0 -CELERY_RESULT_BACKEND=redis://redis:6379/0 -CELERY_TASK_SOFT_TIME_LIMIT=600 -CELERY_TASK_TIME_LIMIT=900 -CELERY_WORKER_CONCURRENCY=1 -CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 -CELERY_BEAT_SYNC_LIMIT=26 +# Encar Beat — полный прогон каждые 60 минут +ENCAR_BEAT_INTERVAL_MINUTES=60 +# 0 = без лимита, парсит ВСЕ авто +ENCAR_BEAT_LIMIT=0 diff --git a/encar_scraper/__init__.py b/encar_scraper/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/encar_scraper/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/encar_scraper/api/__init__.py b/encar_scraper/api/__init__.py new file mode 100644 index 0000000..b94a1e8 --- /dev/null +++ b/encar_scraper/api/__init__.py @@ -0,0 +1,3 @@ +from .app import create_app + +__all__ = ["create_app"] diff --git a/encar_scraper/api/app.py b/encar_scraper/api/app.py new file mode 100644 index 0000000..703bd64 --- /dev/null +++ b/encar_scraper/api/app.py @@ -0,0 +1,40 @@ +# Создание FastAPI-приложения и настройка его жизненного цикла. + +from contextlib import asynccontextmanager + +from fastapi import FastAPI + +from ..core.config import Settings +from ..storage.db import PersistenceService +from .routes import cars, health, tasks + + +@asynccontextmanager +async def lifespan(app: FastAPI): + # Жизненный цикл. + # Таблицы создаются через Alembic-миграции (сервис migrate). + yield + + +def create_app(settings: Settings | None = None) -> FastAPI: + _settings = settings or Settings() + + app = FastAPI( + title="Encar Scraper API", + description="REST API для управления задачами скрапинга Encar и просмотра данных", + version="1.0.0", + lifespan=lifespan, + ) + + app.state.settings = _settings + app.state.persistence = PersistenceService(_settings) + + app.include_router(health.router, tags=["health"]) + app.include_router(cars.router, prefix="/api/v1", tags=["cars"]) + app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"]) + + return app + + +# Экземпляр приложения для запуска через uvicorn. +app = create_app() diff --git a/encar_scraper/api/deps.py b/encar_scraper/api/deps.py new file mode 100644 index 0000000..86b4bc3 --- /dev/null +++ b/encar_scraper/api/deps.py @@ -0,0 +1,9 @@ +# Вспомогательные зависимости для FastAPI-роутов. + +from fastapi import Request + +from ..storage.db import PersistenceService + + +def get_persistence(request: Request) -> PersistenceService: + return request.app.state.persistence diff --git a/encar_scraper/api/routes/__init__.py b/encar_scraper/api/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/encar_scraper/api/routes/cars.py b/encar_scraper/api/routes/cars.py new file mode 100644 index 0000000..8b1a8b3 --- /dev/null +++ b/encar_scraper/api/routes/cars.py @@ -0,0 +1,104 @@ +# Роуты для просмотра автомобилей и агрегированной статистики. + +from fastapi import APIRouter, Depends, HTTPException, Query +from sqlalchemy import func, select +from sqlalchemy.orm import selectinload + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import Car, Image +from ...storage.schemas import CarRead + +router = APIRouter() + + +@router.get("/cars") +def list_cars( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + brand: str | None = None, + model: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + is_sold: bool | None = None, + persistence: PersistenceService = Depends(get_persistence), +): + # Список автомобилей с пагинацией и фильтрами + with persistence.session_scope() as session: + query = select(Car) + + if brand: + query = query.where(Car.brand.ilike(f"%{brand}%")) + if model: + query = query.where(Car.model.ilike(f"%{model}%")) + if year_min is not None: + query = query.where(Car.year >= year_min) + if year_max is not None: + query = query.where(Car.year <= year_max) + if is_sold is not None: + query = query.where(Car.is_sold == is_sold) + + count_query = select(func.count()).select_from(query.subquery()) + total = session.execute(count_query).scalar() or 0 + + offset = (page - 1) * per_page + cars = session.execute( + query.options(selectinload(Car.images)).order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "pages": (total + per_page - 1) // per_page if per_page else 0, + "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars], + } + + +@router.get("/cars/{car_id}") +def get_car( + car_id: int, + persistence: PersistenceService = Depends(get_persistence), +): + # Детальная информация об автомобиле с изображениями + with persistence.session_scope() as session: + car = session.get(Car, car_id) + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/cars/by-origin/{origin_id}") +def get_car_by_origin( + origin_id: str, + persistence: PersistenceService = Depends(get_persistence), +): + # Поиск автомобиля по origin_id + with persistence.session_scope() as session: + car = session.execute( + select(Car).where(Car.origin_id == origin_id) + ).scalars().first() + if car is None: + raise HTTPException(status_code=404, detail="Car not found") + return CarRead.model_validate(car).model_dump(mode="json") + + +@router.get("/stats") +def get_stats(persistence: PersistenceService = Depends(get_persistence)): + # Общая статистика по БД + with persistence.session_scope() as session: + total_cars = session.execute(select(func.count(Car.id))).scalar() or 0 + total_images = session.execute(select(func.count(Image.id))).scalar() or 0 + brands = session.execute( + select(Car.brand, func.count(Car.id)) + .group_by(Car.brand) + .order_by(func.count(Car.id).desc()) + .limit(20) + ).all() + + return { + "total_cars": total_cars, + "total_images": total_images, + "top_brands": [{"brand": b, "count": c} for b, c in brands], + } + diff --git a/encar_scraper/api/routes/health.py b/encar_scraper/api/routes/health.py new file mode 100644 index 0000000..1a347f3 --- /dev/null +++ b/encar_scraper/api/routes/health.py @@ -0,0 +1,30 @@ +# Роут проверки доступности сервиса и соединения с БД. + +import logging + +from fastapi import APIRouter, Depends +from sqlalchemy import text + +from ..deps import get_persistence +from ...storage.db import PersistenceService + +router = APIRouter() +logger = logging.getLogger("encar_scraper.api.health") + + +@router.get("/health") +def health_check(persistence: PersistenceService = Depends(get_persistence)): + # Проверка API и БД + db_ok = False + try: + with persistence.session_scope() as session: + session.execute(text("SELECT 1")) + db_ok = True + except Exception: + logger.warning("Health DB check failed", exc_info=True) + + return { + "status": "ok" if db_ok else "degraded", + "service": "encar-scraper-api", + "database": "connected" if db_ok else "unavailable", + } diff --git a/encar_scraper/api/routes/tasks.py b/encar_scraper/api/routes/tasks.py new file mode 100644 index 0000000..e5c1ab0 --- /dev/null +++ b/encar_scraper/api/routes/tasks.py @@ -0,0 +1,121 @@ +# Роуты запуска задач синхронизации Encar и просмотра истории sync-runs. + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel +from sqlalchemy import select, func + +from ..deps import get_persistence +from ...storage.db import PersistenceService +from ...storage.models import SyncRun +from ...worker.celery_app import celery_app +from ...worker.tasks import encar_sync_listing_task, encar_sync_vehicle_task + +router = APIRouter() + + +class SyncVehicleRequest(BaseModel): + vehicle_url: str + lane: str = "encar" + + +class SyncListingRequest(BaseModel): + car_type: str = "all" # all, domestic, import + manufacturer: str | None = None + year_from: int | None = None + year_to: int | None = None + price_max: int | None = None + lane: str = "encar" + limit: int | None = None + + +@router.post("/tasks/sync-vehicle") +def start_sync_vehicle(body: SyncVehicleRequest): + """Синхронизация одного авто Encar.""" + result = encar_sync_vehicle_task.apply_async( + kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane}, + queue="encar", + ) + return { + "task_id": result.id, + "status": "queued", + "vehicle_url": body.vehicle_url, + } + + +@router.post("/tasks/sync-listing") +def start_sync_listing(body: SyncListingRequest): + """Синхронизация листинга Encar через публичный API.""" + result = encar_sync_listing_task.apply_async( + kwargs={ + "car_type": body.car_type, + "manufacturer": body.manufacturer, + "year_from": body.year_from, + "year_to": body.year_to, + "price_max": body.price_max, + "lane": body.lane, + "limit": body.limit, + }, + queue="encar", + ) + return { + "task_id": result.id, + "status": "queued", + "car_type": body.car_type, + "manufacturer": body.manufacturer, + } + + +@router.get("/tasks/{task_id}") +def get_task_status(task_id: str): + """Статус задачи Celery.""" + result = celery_app.AsyncResult(task_id) + + payload: dict = { + "task_id": task_id, + "state": result.state, + } + + if result.successful(): + payload["result"] = result.result + elif result.failed(): + payload["error"] = str(result.result) + elif result.info is not None: + payload["meta"] = result.info + + return payload + + +@router.get("/sync-runs") +def list_sync_runs( + page: int = Query(1, ge=1), + per_page: int = Query(20, ge=1, le=100), + persistence: PersistenceService = Depends(get_persistence), +): + """История запусков синхронизации.""" + with persistence.session_scope() as session: + total = session.execute(select(func.count(SyncRun.id))).scalar() or 0 + offset = (page - 1) * per_page + runs = session.execute( + select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page) + ).scalars().all() + + return { + "total": total, + "page": page, + "per_page": per_page, + "items": [ + { + "id": run.id, + "started_at": run.started_at.isoformat() if run.started_at else None, + "finished_at": run.finished_at.isoformat() if run.finished_at else None, + "status": run.status, + "lane": run.lane, + "ids_fetched": run.ids_fetched, + "cars_upserted": run.cars_upserted, + "cars_failed": run.cars_failed, + "images_upserted": run.images_upserted, + "error_summary": run.error_summary, + } + for run in runs + ], + } diff --git a/encar_scraper/cli.py b/encar_scraper/cli.py new file mode 100644 index 0000000..03a239f --- /dev/null +++ b/encar_scraper/cli.py @@ -0,0 +1,103 @@ +import argparse +from pathlib import Path + +from .core.config import Settings +from .core.utils import save_to_json +from .encar import EncarScraper, EncarFilters, CAR_TYPE_ALL, CAR_TYPE_DOMESTIC, CAR_TYPE_IMPORT + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description="Encar scraper CLI") + subparsers = parser.add_subparsers(dest="command", required=True) + + default_output_dir = Path("artifacts/json") + + subparsers.add_parser("init-db", help="Create DB tables") + + # collect-listing + collect_parser = subparsers.add_parser("collect-listing", help="Collect Encar listing") + collect_parser.add_argument("--limit", type=int, default=None, help="Max items to collect") + collect_parser.add_argument("--car-type", choices=["all", "domestic", "import"], default="all", + help="all=все, domestic=корейские, import=импорт") + collect_parser.add_argument("--manufacturer", default=None, help="Filter by manufacturer (e.g. BMW, 현대)") + collect_parser.add_argument("--year-from", type=int, default=None, help="Min year") + collect_parser.add_argument("--year-to", type=int, default=None, help="Max year") + collect_parser.add_argument("--price-max", type=int, default=None, help="Max price in 만원 (10k KRW)") + collect_parser.add_argument("--output", default=str(default_output_dir / "encar_listing.json")) + + # scrape-vehicle + scrape_parser = subparsers.add_parser("scrape-vehicle", help="Fetch single vehicle data") + scrape_parser.add_argument("vehicle_url") + scrape_parser.add_argument("--output", default=str(default_output_dir / "encar_vehicle_detail.json")) + + # sync-vehicle + sync_parser = subparsers.add_parser("sync-vehicle", help="Sync single vehicle into DB") + sync_parser.add_argument("vehicle_url") + sync_parser.add_argument("--lane", default="encar") + sync_parser.add_argument("--output", default=str(default_output_dir / "encar_sync_vehicle.json")) + + # sync-listing + sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing and sync all to DB") + sync_listing_parser.add_argument("--limit", type=int, default=None, help="Max items to sync") + sync_listing_parser.add_argument("--car-type", choices=["all", "domestic", "import"], default="all") + sync_listing_parser.add_argument("--manufacturer", default=None, help="Filter by manufacturer") + sync_listing_parser.add_argument("--year-from", type=int, default=None) + sync_listing_parser.add_argument("--year-to", type=int, default=None) + sync_listing_parser.add_argument("--price-max", type=int, default=None) + sync_listing_parser.add_argument("--lane", default="encar") + sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default="true") + sync_listing_parser.add_argument("--output", default=str(default_output_dir / "encar_sync_listing.json")) + + return parser + + +def _build_filters_from_args(args) -> EncarFilters: + """Строит фильтры из аргументов CLI.""" + car_type_map = {"all": CAR_TYPE_ALL, "domestic": CAR_TYPE_DOMESTIC, "import": CAR_TYPE_IMPORT} + return EncarFilters( + car_type=car_type_map.get(getattr(args, "car_type", "all"), CAR_TYPE_ALL), + manufacturer=getattr(args, "manufacturer", None), + year_from=getattr(args, "year_from", None), + year_to=getattr(args, "year_to", None), + price_to=getattr(args, "price_max", None), + ) + + +def main() -> None: + import sys + import logging + + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + stream=sys.stderr, + ) + + parser = build_parser() + args = parser.parse_args() + scraper = EncarScraper() + + if args.command == "init-db": + result = scraper.init_db() + print(f"DB initialized: {result}") + return + elif args.command == "collect-listing": + filters = _build_filters_from_args(args) + result = scraper.collect_listing(limit=args.limit, filters=filters) + elif args.command == "scrape-vehicle": + result = scraper.scrape_vehicle_detail(args.vehicle_url) + elif args.command == "sync-vehicle": + result = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) + elif args.command == "sync-listing": + filters = _build_filters_from_args(args) + only_new = args.only_new == "true" + result = scraper.sync_listing(limit=args.limit, filters=filters, lane=args.lane, only_new=only_new) + else: + raise SystemExit(f"Unknown command: {args.command}") + + save_to_json(result, Path(args.output)) + print(f"Saved to {Path(args.output).resolve()}") + + +if __name__ == "__main__": + main() diff --git a/encar_scraper/core/__init__.py b/encar_scraper/core/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/encar_scraper/core/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/encar_scraper/core/config.py b/encar_scraper/core/config.py new file mode 100644 index 0000000..92d8331 --- /dev/null +++ b/encar_scraper/core/config.py @@ -0,0 +1,106 @@ +import os +from dataclasses import dataclass, field +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +# Хелперы для чтения env-переменных с приведением типов + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + return value if value is not None else default + + +def _env_optional_str(name: str) -> str | None: + value = os.getenv(name) + if value is None: + return None + value = value.strip() + return value or None + + +def _env_path_str(name: str) -> str | None: + value = _env_optional_str(name) + if value is None: + return None + return str(Path(value).expanduser()) + + +def _env_bool(name: str, default: bool) -> bool: + fallback = "true" if default else "false" + return _env_str(name, fallback).strip().lower() in TRUE_VALUES + + +def _env_int(name: str, default: int) -> int: + return int(_env_str(name, str(default)).strip()) + + +def _env_float(name: str, default: float) -> float: + return float(_env_str(name, str(default)).strip()) + + +# Конфиг браузерного отпечатка (User-Agent для HTTP запросов) + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + + +# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) + +@dataclass(slots=True) +class DatabaseConfig: + url: str = _env_str("ENCAR_DATABASE_URL", "postgresql+psycopg2://encar:encar@localhost:5434/encar_db") + echo: bool = _env_bool("ENCAR_DATABASE_ECHO", False) + pool_size: int = _env_int("ENCAR_DATABASE_POOL_SIZE", 5) + max_overflow: int = _env_int("ENCAR_DATABASE_MAX_OVERFLOW", 10) + pool_recycle_seconds: int = _env_int("ENCAR_DATABASE_POOL_RECYCLE_SECONDS", 1800) + auto_create_tables: bool = _env_bool("ENCAR_DATABASE_AUTO_CREATE_TABLES", False) + + +# --- Конфиг Redis (URL для Celery broker) --- + +@dataclass(slots=True) +class RedisConfig: + url: str = _env_str("ENCAR_REDIS_URL", "redis://localhost:6380/1") + + +# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = _env_str("CELERY_BROKER_URL", "") + result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") + task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 14400) + task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 14520) + worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) + worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 50) + broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) + # Encar settings + encar_beat_interval_minutes: int = _env_int("ENCAR_BEAT_INTERVAL_MINUTES", 60) + encar_beat_limit: int | None = _env_int("ENCAR_BEAT_LIMIT", 0) or None + + +# --- Главный объект настроек: собирает все блоки конфигурации --- + +@dataclass(slots=True) +class Settings: + log_level: str = _env_str("ENCAR_LOG_LEVEL", "INFO") + log_file: str | None = _env_optional_str("ENCAR_LOG_FILE") + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + +# Глобальный синглтон — используется по умолчанию во всех модулях. +settings = Settings() diff --git a/encar_scraper/core/exceptions.py b/encar_scraper/core/exceptions.py new file mode 100644 index 0000000..78fec35 --- /dev/null +++ b/encar_scraper/core/exceptions.py @@ -0,0 +1,2 @@ +class ScraperError(Exception): + """Базовое исключение скрапера.""" diff --git a/encar_scraper/core/logs.py b/encar_scraper/core/logs.py new file mode 100644 index 0000000..32a312d --- /dev/null +++ b/encar_scraper/core/logs.py @@ -0,0 +1,32 @@ +import logging +import sys +from contextvars import ContextVar + +# Храним trace_id текущего потока/корутины. +TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-") + + +class TraceIdFilter(logging.Filter): + # Добавляет trace_id в каждую запись лога для сквозной трассировки. + def filter(self, record: logging.LogRecord) -> bool: + record.trace_id = TRACE_ID.get() + return True + + +def set_trace_id(trace_id: str) -> None: + TRACE_ID.set(trace_id) + + +def setup_logging(level: str = "INFO", log_file: str | None = None) -> None: + handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)] + if log_file: + handlers.append(logging.FileHandler(log_file, encoding="utf-8")) + trace_filter = TraceIdFilter() + for handler in handlers: + handler.addFilter(trace_filter) + logging.basicConfig( + level=getattr(logging, level.upper(), logging.INFO), + format="%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s", + handlers=handlers, + force=True, + ) diff --git a/encar_scraper/core/runtime_config.py b/encar_scraper/core/runtime_config.py new file mode 100644 index 0000000..8ec952c --- /dev/null +++ b/encar_scraper/core/runtime_config.py @@ -0,0 +1,97 @@ +"""Загрузчик runtime_config.json для управления фильтрами синхронизации.""" +from __future__ import annotations + +import json +import logging +from dataclasses import dataclass, field +from pathlib import Path +from typing import Any + +logger = logging.getLogger("encar_scraper.runtime_config") + +DEFAULT_CONFIG_PATH = Path(__file__).resolve().parent.parent.parent / "runtime_config.json" + + +@dataclass +class SyncConfig: + name: str | None = None + lane: str = "encar_cars" + only_new: bool = False + limit: int | None = None + probe_all_photos: bool = False + + +@dataclass +class FiltersConfig: + price_min: int | None = None + price_max: int | None = None + mileage_min: int | None = None + mileage_max: int | None = None + brands: list[str] = field(default_factory=list) + models: list[str] = field(default_factory=list) + years: list[int] = field(default_factory=list) + body_types: list[str] = field(default_factory=list) + colors: list[str] = field(default_factory=list) + drives: list[str] = field(default_factory=list) + gearboxes: list[str] = field(default_factory=list) + exclude_brands: list[str] = field(default_factory=list) + exclude_models: list[str] = field(default_factory=list) + exclude_years: list[int] = field(default_factory=list) + exclude_body_types: list[str] = field(default_factory=list) + + +@dataclass +class RuntimeConfig: + sync: SyncConfig = field(default_factory=SyncConfig) + filters: FiltersConfig = field(default_factory=FiltersConfig) + + +def load_runtime_config(path: str | Path | None = None) -> RuntimeConfig: + """Загружает runtime_config.json. Если файл не найден — возвращает дефолт.""" + config_path = Path(path) if path else DEFAULT_CONFIG_PATH + if not config_path.exists(): + logger.info("runtime_config.json not found at %s, using defaults", config_path) + return RuntimeConfig() + + try: + raw: dict[str, Any] = json.loads(config_path.read_text(encoding="utf-8")) + except Exception as exc: + logger.warning("Failed to parse runtime_config.json: %s, using defaults", exc) + return RuntimeConfig() + + sync_raw = raw.get("sync") or {} + filters_raw = raw.get("filters") or {} + price_raw = filters_raw.get("price") or {} + mileage_raw = filters_raw.get("mileage") or {} + + sync = SyncConfig( + name=sync_raw.get("name"), + lane=sync_raw.get("lane", "encar_cars"), + only_new=bool(sync_raw.get("only_new", False)), + limit=sync_raw.get("limit"), + probe_all_photos=bool(sync_raw.get("probe_all_photos", False)), + ) + + filters = FiltersConfig( + price_min=price_raw.get("min"), + price_max=price_raw.get("max"), + mileage_min=mileage_raw.get("min"), + mileage_max=mileage_raw.get("max"), + brands=[b.lower() for b in (filters_raw.get("brands") or [])], + models=[m.lower() for m in (filters_raw.get("models") or [])], + years=filters_raw.get("years") or [], + body_types=[b.lower() for b in (filters_raw.get("body_types") or [])], + colors=[c.lower() for c in (filters_raw.get("colors") or [])], + drives=[d.lower() for d in (filters_raw.get("drives") or [])], + gearboxes=[g.lower() for g in (filters_raw.get("gearboxes") or [])], + exclude_brands=[b.lower() for b in (filters_raw.get("exclude_brands") or [])], + exclude_models=[m.lower() for m in (filters_raw.get("exclude_models") or [])], + exclude_years=filters_raw.get("exclude_years") or [], + exclude_body_types=[b.lower() for b in (filters_raw.get("exclude_body_types") or [])], + ) + + logger.info( + "runtime_config loaded: lane=%s, limit=%s, brands=%d, exclude_brands=%d", + sync.lane, sync.limit, len(filters.brands), len(filters.exclude_brands), + ) + return RuntimeConfig(sync=sync, filters=filters) diff --git a/encar_scraper/core/utils.py b/encar_scraper/core/utils.py new file mode 100644 index 0000000..8c8cad5 --- /dev/null +++ b/encar_scraper/core/utils.py @@ -0,0 +1,72 @@ +import json +import re +from pathlib import Path +from typing import Any, Callable, Iterable + + +def save_to_json(data: Any, filename: str | Path) -> None: + path = Path(filename) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8") + + +def first_non_empty(values: Iterable[Any]) -> Any | None: + for value in values: + if value not in (None, "", [], {}, ()): + return value + return None + + +# Регулярные выражения для VIN, lot и price. +VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE) +LOT_RE = re.compile(r"\b(\d{7,10})\b") +PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)") + + +def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list: + # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве. + found = [] + if _depth >= max_depth: + return found + if isinstance(obj, dict): + for key, value in obj.items(): + if key.lower() in target_keys: + found.append(value) + found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1)) + elif isinstance(obj, list): + for item in obj: + found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1)) + return found + + +def deep_find_all_keys( + payloads: list, + field_map: dict[str, set[str]], + max_depth: int = 64, +) -> dict[str, list]: + """Извлекает все нужные поля за один проход по JSON.""" + # Готовим обратную карту: нормализованный ключ -> имя поля. + reverse: dict[str, str] = {} + for field_name, keys in field_map.items(): + for k in keys: + reverse[k.lower()] = field_name + + result: dict[str, list] = {f: [] for f in field_map} + + def _recurse(obj: Any, depth: int) -> None: + if depth >= max_depth: + return + if isinstance(obj, dict): + for k, v in obj.items(): + field = reverse.get(k.lower()) + if field is not None: + result[field].append(v) + _recurse(v, depth + 1) + elif isinstance(obj, list): + for item in obj: + _recurse(item, depth + 1) + + for payload in payloads: + _recurse(payload, 0) + + return result diff --git a/encar_scraper/encar.py b/encar_scraper/encar.py new file mode 100644 index 0000000..60a15f2 --- /dev/null +++ b/encar_scraper/encar.py @@ -0,0 +1,1653 @@ +import json +import logging +import re +import time +from concurrent.futures import ThreadPoolExecutor, as_completed +from dataclasses import dataclass, field, replace +from pathlib import Path +from typing import Any +from urllib.error import HTTPError, URLError +from urllib.parse import urlencode, quote +from urllib.request import Request, urlopen + +import urllib3 + +from .core.config import Settings +from .storage.db import PersistenceService +from .storage.schemas import CarRecord, ImageRecord + +logger = logging.getLogger("encar_scraper") + +ENCAR_LISTING_API = "https://api.encar.com/search/car/list/general" +ENCAR_DETAIL_API_TEMPLATE = "https://api.encar.com/v1/readside/vehicle/{vehicle_id}" +ENCAR_PHOTO_API_TEMPLATE = "https://api.encar.com/legacy/usedcar/sale/car/photo/{vehicle_id}" +# Batch endpoint из фронтенд JS (fem.encar.com) — до 20 машин за запрос, все фото +ENCAR_BATCH_VEHICLES_API = "https://api.encar.com/v1/readside/vehicles" +BATCH_VEHICLES_CHUNK_SIZE = 20 +ENCAR_DETAIL_URL_TEMPLATE = "https://www.encar.com/dc/dc_cardetailview.do?carid={vehicle_id}" +ENCAR_IMAGE_BASE = "https://ci.encar.com" +VEHICLE_ID_RE = re.compile(r"(?:carid|vehicleId)=?(\d+)") + +# Максимальный номер фото для пробинга и допустимые промахи подряд +PHOTO_PROBE_MAX = 50 +PHOTO_PROBE_MAX_CONSECUTIVE_MISS = 7 + +# Типы листинга +CAR_TYPE_ALL = "A" # все (отечественные + импорт) +CAR_TYPE_DOMESTIC = "Y" # только отечественные +CAR_TYPE_IMPORT = "N" # только импорт +CAR_TYPE_ELECTRIC = "G" # электро/гибрид + +# Корейские → латинские для fuel +FUEL_TYPE_MAP = { + "가솔린": "petrol", + "gasoline": "petrol", + "디젤": "diesel", + "diesel": "diesel", + "전기": "electric", + "electric": "electric", + "하이브리드": "hybrid", + "hybrid": "hybrid", + "lpg": "lpg", + "LPG": "lpg", + "LPG(일반인 구입)": "lpg", + "LPG(장애인 구입)": "lpg", + "가솔린+전기": "hybrid", + "디젤+전기": "hybrid", + "CNG": "cng", +} + +GEARBOX_MAP_KO = { + "오토": "AT", + "자동": "AT", + "수동": "MT", + "cvt": "CVT", + "CVT": "CVT", +} + +SELL_TYPE_MAP = { + "일반": "STOCK", + "경매": "AUCTION", + "입찰": "TENDER", +} + +CITY_TO_COUNTRY = { + "서울": "KR", "경기": "KR", "인천": "KR", "부산": "KR", "대구": "KR", + "광주": "KR", "대전": "KR", "울산": "KR", "세종": "KR", "강원": "KR", + "충북": "KR", "충남": "KR", "전북": "KR", "전남": "KR", "경북": "KR", + "경남": "KR", "제주": "KR", +} + +# Переводы брендов с корейского/английского на русский +BRAND_TRANSLATIONS = { + "현대": "Хёндэ", + "기아": "Киа", + "쉐보레(GM대우)": "Шевроле", + "벤츠": "Мерседес-Бенц", + "BMW": "BMW", + "아우디": "Ауди", + "폭스바겐": "Фольксваген", + "토요타": "Тойота", + "도요타": "Тойота", + "닛산": "Ниссан", + "혼다": "Хонда", + "포르쉐": "Порше", + "랜드로버": "Ленд Ровер", + "재규어": "Ягуар", + "볼보": "Вольво", + "푸조": "Пежо", + "시트로엥": "Ситроен", + "르노": "Рено", + "미니": "Мини", + "링컨": "Линкольн", + "캐딜락": "Кадиллак", + "크라이슬러": "Крайслер", + "지프": "Джип", + "포드": "Форд", + "테슬라": "Тесла", + "폴크스바겐": "Фольксваген", + "스코다": "Шкода", + "세아트": "SEAT", + "오펠": "Опель", + "피아트": "Фиат", + "알파로메오": "Альфа Ромео", + "마세라티": "Мазерати", + "람보르기니": "Ламборгини", + "페라리": "Феррари", + "벤틀리": "Бентли", + "롤스로이스": "Роллс-Ройс", + "애스턴마틴": "Астон Мартин", + "맥라렌": "Макларен", + "부가티": "Бугатти", + "코닉세그": "Кёнигсегг", + "파가니": "Пагани", + "헤네시": "Hennessey", + "쉘비": "Shelby", + "다임러": "Daimler", + "마이바흐": "Майбах", + "스마트": "Смарт", + "DS": "DS", + "렉서스": "Лексус", + "인피니티": "Инфинити", + "아큐라": "Акура", + "스바루": "Субару", + "미쓰비시": "Мицубиси", + "마쓰다": "Мазда", + "이스즈": "Исузу", + "다이하츠": "Daihatsu", + "스즈키": "Сузуки", + "KG모빌리티(쌍용)": "KG Mobility", + "르노코리아(삼성)": "Рено Корея", + "한국GM": "GM Korea", + "쌍용": "СсангЙонг", + "대우": "Дэу", + "제네시스": "Генезис", + "르노삼성": "Рено Самсунг", + "닷지": "Dodge", + "험머": "Hummer", + "폴스타": "Polestar", + "사브": "Saab", + "로터스": "Lotus", + "리비안": "Rivian", + "루시드": "Lucid", + "리막": "Rimac", + "알파 로메오": "Альфа Ромео", + "마쯔다": "Мазда", + "어큐라": "Акура", + "다이하쯔": "Daihatsu", +} + +# Маппинг английских названий брендов → все варианты (корейский + русский перевод). +# Используется для расширения фильтра allowed_brands из runtime_config. +BRAND_ENGLISH_ALIASES: dict[str, list[str]] = { + "hyundai": ["현대", "Хёндэ"], + "kia": ["기아", "Киа"], + "genesis": ["제네시스", "Генезис"], + "chevrolet": ["쉐보레(GM대우)", "쉐보레", "Шевроле"], + "mercedes": ["벤츠", "Мерседес-Бенц"], + "mercedes-benz": ["벤츠", "Мерседес-Бенц"], + "bmw": ["BMW"], + "audi": ["아우디", "Ауди"], + "volkswagen": ["폭스바겐", "폴크스바겐", "Фольксваген"], + "toyota": ["토요타", "도요타", "Тойота"], + "nissan": ["닛산", "Ниссан"], + "honda": ["혼다", "Хонда"], + "porsche": ["포르쉐", "Порше"], + "land rover": ["랜드로버", "Ленд Ровер"], + "jaguar": ["재규어", "Ягуар"], + "volvo": ["볼보", "Вольво"], + "peugeot": ["푸조", "Пежо"], + "citroen": ["시트로엥", "Ситроен", "시트로엥/DS"], + "renault": ["르노", "Рено"], + "mini": ["미니", "Мини"], + "lincoln": ["링컨", "Линкольн"], + "cadillac": ["캐딜락", "Кадиллак"], + "chrysler": ["크라이슬러", "Крайслер"], + "jeep": ["지프", "Джип"], + "ford": ["포드", "Форд"], + "tesla": ["테슬라", "Тесла"], + "lexus": ["렉서스", "Лексус"], + "infiniti": ["인피니티", "Инфинити"], + "acura": ["아큐라", "Акура"], + "subaru": ["스바루", "Субару"], + "mitsubishi": ["미쓰비시", "Мицубиси"], + "mazda": ["마쓰다", "Мазда"], + "suzuki": ["스즈키", "Сузуки"], + "ferrari": ["페라리", "Феррари"], + "lamborghini": ["람보르기니", "Ламборгини"], + "maserati": ["마세라티", "Мазерати"], + "bentley": ["벤틀리", "Бентли"], + "rolls-royce": ["롤스로이스", "Роллс-Ройс"], + "aston martin": ["애스턴마틴", "Астон Мартин"], + "mclaren": ["맥라렌", "Макларен"], + "ssangyong": ["쌍용", "СсангЙонг", "KG모빌리티(쌍용)", "KG Mobility"], + "daewoo": ["대우", "Дэу"], + "dodge": ["닷지"], + "hummer": ["험머"], + "gmc": ["GMC"], + "polestar": ["폴스타"], + "ds": ["DS", "시트로엥/DS"], + "renault korea": ["르노코리아(삼성)", "Рено Корея", "르노삼성", "Рено Самсунг"], + "saab": ["사브"], + "lotus": ["로터스"], + "alfa romeo": ["알파로메오", "Альфа Ромео", "알파 로메오"], + "fiat": ["피아트", "Фиат"], + "smart": ["스마트", "Смарт"], + "skoda": ["스코다", "Шкода"], + "daihatsu": ["다이하츠", "다이하쯔", "Daihatsu"], + "koenigsegg": ["코닉세그", "Кёнигсегг"], + "pagani": ["파가니", "Пагани"], + "bugatti": ["부가티", "Бугатти"], + "maybach": ["마이바흐", "Майбах"], + "rivian": ["리비안"], + "lucid": ["루시드"], + "rimac": ["리막"], +} + + +def expand_allowed_brands(brands: set[str] | None) -> set[str] | None: + """Расширяет набор английских брендов всеми известными alias'ами (корейский + русский).""" + if not brands: + return brands + expanded = set(brands) + for brand in brands: + aliases = BRAND_ENGLISH_ALIASES.get(brand.lower(), []) + for alias in aliases: + expanded.add(alias.lower()) + return expanded + +MODEL_TRANSLATIONS = { + "가솔린": "бензин", + "디젤": "дизель", + "하이브리드": "гибрид", + "전기": "электро", + "터보": "турбо", + "쿠페": "купе", + "세단": "седан", + "왜건": "универсал", + "해치백": "хэтчбек", + "컨버터블": "кабриолет", + "오토": "автомат", + "수동": "механика", + "프레스티지": "престиж", + "시그니처": "сигнатур", + "럭셔리": "люкс", + "스포츠": "спорт", + "롱레인지": "дальний запас хода", + "스탠다드": "стандарт", + "플러스": "плюс", + "리미티드": "лимитед", + "스페셜": "спешл", + "모던": "модерн", + "익스클루시브": "эксклюзив", + "캘리그래피": "каллиграфи", + "노블레스": "ноблесс", + "트림": "трим", + "후륜": "задний привод", + "전륜": "передний привод", + "사륜": "полный привод", +} + +# Переводы цветов с корейского на английский +COLOR_TRANSLATIONS = { + "흰색": "white", + "검정색": "black", + "회색": "gray", + "은색": "silver", + "진회색": "dark gray", + "밝은 회색": "light gray", + "청색": "blue", + "파란색": "blue", + "진청색": "dark blue", + "밝은 청색": "light blue", + "녹색": "green", + "초록색": "green", + "진녹색": "dark green", + "밝은 녹색": "light green", + "적색": "red", + "빨간색": "red", + "진적색": "dark red", + "밝은 적색": "light red", + "주황색": "orange", + "노란색": "yellow", + "갈색": "brown", + "베이지색": "beige", + "아이보리": "ivory", + "크림색": "cream", + "보라색": "purple", + "분홍색": "pink", + "금색": "gold", + "은색": "silver", + "청금색": "blue-gold", + "청은색": "blue-silver", + "흑청색": "black-blue", + "흑적색": "black-red", + "흑녹색": "black-green", + "흑회색": "black-gray", + "흰청색": "white-blue", + "흰적색": "white-red", + "흰녹색": "white-green", + "흰회색": "white-gray", + "청적색": "blue-red", + "청녹색": "blue-green", + "청회색": "blue-gray", + "적녹색": "red-green", + "적회색": "red-gray", + "녹회색": "green-gray", + "기타": "other", + "기타색상": "other", + "특별색상": "special", + "무채색": "neutral", + "메탈릭": "metallic", + "펄": "pearl", + "무광": "matte", + "광택": "glossy", +} + +# Типы кузова +BODY_TYPE_MAP = { + "세단": "SEDAN", + "SUV": "SUV", + "SUV ": "SUV", + "해치백": "HATCHBACK", + "왜건": "WAGON", + "컨버터블": "CONVERTIBLE", + "쿠페": "COUPE", + "픽업": "PICKUP", + "밴": "VAN", + "트럭": "TRUCK", + "버스": "BUS", + "기타": "OTHER", + "승용차": "SEDAN", + "승합차": "VAN", + "화물차": "TRUCK", + "특장차": "SPECIAL", + "경차": "COMPACT", + "중형차": "MID_SIZE", + "대형차": "FULL_SIZE", + "스포츠카": "SPORTS", + "클래식카": "CLASSIC", + "전기차": "ELECTRIC", + "하이브리드": "HYBRID", +} + +# Определение body_type по модели +MODEL_BODY_TYPE_MAP: dict[str, str] = { + # SUV / кроссоверы + "투싼": "SUV", "싼타페": "SUV", "팰리세이드": "SUV", "코나": "SUV", "베뉴": "SUV", + "크레타": "SUV", "넥쏘": "SUV", "아이오닉5": "SUV", "아이오닉6": "SEDAN", + "셀토스": "SUV", "스포티지": "SUV", "쏘렌토": "SUV", "모하비": "SUV", + "니로": "SUV", "EV6": "SUV", "EV9": "SUV", + "티볼리": "SUV", "코란도": "SUV", "렉스턴": "SUV", "토레스": "SUV", "액티언": "SUV", + "GV70": "SUV", "GV80": "SUV", "GV60": "SUV", + "X1": "SUV", "X2": "SUV", "X3": "SUV", "X4": "SUV", "X5": "SUV", "X6": "SUV", "X7": "SUV", "iX": "SUV", "iX3": "SUV", + "Q2": "SUV", "Q3": "SUV", "Q4": "SUV", "Q5": "SUV", "Q7": "SUV", "Q8": "SUV", + "GLA": "SUV", "GLB": "SUV", "GLC": "SUV", "GLE": "SUV", "GLS": "SUV", + "카이엔": "SUV", "마칸": "SUV", + "레인지로버": "SUV", "디스커버리": "SUV", "디펜더": "SUV", + "래브4": "SUV", "하이랜더": "SUV", "FJ 크루져": "SUV", + "CR-V": "SUV", "HR-V": "SUV", + "티구안": "SUV", "투아렉": "SUV", "아틀라스": "SUV", "ID.4": "SUV", + "트레일블레이저": "SUV", "이쿼녹스": "SUV", "트래버스": "SUV", "트랙스": "SUV", "캡티바": "SUV", "올란도": "SUV", + "XC40": "SUV", "XC60": "SUV", "XC90": "SUV", "V40": "HATCHBACK", "V60": "STATION_WAGON", "V90": "STATION_WAGON", + "F-PACE": "SUV", "E-PACE": "SUV", "I-PACE": "SUV", + "모델 Y": "SUV", "Model Y": "SUV", "모델 X": "SUV", "Model X": "SUV", + "RX": "SUV", "NX": "SUV", "UX": "SUV", "LX": "SUV", + "QX50": "SUV", "QX60": "SUV", "QX80": "SUV", + "QM3": "SUV", "QM6": "SUV", + "레니게이드": "SUV", "컴패스": "SUV", "체로키": "SUV", "랭글러": "SUV", "글래디에이터": "PICKUP", + "에비에이터": "SUV", "노틸러스": "SUV", "코세어": "SUV", + "마칸": "SUV", "카이엔": "SUV", + "르반떼": "SUV", "그레칼레": "SUV", + "Z4": "CONVERTIBLE", + # 세단 + "쏘나타": "SEDAN", "아반떼": "SEDAN", "그랜저": "SEDAN", "K5": "SEDAN", + "K8": "SEDAN", "K9": "SEDAN", "K3": "SEDAN", "K7": "SEDAN", + "G70": "SEDAN", "G80": "SEDAN", "G90": "SEDAN", + "1시리즈": "HATCHBACK", + "2시리즈 그란쿠페": "SEDAN", "2시리즈 액티브투어러": "MINIVAN", + "3시리즈": "SEDAN", "3시리즈 GT": "SEDAN", + "5시리즈": "SEDAN", "7시리즈": "SEDAN", "i4": "SEDAN", "i7": "SEDAN", + "A1": "HATCHBACK", "A3": "SEDAN", "A4": "SEDAN", "A6": "SEDAN", "A7": "SEDAN", "A8": "SEDAN", + "A-클래스": "SEDAN", "CLA": "SEDAN", + "C-클래스": "SEDAN", "E-클래스": "SEDAN", "S-클래스": "SEDAN", + "캠리": "SEDAN", "아발론": "SEDAN", "프리우스": "SEDAN", + "어코드": "SEDAN", "시빅": "SEDAN", + "파사트": "SEDAN", "제타": "SEDAN", "아테온": "SEDAN", + "말리부": "SEDAN", "크루즈": "SEDAN", "임팔라": "SEDAN", + "S60": "SEDAN", "S90": "SEDAN", + "IS": "SEDAN", "ES": "SEDAN", "GS": "SEDAN", "LS": "SEDAN", + "Q50": "SEDAN", "Q70": "SEDAN", + "모델 3": "SEDAN", "Model 3": "SEDAN", "모델 S": "SEDAN", "Model S": "SEDAN", + "SM6": "SEDAN", "SM5": "SEDAN", + # 해치백 + "스파크": "HATCHBACK", "모닝": "HATCHBACK", "레이": "HATCHBACK", + "i30": "HATCHBACK", "벨로스터": "HATCHBACK", + "골프": "HATCHBACK", "폴로": "HATCHBACK", "ID.3": "HATCHBACK", + "캐스퍼": "HATCHBACK", + # 미니밴/밴 + "카니발": "MINIVAN", "스타리아": "MINIVAN", "스타렉스": "MINIVAN", + "V-클래스": "MINIVAN", "비토": "VAN", + "시에나": "MINIVAN", "알파드": "MINIVAN", + # 트럭/픽업 + "포터": "PICKUP", "봉고": "PICKUP", "마이티": "TRUCK", + "타코마": "PICKUP", "터프": "PICKUP", + "콜로라도": "PICKUP", "실버라도": "PICKUP", + # 쿠페 + "4시리즈": "COUPE", "8시리즈": "COUPE", "2시리즈": "COUPE", + "A5": "COUPE", "TT": "COUPE", + "RC": "COUPE", "LC": "COUPE", +} + +# Regex для парсинга объёма двигателя из Badge +ENGINE_VOLUME_RE = re.compile(r"(\d+\.\d+)\s*(?:T|터보)?") +# Regex для европейских кодов: "530i", "C200", "A220", "GLC300", "740d", "M340i", "S63", "120i", "E220d" и т.д. +# Ищем 3-значные числа в начале или после буквы: 530, 200, 220, 300, 740, 340 ... +EURO_ENGINE_CODE_RE = re.compile( + r"(?:^|[A-Za-z])(\d{3})(?:[a-zA-Z]|$|\s)" # C200, A220, 530i, 740d, etc. +) +# Маппинг европейских кодов → объём (cc). Первые 1-2 цифры = литраж +EURO_CODE_ENGINE_MAP: dict[str, int] = { + # BMW + "118": 1500, "120": 2000, "125": 2000, "128": 2000, + "218": 1500, "220": 2000, "225": 2000, "228": 2000, "230": 2000, "235": 2000, + "318": 1500, "320": 2000, "325": 2500, "328": 2000, "330": 2000, "335": 3000, "340": 3000, + "418": 2000, "420": 2000, "425": 2000, "428": 2000, "430": 2000, "435": 3000, "440": 3000, + "520": 2000, "525": 2500, "528": 2000, "530": 2000, "535": 3000, "540": 3000, + "620": 2000, "630": 3000, "640": 3000, "650": 4400, + "725": 2500, "730": 3000, "740": 3000, "745": 3000, "750": 4400, "760": 6600, + "X20": 2000, "X25": 2500, "X28": 2000, "X30": 2000, "X35": 3000, "X40": 3000, "X50": 4400, + # Mercedes-Benz + "160": 1600, "180": 1600, "200": 2000, "220": 2000, "250": 2000, "260": 2000, + "280": 3000, "300": 3000, "350": 3500, "400": 3000, "450": 3000, + "500": 4000, "550": 4700, "560": 5500, "600": 6000, "630": 6300, + "43": 3000, "53": 3000, "55": 5500, "63": 4000, + # Audi + "25": 1400, "30": 2000, "35": 2000, "40": 2000, "45": 2000, "50": 3000, "55": 3000, +} + +# Маппинг для извлечения engine_volume из числовых кодов типа "M16 GDI" +BADGE_SHORTCODE_RE = re.compile(r"M(\d{2})\s") # M16 GDI → 1600 + +@dataclass +class EncarFilters: + """Фильтры для API запросов.""" + car_type: str = CAR_TYPE_ALL # A=все, Y=корейские, N=импорт + manufacturer: str | None = None # BMW, 현대, etc. + model: str | None = None + year_from: int | None = None + year_to: int | None = None + price_from: int | None = None # в 만원 (10k KRW) + price_to: int | None = None + mileage_max: int | None = None + + def build_query(self) -> str: + """Строит строку запроса для API.""" + parts = ["Hidden.N"] + + if self.car_type: + parts.append(f"CarType.{self.car_type}") + + if self.manufacturer: + parts.append(f"Manufacturer.{self.manufacturer}") + + if self.model: + parts.append(f"Model.{self.model}") + + # Год: Year в формате YYYYMM (напр. 202001) + # year_from/year_to > 9999 → уже в формате YYYYMM, иначе YYYY → прибавляем 01/12 + if self.year_from or self.year_to: + if self.year_from and self.year_from > 9999: + year_min = str(self.year_from) + else: + year_min = f"{self.year_from}01" if self.year_from else "" + if self.year_to and self.year_to > 9999: + year_max = str(self.year_to) + else: + year_max = f"{self.year_to}12" if self.year_to else "" + parts.append(f"Year.range({year_min}..{year_max})") + + # Цена в 만원 + if self.price_from or self.price_to: + p_min = str(self.price_from) if self.price_from else "" + p_max = str(self.price_to) if self.price_to else "" + parts.append(f"Price.range({p_min}..{p_max})") + + # Пробег + if self.mileage_max: + parts.append(f"Mileage.range(..{self.mileage_max})") + + return "(And." + "._." .join(parts) + ".)" + + +@dataclass(slots=True) +class EncarMapper: + def map_to_car_record(self, vehicle_url: str, payload: dict[str, Any], probe_all_photos: bool = False) -> CarRecord: + vehicle_id = self._extract_vehicle_id(vehicle_url) or str( + payload.get("Id") or payload.get("vehicleId") or "" + ) + origin_id = f"encar:{vehicle_id}" + parser_id = f"encar:{vehicle_id}" + origin_url = vehicle_url or ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=vehicle_id) + + brand = self._as_str(payload.get("Manufacturer") or payload.get("Brand") or "UNKNOWN") + # Перевод бренда на русский + brand = BRAND_TRANSLATIONS.get(brand, brand) + + # Model + Badge + BadgeDetail = полное название + model_raw = self._as_str(payload.get("Model") or "UNKNOWN") + badge = self._as_str(payload.get("Badge") or "") + badge_detail = self._as_str(payload.get("BadgeDetail") or "") + + # Полное описание модели: "Model / Badge BadgeDetail" + model_parts = [model_raw] + if badge: + model_parts.append(badge) + if badge_detail and badge_detail not in badge: + model_parts.append(badge_detail) + model = " / ".join(model_parts) if len(model_parts) > 1 else model_parts[0] + model = self._translate_text(model) + + year = self._to_year(payload.get("FormYear") or payload.get("Year")) + # Price в 만원 (10 000 KRW) → переводим в KRW + price_man_won = self._to_float(payload.get("Price")) + price = int(price_man_won * 10_000) if price_man_won is not None else None + mileage = self._to_int(payload.get("Mileage")) + + # Цвет: листинг не содержит цвет, проверяем все варианты + color_raw = self._as_str( + payload.get("ExteriorColor") or payload.get("Color") or "" + ) + color = self._translate_color(color_raw) if color_raw else "other" + + currency = "KRW" + country = "KR" + + # Топливо и коробка передач + fuel_raw = self._as_str(payload.get("FuelType") or "") + green_type = self._as_str(payload.get("GreenType") or "") + + # Gearbox: Encar листинг не даёт Transmission, определяем по топливу + if fuel_raw == "전기": + # Чистый электромобиль + gearbox = "EV" + else: + # Гибрид (가솔린+전기, 디젤+전기) или ДВС → AT + gearbox_raw = self._as_str(payload.get("Transmission") or "") + if gearbox_raw: + gearbox = GEARBOX_MAP_KO.get(gearbox_raw) or GEARBOX_MAP_KO.get(gearbox_raw.lower()) or "AT" + else: + gearbox = "AT" + + sell_type_raw = self._as_str(payload.get("SellType") or "") + selling_type = SELL_TYPE_MAP.get(sell_type_raw, "STOCK") + + # Парсинг drive из Badge + drive = self._parse_drive(badge, model_raw) + + # Парсинг body_type из Model (по ключевым словам в модели) + body_type_raw = self._as_str(payload.get("BodyType") or "") + if body_type_raw: + body_type = BODY_TYPE_MAP.get(body_type_raw, "OTHER") + else: + body_type = self._detect_body_type(model_raw) + + # Парсинг engine_volume из Badge (напр. "가솔린 2.5 터보 AWD" → 2500) + engine_volume_raw = payload.get("Displacement") or payload.get("EngineCc") + if engine_volume_raw: + engine_volume = self._to_int(engine_volume_raw) + else: + engine_volume = self._parse_engine_volume(badge) + + # "엔카진단" флаги + service_marks = payload.get("ServiceMark") or [] + condition_list = payload.get("Condition") or [] + has_inspection = bool(condition_list) or ( + "EncarDiagnosisP1" in service_marks or "EncarDiagnosisP2" in service_marks + ) + + photo_prefix = self._as_str(payload.get("Photo") or "") + images = self._map_images( + payload.get("Photos") or [], + probe_all=probe_all_photos, + photo_prefix=photo_prefix or None, + ) + + return CarRecord( + parser_id=parser_id, + brand=brand, + model=model, + year=year, + price=price, + currency=currency, + mileage=mileage or 0, + country=country, + is_sold=False, + color=color or "other", + drive=drive, + gearbox=gearbox, + steering_wheel="LEFT", # Корея — левый руль + body_type=body_type, + engine_volume=engine_volume, + selling_type=selling_type, + one_owner=False, + new_car=False, + is_hidden=False, + origin="ENCAR", + origin_url=origin_url, + origin_id=origin_id, + is_damaged=False, + evaluation="diagnosis" if has_inspection else None, + non_smoking=False, + rental=False, + repair_history=False, + slug=self._build_slug(brand, model, vehicle_id), + images=images, + ) + + def _extract_vehicle_id(self, vehicle_url: str | None) -> str | None: + if not vehicle_url: + return None + match = VEHICLE_ID_RE.search(vehicle_url) + return match.group(1) if match else None + + def _build_slug(self, brand: str, model: str, vehicle_id: str) -> str: + raw = re.sub(r"[^a-z0-9]+", "-", f"{brand}-{model}-{vehicle_id}".lower()).strip("-") + return raw or f"encar-{vehicle_id}" + + def _as_str(self, value: Any) -> str: + return str(value).strip() if value is not None else "" + + def _translate_text(self, text: str) -> str: + translated = self._as_str(text) + for source, target in sorted(MODEL_TRANSLATIONS.items(), key=lambda item: len(item[0]), reverse=True): + translated = translated.replace(source, target) + translated = re.sub(r"\s+", " ", translated).strip() + return translated + + def _translate_color(self, color_raw: str) -> str: + value = self._as_str(color_raw) + if not value: + return "other" + return COLOR_TRANSLATIONS.get(value, self._translate_text(value).lower() or "other") + + def _to_float(self, value: Any) -> float | None: + if value is None: + return None + try: + return float(value) + except (ValueError, TypeError): + digits = re.sub(r"[^0-9.]", "", str(value)) + return float(digits) if digits else None + + def _to_int(self, value: Any) -> int | None: + f = self._to_float(value) + return int(f) if f is not None else None + + def _to_year(self, value: Any) -> int | None: + """Парсинг года. + - FormYear: "2021" → 2021 + - Year: 202102.0 → YYYYMM формат → берём YYYY часть + """ + if value is None: + return None + try: + s = str(value).strip().split(".")[0] # убираем .0 + if len(s) >= 6: + # YYYYMM формат + return int(s[:4]) + if len(s) == 4: + return int(s) + except (ValueError, TypeError): + pass + return None + + def _map_images(self, photos: Any, probe_all: bool = False, photo_prefix: str | None = None) -> list[ImageRecord]: + if not isinstance(photos, list): + photos = [] + images: list[ImageRecord] = [] + known_orders: set[int] = set() + url_prefix: str | None = None + + for raw in photos: + if not isinstance(raw, dict): + continue + location = raw.get("location") or raw.get("Location") or "" + if not location: + continue + url = self._build_image_url(str(location)) + order = int(raw.get("ordering") or 0) + images.append(ImageRecord(fullres_image=url, preview_image=url, order_index=order)) + known_orders.add(order) + if url_prefix is None and "_" in location: + url_prefix = self._build_image_url(location.rsplit("_", 1)[0]) + + # Если Photos не дал prefix, используем Photo (путь-префикс из premium API) + if url_prefix is None and photo_prefix: + url_prefix = self._build_image_url(photo_prefix.rstrip("_")) + + if probe_all and url_prefix: + extra = self._probe_additional_photos(url_prefix, known_orders) + images.extend(extra) + + images.sort(key=lambda item: item.order_index) + return images + + @staticmethod + def _head_check(url: str) -> bool: + """Проверка существования фото через HEAD-запрос.""" + try: + req = Request(url, method="HEAD", headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Referer": "https://www.encar.com/", + }) + with urlopen(req, timeout=3) as resp: + return resp.status == 200 + except (HTTPError, URLError, OSError): + return False + + @staticmethod + def _head_check_pooled(pool: urllib3.HTTPSConnectionPool, path: str) -> bool: + """HEAD-запрос через connection pool (быстрее за счёт reuse TCP/SSL).""" + try: + resp = pool.request( + "HEAD", path, + headers={ + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36", + "Referer": "https://www.encar.com/", + }, + timeout=2.0, + retries=False, + ) + return resp.status == 200 + except Exception: + return False + + def _probe_additional_photos(self, url_prefix: str, known_orders: set[int]) -> list[ImageRecord]: + """Пробинг дополнительных фото параллельно через HEAD-запросы.""" + candidates = [] + for i in range(1, PHOTO_PROBE_MAX + 1): + if i in known_orders: + continue + candidates.append((i, f"{url_prefix}_{i:03d}.jpg")) + + if not candidates: + return [] + + found: list[ImageRecord] = [] + with ThreadPoolExecutor(max_workers=10) as pool: + futures = {pool.submit(self._head_check, url): (order, url) for order, url in candidates} + for future in as_completed(futures): + order, url = futures[future] + if future.result(): + found.append(ImageRecord(fullres_image=url, preview_image=url, order_index=order)) + + found.sort(key=lambda x: x.order_index) + + # Отсекаем хвост после PHOTO_PROBE_MAX_CONSECUTIVE_MISS промахов подряд + all_found_orders = known_orders | {img.order_index for img in found} + cutoff = PHOTO_PROBE_MAX + 1 + consecutive_miss = 0 + for i in range(1, PHOTO_PROBE_MAX + 1): + if i in all_found_orders: + consecutive_miss = 0 + else: + consecutive_miss += 1 + if consecutive_miss >= PHOTO_PROBE_MAX_CONSECUTIVE_MISS: + cutoff = i - PHOTO_PROBE_MAX_CONSECUTIVE_MISS + 1 + break + + return [img for img in found if img.order_index < cutoff] + + def _build_image_url(self, location: str) -> str: + if location.startswith("http"): + return location + if location.startswith("/"): + return f"{ENCAR_IMAGE_BASE}{location}" + return f"{ENCAR_IMAGE_BASE}/{location}" + + def _parse_drive(self, badge: str, model: str) -> str | None: + """Парсит тип привода из Badge и Model.""" + text = f"{badge} {model}".lower() + if "awd" in text or "4wd" in text or "사륜" in text or "4륜" in text: + return "4WD" + elif "2wd" in text or "fwd" in text or "이륜" in text or "전륜" in text: + return "FWD" + elif "rwd" in text or "후륜" in text: + return "RWD" + return None + + def _parse_engine_volume(self, badge: str) -> int | None: + """Парсит объём двигателя из Badge. + + Примеры: + "가솔린 2.5 터보 AWD" → 2500 + "1.6 GDI 유니크" → 1600 + "3.8 4WD" → 3800 + "530i 럭셔리" → 2000 (BMW code) + "C200 AMG Line" → 2000 (Mercedes code) + "롱레인지 AWD" → None (электро, нет объёма) + """ + if not badge: + return None + + # 1. Прямой литраж: "2.5", "1.6", "3.8" + m = ENGINE_VOLUME_RE.search(badge) + if m: + vol = float(m.group(1)) + if 0.5 <= vol <= 8.0: + return int(vol * 1000) + + # 2. Европейские коды: "530i", "C200", "GLC300", "M340i", "E220d" + euro_match = EURO_ENGINE_CODE_RE.search(badge) + if euro_match: + code = euro_match.group(1) + if code in EURO_CODE_ENGINE_MAP: + return EURO_CODE_ENGINE_MAP[code] + + # 3. Корейские shortcode: "M16 GDI" → 1600 + m_short = BADGE_SHORTCODE_RE.search(badge) + if m_short: + code = int(m_short.group(1)) + if 10 <= code <= 60: + return code * 100 + + return None + + def _detect_body_type(self, model_raw: str) -> str: + """Определяет тип кузова из названия модели.""" + if not model_raw: + return "OTHER" + # Проверяем каждый ключ из MODEL_BODY_TYPE_MAP + for keyword, body in MODEL_BODY_TYPE_MAP.items(): + if keyword in model_raw: + return body + return "OTHER" + + +class EncarScraper: + DEFAULT_PAGE_SIZE = 1000 # API поддерживает до 1000 + + def __init__(self, settings: Settings | None = None) -> None: + self.settings = settings or Settings() + self.persistence = PersistenceService(self.settings) + self.car_mapper = EncarMapper() + self._batch_pool: urllib3.HTTPSConnectionPool | None = None + + def _get_batch_pool(self) -> urllib3.HTTPSConnectionPool: + """Возвращает переиспользуемый connection pool для batch photo API.""" + if self._batch_pool is None: + self._batch_pool = urllib3.HTTPSConnectionPool( + "api.encar.com", + port=443, + maxsize=30, + block=True, + headers={ + "User-Agent": self.settings.fingerprint.user_agent, + "Accept": "application/json", + }, + cert_reqs="CERT_REQUIRED", + ) + return self._batch_pool + + def collect_listing( + self, + limit: int | None = None, + page_size: int = DEFAULT_PAGE_SIZE, + filters: EncarFilters | None = None, + ) -> dict[str, Any]: + """Собирает список авто из API.""" + filters = filters or EncarFilters() + vehicle_items: list[dict] = [] + page = 0 + total = None + + logger.info("Collecting Encar listing with query: %s", filters.build_query()) + + while True: + offset = page * page_size + response = self._fetch_listing_page(offset=offset, limit=page_size, filters=filters) + results = response.get("SearchResults") or [] + + if total is None: + total = int(response.get("Count") or len(results)) + logger.info("Total available: %d", total) + + if not results: + break + + for item in results: + vehicle_id = str(item.get("Id") or "") + if not vehicle_id: + continue + vehicle_items.append(item) + if limit is not None and len(vehicle_items) >= limit: + break + + if limit is not None and len(vehicle_items) >= limit: + break + if len(results) < page_size: + break + + page += 1 + if page % 10 == 0: + logger.info("Collected %d items from %d pages...", len(vehicle_items), page) + + logger.info("Finished collecting: %d items from %d pages", len(vehicle_items), page + 1) + + return { + "items": vehicle_items, + "vehicle_urls": [ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=item["Id"]) for item in vehicle_items], + "pages_collected": page + 1, + "items_collected": len(vehicle_items), + "total_available": total, + } + + def scrape_vehicle_detail(self, vehicle_url: str) -> dict[str, Any]: + """Получает детальную информацию по одному авто.""" + vehicle_id = self._extract_vehicle_id(vehicle_url) + if not vehicle_id: + raise ValueError(f"Cannot extract Encar vehicle id from {vehicle_url}") + + # Сначала пробуем получить из листинга (там больше данных) + payload = self._fetch_listing_item(vehicle_id) + + # Если нет в листинге — берём из детального API + if payload is None: + payload = self._fetch_vehicle_detail_api(vehicle_id) + + if payload is None: + raise ValueError(f"Encar vehicle {vehicle_id} not found") + + record = self.car_mapper.map_to_car_record(vehicle_url, payload) + return { + "vehicle_id": vehicle_id, + "payload": payload, + "db_record": record.model_dump(mode="json"), + } + + def init_db(self) -> dict[str, Any]: + self.persistence.create_tables() + return {"status": "ok"} + + def sync_vehicle(self, vehicle_url: str, lane: str = "encar") -> dict[str, Any]: + """Синхронизирует одно авто в БД.""" + self.persistence.create_tables() + payload = self.scrape_vehicle_detail(vehicle_url) + record = CarRecord.model_validate(payload["db_record"]) + result = self.persistence.upsert_car(record) + return { + "status": "success", + "lane": lane, + "vehicle_url": vehicle_url, + "origin_id": record.origin_id, + **result, + } + + CHECKPOINT_KEY = "encar:sync:checkpoint" + CHECKPOINT_SAVE_INTERVAL = 10 # сохранять чекпоинт каждые N страниц + SHARD_MAX_ITEMS = 9500 # макс. элементов в одном шарде (API лимит ~10k) + + # ------------------------------------------------------------------ sharding + + def _get_listing_count(self, filters: EncarFilters) -> int: + """Получить Count для фильтра без выгрузки данных.""" + resp = self._fetch_listing_page(offset=0, limit=1, filters=filters) + return int(resp.get("Count") or 0) + + def _build_sync_shards(self, base_filters: EncarFilters) -> list[EncarFilters]: + """Авто-сегментация фильтров для обхода лимита пагинации API (~10k). + + Быстрый алгоритм (~15 API calls вместо ~90): + 1. total <= 9500 → 1 шард. + 2. Разбиваем по car_type (Y/N). + 3. Группируем старые годы декадами (2000-2009 → один шард). + 4. Только годы > 9500 разбиваем на полугодия. + + seen_vehicle_ids в sync_listing убирает дубли между шардами. + """ + MAX = self.SHARD_MAX_ITEMS + + total = self._get_listing_count(base_filters) + if total <= MAX: + logger.info("Total %d ≤ %d — single shard", total, MAX) + return [base_filters] + + logger.info("Total %d > %d — building shards", total, MAX) + + if base_filters.car_type in (CAR_TYPE_ALL, None, ""): + car_types = ["Y", "N"] + else: + car_types = [base_filters.car_type] + + shards: list[EncarFilters] = [] + shard_items = 0 + + for ct in car_types: + ct_f = replace(base_filters, car_type=ct) + ct_count = self._get_listing_count(ct_f) + if ct_count == 0: + continue + if ct_count <= MAX: + shards.append(ct_f) + shard_items += ct_count + continue + + # Сначала пробуем группировку: все годы до 2014 одним шардом + old_f = replace(ct_f, year_from=None, year_to=201412) + old_count = self._get_listing_count(old_f) + if old_count > 0: + if old_count <= MAX: + shards.append(old_f) + shard_items += old_count + else: + # Разбиваем на 2 части: до 2009 и 2010-2014 + for ys, ye in [(None, 200912), (201001, 201412)]: + part_f = replace(ct_f, year_from=ys, year_to=ye) + part_c = self._get_listing_count(part_f) + if part_c > 0: + shards.append(part_f) + shard_items += part_c + + # Годы 2015-2027: проверяем каждый + current_year = time.localtime().tm_year + for year in range(2015, current_year + 2): + yf = replace(ct_f, year_from=year, year_to=year) + yc = self._get_listing_count(yf) + if yc == 0: + continue + if yc <= MAX: + shards.append(yf) + shard_items += yc + else: + # Разбиваем на полугодия + for ms, me in [(year * 100 + 1, year * 100 + 6), + (year * 100 + 7, year * 100 + 12)]: + hf = replace(ct_f, year_from=ms, year_to=me) + hc = self._get_listing_count(hf) + if hc == 0: + continue + if hc <= MAX: + shards.append(hf) + shard_items += hc + else: + # Полугодие > MAX — разбиваем на кварталы + q_ranges = [(ms, ms + 2), (ms + 3, me)] if me - ms == 5 else [(ms, ms + 2), (ms + 3, me)] + for qs, qe in q_ranges: + qf = replace(ct_f, year_from=qs, year_to=qe) + qc = self._get_listing_count(qf) + if qc > 0: + if qc > MAX: + logger.warning("Quarter shard %s: %d items (> %d)", qf.build_query(), qc, MAX) + shards.append(qf) + shard_items += qc + + logger.info("Generated %d shards (≈ %d total items)", len(shards), shard_items) + return shards + + # ------------------------------------------------------------------ checkpoint + + def _load_checkpoint(self, redis_client: Any | None) -> dict[str, Any] | None: + """Загрузить чекпоинт из Redis.""" + if redis_client is None: + return None + try: + data = redis_client.get(self.CHECKPOINT_KEY) + if data: + cp = json.loads(data) + logger.info( + "Resuming from checkpoint: shard=%d, page=%d, synced=%d, collected=%d", + cp.get("shard_idx", 0), cp.get("page", 0), + cp.get("synced", 0), cp.get("items_collected", 0), + ) + return cp + except Exception as exc: + logger.warning("Failed to load checkpoint: %s", exc) + return None + + def _save_checkpoint(self, redis_client: Any | None, state: dict[str, Any]) -> None: + """Сохранить чекпоинт в Redis (TTL = 6 часов).""" + if redis_client is None: + return + try: + redis_client.set(self.CHECKPOINT_KEY, json.dumps(state), ex=21600) + except Exception as exc: + logger.warning("Failed to save checkpoint: %s", exc) + + def _clear_checkpoint(self, redis_client: Any | None) -> None: + """Удалить чекпоинт после успешного завершения.""" + if redis_client is None: + return + try: + redis_client.delete(self.CHECKPOINT_KEY) + except Exception: + pass + + def sync_listing( + self, + limit: int | None = None, + filters: EncarFilters | None = None, + lane: str = "encar", + only_new: bool = False, + batch_size: int = 1000, + redis_client: Any | None = None, + allowed_brands: set[str] | None = None, + excluded_brands: set[str] | None = None, + probe_all_photos: bool = False, + ) -> dict[str, Any]: + """Полная синхронизация листинга Encar. + + Encar API ограничивает пагинацию ~10k записей на один query. + Для получения всех 160k+ авто запрос автоматически разбивается + на шарды по car_type → year → quarter (через _build_sync_shards). + + 1. Генерирует шарды (фильтры), каждый ≤ 9500 записей. + 2. Для каждого шарда собирает все авто, upsert в БД. + 3. Помечает is_sold=True авто, которых нет в листинге. + + Чекпоинт через Redis: сохраняет shard_idx + page для продолжения + после рестарта. + """ + self.persistence.create_tables() + filters = filters or EncarFilters() + page_size = self.DEFAULT_PAGE_SIZE + + # --- Генерация шардов --- + if limit: + # С limit не разбиваем на шарды + shards = [filters] + else: + shards = self._build_sync_shards(filters) + + # --- Загрузка чекпоинта --- + checkpoint = self._load_checkpoint(redis_client) + start_shard_idx = 0 + start_page = 0 + total_available = 0 + items_collected = 0 + synced = 0 + failed = 0 + failed_ids: list[str] = [] + all_origin_ids: set[str] = set() + + if checkpoint and not limit: + start_shard_idx = checkpoint.get("shard_idx", 0) + start_page = checkpoint.get("page", 0) + items_collected = checkpoint.get("items_collected", 0) + synced = checkpoint.get("synced", 0) + failed = checkpoint.get("failed", 0) + total_available = checkpoint.get("total_available", 0) + logger.info( + "Checkpoint loaded: shard=%d/%d, page=%d, synced=%d", + start_shard_idx, len(shards), start_page, synced, + ) + + logger.info( + "Full sync Encar: %d shards, base query: %s", + len(shards), filters.build_query(), + ) + + # --- Фаза 1: Сбор и upsert по шардам --- + seen_vehicle_ids: set[str] = set() # Глобальный дедуп между шардами + + for shard_idx, shard_filters in enumerate(shards): + if shard_idx < start_shard_idx: + continue + + page = start_page if shard_idx == start_shard_idx else 0 + start_page = 0 # сбрасываем после первого шарда + + batch: list[Any] = [] + consecutive_errors = 0 + max_consecutive_errors = 5 + consecutive_zero_new = 0 + max_consecutive_zero_new = 5 + + shard_query = shard_filters.build_query() + shard_collected = 0 + + while True: + offset = page * page_size + try: + response = self._fetch_listing_page( + offset=offset, limit=page_size, filters=shard_filters, + ) + except Exception as exc: + consecutive_errors += 1 + logger.warning( + "Shard %d page %d fetch error (%d/%d): %s", + shard_idx, page, consecutive_errors, max_consecutive_errors, exc, + ) + if consecutive_errors >= max_consecutive_errors: + logger.error("Too many errors in shard %d, moving to next", shard_idx) + break + time.sleep(5 * consecutive_errors) + continue + + consecutive_errors = 0 + results = response.get("SearchResults") or [] + + if page == 0: + shard_total = int(response.get("Count") or len(results)) + if shard_idx == 0 or total_available == 0: + total_available = self._get_listing_count(filters) + if len(shards) > 1: + logger.info( + "Shard %d/%d (%s): %d items", + shard_idx + 1, len(shards), shard_query, shard_total, + ) + + if not results: + break + + page += 1 + + new_in_page = 0 + for item in results: + vehicle_id = str(item.get("Id") or "") + if not vehicle_id: + continue + if vehicle_id in seen_vehicle_ids: + continue + seen_vehicle_ids.add(vehicle_id) + batch.append(item) + items_collected += 1 + shard_collected += 1 + new_in_page += 1 + + if limit is not None and items_collected >= limit: + break + + # API зацикливает после ~10k → выход из шарда + if new_in_page == 0: + consecutive_zero_new += 1 + if consecutive_zero_new >= max_consecutive_zero_new: + logger.info( + "Shard %d pagination exhausted at page=%d, collected=%d", + shard_idx, page, shard_collected, + ) + break + else: + consecutive_zero_new = 0 + + # Flush batch + if len(batch) >= batch_size: + s, f, fids = self._flush_batch( + batch, all_origin_ids, allowed_brands, + excluded_brands, probe_all_photos=probe_all_photos, + ) + synced += s + failed += f + failed_ids.extend(fids) + batch = [] + + logger.info( + "Progress: shard=%d/%d, page=%d, collected=%d, synced=%d, failed=%d", + shard_idx + 1, len(shards), page, items_collected, synced, failed, + ) + + # Чекпоинт + if page % self.CHECKPOINT_SAVE_INTERVAL == 0: + self._save_checkpoint(redis_client, { + "shard_idx": shard_idx, + "page": page, + "items_collected": items_collected, + "synced": synced, + "failed": failed, + "total_available": total_available, + }) + + if limit is not None and items_collected >= limit: + break + + if len(results) < page_size: + break + + if page % 10 == 0: + time.sleep(0.5) + + # Flush remaining batch после каждого шарда + if batch: + s, f, fids = self._flush_batch( + batch, all_origin_ids, allowed_brands, + excluded_brands, probe_all_photos=probe_all_photos, + ) + synced += s + failed += f + failed_ids.extend(fids) + batch = [] + + # Чекпоинт после завершения шарда + self._save_checkpoint(redis_client, { + "shard_idx": shard_idx + 1, + "page": 0, + "items_collected": items_collected, + "synced": synced, + "failed": failed, + "total_available": total_available, + }) + + if limit is not None and items_collected >= limit: + break + + # Пауза между шардами + if shard_idx < len(shards) - 1: + time.sleep(1) + + if items_collected == 0: + logger.warning("No items collected from listing") + return { + "status": "empty", + "lane": lane, + "total_available": total_available, + "items_collected": 0, + "cars_synced": 0, + "cars_failed": 0, + "cars_marked_sold": 0, + } + + # --- Фаза 2: Пометить sold --- + # Помечаем sold только если прошли ВСЕ шарды полностью (без resume из середины). + # Иначе all_origin_ids неполный и мы ошибочно пометим активные авто. + marked_sold = 0 + if not limit and checkpoint is None: + marked_sold = self.persistence.mark_sold_not_in_listing( + active_origin_ids=all_origin_ids, lane=lane, + ) + elif not limit: + logger.info( + "Skipping mark_sold: sync resumed from shard %d, origin_ids may be incomplete", + start_shard_idx, + ) + + self._clear_checkpoint(redis_client) + + # Закрываем pool batch API + if self._batch_pool is not None: + self._batch_pool.close() + self._batch_pool = None + + logger.info( + "Full sync complete: %d shards, %d collected, %d synced, %d failed, %d marked sold", + len(shards), items_collected, synced, failed, marked_sold, + ) + + return { + "status": "success", + "lane": lane, + "total_available": total_available, + "items_collected": items_collected, + "cars_synced": synced, + "cars_failed": failed, + "cars_marked_sold": marked_sold, + "shards_count": len(shards), + "failed_ids": failed_ids[:20], + } + + def _flush_batch( + self, + items: list[Any], + all_origin_ids: set[str], + allowed_brands: set[str] | None = None, + excluded_brands: set[str] | None = None, + probe_all_photos: bool = False, + ) -> tuple[int, int, list[str]]: + """Маппит и upsert'ит батч items в БД. Возвращает (synced, failed, failed_ids).""" + records: list[CarRecord] = [] + # Для пакетного пробинга: (record_index, item) пар, прошедших фильтр + probe_tasks: list[tuple[int, Any]] = [] + failed = 0 + failed_ids: list[str] = [] + skipped_brands = 0 + + # Фаза 1: маппинг без пробинга + фильтр брендов + for item in items: + vehicle_id = str(item.get("Id") or "") + try: + vehicle_url = ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=vehicle_id) + record = self.car_mapper.map_to_car_record(vehicle_url, item, probe_all_photos=False) + + brand_lower = record.brand.lower() if record.brand else "" + if allowed_brands and brand_lower not in allowed_brands: + skipped_brands += 1 + continue + if excluded_brands and brand_lower in excluded_brands: + skipped_brands += 1 + continue + + records.append(record) + probe_tasks.append((len(records) - 1, item)) + all_origin_ids.add(record.origin_id) + except Exception as exc: + logger.warning("Failed to map vehicle %s: %s", vehicle_id, exc) + failed += 1 + failed_ids.append(vehicle_id) + + # Фаза 2: фото — берём из photo API (все фото) или только из листинга (4 фото) + if records and probe_all_photos: + t0 = time.monotonic() + self._batch_fetch_photo_lists(records, probe_tasks) + probe_time = time.monotonic() - t0 + total_images_fetched = sum(len(r.images) for r in records) + logger.info( + "Photo API fetch: %d cars, %d total images in %.1fs", + len(records), total_images_fetched, probe_time, + ) + elif records: + # Без пробинга: берём только фото из API (как reference-проект) + self._apply_api_photos(records, probe_tasks) + + # Фаза 3: upsert в БД + if records: + try: + result = self.persistence.upsert_cars_batch(records) + synced = result.get("inserted", 0) + result.get("updated", 0) + except Exception as exc: + logger.error("Batch upsert failed: %s", exc) + synced = 0 + failed += len(records) + failed_ids.extend(r.origin_id for r in records) + else: + synced = 0 + + total_images = sum(len(r.images) for r in records) + logger.info("Batch stats: %d items → %d after brand filter (-%d), %d synced, %d total images", + len(items), len(records), skipped_brands, synced, total_images) + + return synced, failed, failed_ids + + def _apply_api_photos( + self, + records: list[CarRecord], + probe_tasks: list[tuple[int, Any]], + ) -> None: + """Применяет фото из API (Photo + Photos) без HEAD-пробинга.""" + for rec_idx, item in probe_tasks: + photos = item.get("Photos") or [] + photo_prefix = self.car_mapper._as_str(item.get("Photo") or "") + + images: list[ImageRecord] = [] + for raw in (photos if isinstance(photos, list) else []): + if not isinstance(raw, dict): + continue + location = raw.get("location") or raw.get("Location") or "" + if not location: + continue + url = self.car_mapper._build_image_url(str(location)) + order = int(raw.get("ordering") or 0) + images.append(ImageRecord(fullres_image=url, preview_image=url, order_index=order)) + + # Если Photos пусто, но Photo есть — создаём хотя бы одну запись + if not images and photo_prefix: + url = self.car_mapper._build_image_url(photo_prefix.rstrip("_") + "_001.jpg") + images.append(ImageRecord(fullres_image=url, preview_image=url, order_index=1)) + + images.sort(key=lambda x: x.order_index) + records[rec_idx].images = images + + def _batch_fetch_photo_lists( + self, + records: list[CarRecord], + probe_tasks: list[tuple[int, Any]], + ) -> None: + """Получает полные списки фото через batch API (/v1/readside/vehicles). + + Batch endpoint найден в JS-бандле мобильной SPA (fem.encar.com). + Принимает до 20 vehicleIds за запрос, возвращает все фото каждой машины. + В ~4.4x быстрее legacy photo API (370 vs 61 машин/с) и без connection errors. + """ + tasks: list[tuple[int, str]] = [] + for rec_idx, item in probe_tasks: + vehicle_id = str(item.get("Id") or "") + if vehicle_id: + tasks.append((rec_idx, vehicle_id)) + + if not tasks: + return + + pool = self._get_batch_pool() + + # Разбиваем на чанки по BATCH_VEHICLES_CHUNK_SIZE (20) ID + chunks: list[list[tuple[int, str]]] = [] + for i in range(0, len(tasks), BATCH_VEHICLES_CHUNK_SIZE): + chunks.append(tasks[i:i + BATCH_VEHICLES_CHUNK_SIZE]) + + def _fetch_chunk(chunk: list[tuple[int, str]]) -> list[tuple[int, list[ImageRecord]]]: + """Запрашивает фото для чанка машин одним batch-запросом.""" + ids_str = ",".join(vid for _, vid in chunk) + path = f"/v1/readside/vehicles?vehicleIds={ids_str}&include=PHOTOS" + for attempt in range(2): + try: + resp = pool.request("GET", path, timeout=15, retries=False) + if resp.status == 200: + if not resp.data: + return [(ri, []) for ri, _ in chunk] + vehicles = json.loads(resp.data.decode("utf-8", errors="ignore")) + # Маппим по vehicleId, а не по индексу — API может пропускать удалённые ID + id_to_rec: dict[str, int] = {vid: ri for ri, vid in chunk} + result: list[tuple[int, list[ImageRecord]]] = [] + for vehicle_data in vehicles: + vid = str(vehicle_data.get("vehicleId") or vehicle_data.get("id") or "") + rec_idx = id_to_rec.get(vid) + if rec_idx is None: + continue + photos_raw = vehicle_data.get("photos") or [] + images: list[ImageRecord] = [] + seen_paths: set[str] = set() + for photo in photos_raw: + p = photo.get("path") or "" + if not p or p in seen_paths: + continue + seen_paths.add(p) + full_url = self.car_mapper._build_image_url(p) + code = photo.get("code") or "0" + order = int(code) if code.isdigit() else 0 + images.append(ImageRecord( + fullres_image=full_url, + preview_image=full_url, + order_index=order, + )) + images.sort(key=lambda x: x.order_index) + result.append((rec_idx, images)) + return result + if resp.status in (429, 503): + time.sleep(1 + attempt * 2) + continue + return [(ri, []) for ri, _ in chunk] + except json.JSONDecodeError: + return [(ri, []) for ri, _ in chunk] + except Exception as exc: + logger.debug("Batch photo chunk error (attempt %d): %s", attempt + 1, exc) + if attempt < 1: + time.sleep(1) + return [(ri, []) for ri, _ in chunk] + + results: dict[int, list[ImageRecord]] = {} + with ThreadPoolExecutor(max_workers=30) as executor: + futures = {executor.submit(_fetch_chunk, ch): ch for ch in chunks} + for future in as_completed(futures): + try: + for rec_idx, images in future.result(): + if images: + results[rec_idx] = images + except Exception as exc: + logger.warning("Batch photo future failed: %s", exc) + + # Применяем: batch API → fallback на Photos из листинга + api_ok = 0 + fallback_count = 0 + for rec_idx, item in probe_tasks: + if rec_idx in results: + records[rec_idx].images = results[rec_idx] + api_ok += 1 + else: + fallback_count += 1 + photos = item.get("Photos") or [] + photo_prefix = self.car_mapper._as_str(item.get("Photo") or "") + images: list[ImageRecord] = [] + for raw in (photos if isinstance(photos, list) else []): + if not isinstance(raw, dict): + continue + location = raw.get("location") or raw.get("Location") or "" + if not location: + continue + url = self.car_mapper._build_image_url(str(location)) + order = int(raw.get("ordering") or 0) + images.append(ImageRecord(fullres_image=url, preview_image=url, order_index=order)) + if not images and photo_prefix: + url = self.car_mapper._build_image_url(photo_prefix.rstrip("_") + "_001.jpg") + images.append(ImageRecord(fullres_image=url, preview_image=url, order_index=1)) + images.sort(key=lambda x: x.order_index) + records[rec_idx].images = images + if fallback_count: + logger.info("Batch photo API: %d OK, %d fallback to listing photos", api_ok, fallback_count) + + def _fetch_listing_page(self, offset: int, limit: int, filters: EncarFilters | None = None) -> dict[str, Any]: + filters = filters or EncarFilters() + query = filters.build_query() + params = { + "count": "true", + "q": query, + # CreatedDate — стабильная сортировка (дата создания объявления не меняется). + # ModifiedDate нестабильна: машины "всплывают" при обновлении цены/фото, + # вызывая до 50% дублей на поздних страницах длинного синка. + "sr": f"|CreatedDate|{offset}|{limit}", + } + url = f"{ENCAR_LISTING_API}?{urlencode(params)}" + return self._fetch_json(url) + + def _fetch_listing_item(self, vehicle_id: str) -> dict[str, Any] | None: + """Ищет авто в листинге по ID (макс 5 страниц).""" + page = 0 + page_size = 100 + max_pages = 5 # Ограничиваем поиск + + while page < max_pages: + response = self._fetch_listing_page(offset=page * page_size, limit=page_size) + results = response.get("SearchResults") or [] + if not results: + break + for item in results: + if str(item.get("Id")) == vehicle_id: + return item + if len(results) < page_size: + break + page += 1 + return None + + def _fetch_vehicle_detail_api(self, vehicle_id: str) -> dict[str, Any] | None: + url = ENCAR_DETAIL_API_TEMPLATE.format(vehicle_id=vehicle_id) + params = { + "include": ",".join(["ADVERTISEMENT", "VEHICLE", "SPECS", "IMAGE", "CONDITION"]) + } + full_url = f"{url}?{urlencode(params)}" + payload = self._fetch_json(full_url, headers={"Referer": ENCAR_DETAIL_URL_TEMPLATE.format(vehicle_id=vehicle_id)}) + return payload if payload and payload.get("vehicleId") else None + + def _fetch_json(self, url: str, headers: dict[str, str] | None = None) -> dict[str, Any]: + request_headers = { + "User-Agent": self.settings.fingerprint.user_agent, + "Accept": "application/json, text/plain, */*", + "Accept-Language": "ko-KR,ko;q=0.9,en;q=0.8", + "Referer": "https://www.encar.com/", + } + if headers: + request_headers.update(headers) + req = Request(url, headers=request_headers) + for attempt in range(3): + try: + with urlopen(req, timeout=30) as resp: + content = resp.read().decode("utf-8", errors="ignore") + return json.loads(content) + except (HTTPError, URLError, ConnectionError, TimeoutError, ValueError, OSError) as exc: + logger.warning("Encar JSON fetch failed (%s) for %s; attempt %d", exc, url, attempt + 1) + if attempt == 2: + raise + time.sleep(2 + attempt * 2) + return {} + + def _extract_vehicle_id(self, vehicle_url: str) -> str | None: + match = VEHICLE_ID_RE.search(vehicle_url) + if match: + return match.group(1) + return None diff --git a/encar_scraper/storage/__init__.py b/encar_scraper/storage/__init__.py new file mode 100644 index 0000000..c9c2ef6 --- /dev/null +++ b/encar_scraper/storage/__init__.py @@ -0,0 +1 @@ +__all__: list[str] = [] diff --git a/encar_scraper/storage/db.py b/encar_scraper/storage/db.py new file mode 100644 index 0000000..46443b9 --- /dev/null +++ b/encar_scraper/storage/db.py @@ -0,0 +1,494 @@ +import logging +from contextlib import contextmanager +from datetime import datetime, timezone +from typing import Any, Iterator + +from sqlalchemy import create_engine, delete, or_, select, text, update +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.orm import Session, sessionmaker + +from ..core.config import Settings +from .models import Base, Car, Image, SyncRun +from .schemas import CarRecord + +logger = logging.getLogger("encar_scraper.db") + + +CAR_DB_FIELDS = { + col.key for col in Car.__table__.columns + if col.key not in ("id",) +} + +_IN_CHUNK_SIZE = 5000 + + +class PersistenceService: + + def __init__(self, settings: Settings) -> None: + self.settings = settings + engine_kwargs = { + "echo": settings.database.echo, + "future": True, + } + if "postgresql" in settings.database.url: + engine_kwargs["pool_size"] = settings.database.pool_size + engine_kwargs["max_overflow"] = settings.database.max_overflow + engine_kwargs["pool_pre_ping"] = True + engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds + self.engine = create_engine(settings.database.url, **engine_kwargs) + self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True) + + def create_tables(self) -> None: + # В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции. + is_sqlite = self.settings.database.url.startswith("sqlite") + if not is_sqlite and not self.settings.database.auto_create_tables: + return + try: + Base.metadata.create_all(self.engine) + except Exception: + logger.debug("create_tables skipped (schema already exists)") + + @contextmanager + def session_scope(self) -> Iterator[Session]: + session = self.session_factory() + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + finally: + session.close() + + def start_sync_run(self, lane: str) -> int: + with self.session_scope() as session: + now = datetime.now(timezone.utc) + stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all() + for stale in stale_runs: + stale.status = "failed" + stale.finished_at = now + if not stale.error_summary: + stale.error_summary = "Recovered stale running sync run before starting a new run" + + run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0) + session.add(run) + session.flush() + return int(run.id) + + def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None: + with self.session_scope() as session: + run = session.get(SyncRun, run_id) + if run is None: + return + run.finished_at = datetime.now(timezone.utc) + run.status = status + run.ids_fetched = ids_fetched + run.cars_upserted = cars_upserted + run.cars_failed = cars_failed + run.images_upserted = images_upserted + run.error_summary = error_summary + + @staticmethod + def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None: + if not images: + return + session.add_all([ + Image( + fullres_image=str(img["fullres_image"]), + preview_image=str(img["preview_image"]), + order_index=int(img.get("order_index", 0)), + car_id=car_id, + ) + for img in images + ]) + + @staticmethod + def _car_payload(record: CarRecord) -> dict[str, object]: + payload = record.model_dump(mode="python") + return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS} + + def _is_postgres(self) -> bool: + return self.engine.dialect.name == "postgresql" + + def _load_existing_cars( + self, + session: Session, + origin_ids: list[str], + origin_urls: list[str], + ) -> tuple[dict[str, Car], dict[str, Car]]: + existing_by_id: dict[str, Car] = {} + existing_by_url: dict[str, Car] = {} + if not origin_ids and not origin_urls: + return existing_by_id, existing_by_url + + existing_cars: list[Car] = [] + max_len = max(len(origin_ids), len(origin_urls), 1) + for i in range(0, max_len, _IN_CHUNK_SIZE): + id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE] + url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE] + conditions = [] + if id_chunk: + conditions.append(Car.origin_id.in_(id_chunk)) + if url_chunk: + conditions.append(Car.origin_url.in_(url_chunk)) + if not conditions: + continue + rows = session.execute( + select(Car).where(or_(*conditions)) + ).scalars().all() + existing_cars.extend(rows) + + for car in existing_cars: + if car.origin_id: + existing_by_id[car.origin_id] = car + if car.origin_url: + existing_by_url[car.origin_url] = car + return existing_by_id, existing_by_url + + def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]: + existing_images_map: dict[int, set[str]] = {} + if not car_ids: + return existing_images_map + + car_id_list = list(car_ids) + for i in range(0, len(car_id_list), _IN_CHUNK_SIZE): + chunk = car_id_list[i:i + _IN_CHUNK_SIZE] + img_rows = session.execute( + select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk)) + ).all() + for cid, furl in img_rows: + existing_images_map.setdefault(int(cid), set()).add(str(furl)) + return existing_images_map + + @staticmethod + def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]: + return { + key: getattr(insert_stmt.excluded, key) + for key in CAR_DB_FIELDS + } + + def _replace_images_for_car( + self, + session: Session, + car_id: int, + images: list[dict[str, object]], + origin_id: str, + ) -> int: + nested = session.begin_nested() + try: + session.execute(delete(Image).where(Image.car_id == car_id)) + self._add_images(session, car_id, images) + session.flush() + nested.commit() + return len(images) + except Exception: + nested.rollback() + logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True) + return 0 + + def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]: + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + entries: list[dict[str, object]] = [] + upsert_payloads: list[dict[str, object]] = [] + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + car_by_id = existing_by_id.get(record.origin_id) + car_by_url = existing_by_url.get(record.origin_url) + entry: dict[str, object] = {"record": record, "images": images, "car_id": None} + + if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + entry["car_id"] = int(car_by_url.id) + updated += 1 + else: + upsert_payloads.append(payload) + if car_by_id is not None: + updated += 1 + else: + inserted += 1 + entries.append(entry) + + session.flush() + + if upsert_payloads: + insert_stmt = pg_insert(Car).values(upsert_payloads) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id, Car.origin_id) + rows = session.execute(upsert_stmt).all() + car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows} + for entry in entries: + if entry["car_id"] is not None: + continue + record = entry["record"] + car_id = car_ids_by_origin_id.get(record.origin_id) + if car_id is None: + raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}") + entry["car_id"] = car_id + + car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None} + existing_images_map = self._load_existing_image_urls(session, car_ids) + images_by_car_id: dict[int, list[dict[str, object]]] = {} + replace_ids: list[int] = [] + + for entry in entries: + car_id = int(entry["car_id"]) + images = entry["images"] + new_image_urls = { + str(img.get("fullres_image", "")) + for img in images + if img.get("fullres_image") + } + old_image_urls = existing_images_map.get(car_id, set()) + if new_image_urls != old_image_urls: + replace_ids.append(car_id) + images_by_car_id[car_id] = images + else: + images_total += len(old_image_urls) + + if replace_ids: + for i in range(0, len(replace_ids), _IN_CHUNK_SIZE): + chunk = replace_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car_id in replace_ids: + images = images_by_car_id[car_id] + self._add_images(session, car_id, images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def upsert_car(self, record: CarRecord): + # Вставка или обновление автомобиля по origin_id/origin_url. + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + with self.session_scope() as session: + if self._is_postgres(): + car_by_url = session.execute( + select(Car).where(Car.origin_url == record.origin_url) + ).scalar_one_or_none() + if car_by_url is not None and car_by_url.origin_id != record.origin_id: + for key, value in payload.items(): + setattr(car_by_url, key, value) + car_by_url.last_seen_at = record.last_seen_at + session.flush() + car_id = int(car_by_url.id) + action = "updated" + else: + existed = session.execute( + select(Car.id).where(Car.origin_id == record.origin_id) + ).scalar_one_or_none() is not None + insert_stmt = pg_insert(Car).values(**payload) + upsert_stmt = insert_stmt.on_conflict_do_update( + index_elements=[Car.origin_id], + set_=self._postgres_upsert_set_map(insert_stmt), + ).returning(Car.id) + car_id = int(session.execute(upsert_stmt).scalar_one()) + action = "updated" if existed or car_by_url is not None else "inserted" + + images_upserted = self._replace_images_for_car( + session, car_id, images, record.origin_id, + ) + return {"car_id": car_id, "images_upserted": images_upserted, "action": action} + + car = session.execute( + select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url)) + ).scalar_one_or_none() + action = "inserted" + if car is None: + car = Car(**payload) + session.add(car) + session.flush() + else: + action = "updated" + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + session.flush() + images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id) + return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action} + self._add_images(session, int(car.id), images) + session.flush() + return {"car_id": int(car.id), "images_upserted": len(images), "action": action} + def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]: + """Пакетный upsert нескольких автомобилей в одной транзакции. + + Оптимизации: + - Дедупликация записей по origin_id перед вставкой. + - Chunked IN-queries для больших списков (обход лимита PG параметров). + - Пропуск перезаписи изображений, если набор URL не изменился. + - Один DELETE по car_id IN (...) вместо удаления по одному. + - Fallback на по-одному upsert если batch commit упал. + """ + # ── Дедупликация записей внутри батча ── + seen_ids: dict[str, int] = {} + unique_records: list[CarRecord] = [] + for idx, r in enumerate(records): + key = r.origin_id or r.origin_url + if key in seen_ids: + logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key]) + continue + seen_ids[key] = idx + unique_records.append(r) + + if len(unique_records) < len(records): + logger.info("Deduped batch: %d → %d records", len(records), len(unique_records)) + records = unique_records + + try: + return self._upsert_cars_batch_inner(records) + except Exception as exc: + logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc) + return self._upsert_cars_individually(records) + + def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]: + """Внутренняя реализация batched upsert (одна транзакция).""" + if self._is_postgres(): + return self._upsert_cars_batch_postgres(records) + + inserted = 0 + updated = 0 + images_total = 0 + + with self.session_scope() as session: + # Получаем существующие записи chunked-запросами. + origin_ids = [r.origin_id for r in records if r.origin_id] + origin_urls = [r.origin_url for r in records if r.origin_url] + + existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) + + # Предзагружаем ВСЕ изображения для обновляемых машин одним запросом. + existing_car_ids = set() + for record in records: + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is not None: + existing_car_ids.add(int(car.id)) + + # Строим маппинг car_id → set(image_urls) для сравнения. + existing_images_map = self._load_existing_image_urls(session, existing_car_ids) + + new_cars: list[tuple[Car, list[dict]]] = [] + update_cars_needing_images: list[tuple[Car, list[dict]]] = [] + + for record in records: + payload = self._car_payload(record) + images = [image.model_dump(mode="python") for image in record.images] + + car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url) + if car is None: + car = Car(**payload) + session.add(car) + inserted += 1 + new_cars.append((car, images)) + else: + for key, value in payload.items(): + setattr(car, key, value) + car.last_seen_at = record.last_seen_at + updated += 1 + + # Проверяем, изменились ли изображения. + new_image_urls = {img.get("fullres_image", "") for img in images} + old_image_urls = existing_images_map.get(int(car.id), set()) + if new_image_urls != old_image_urls: + update_cars_needing_images.append((car, images)) + else: + images_total += len(old_image_urls) + + # Один flush для всех вставок. + session.flush() + + # Добавляем изображения для новых автомобилей. + for car, images in new_cars: + self._add_images(session, int(car.id), images) + images_total += len(images) + + # Массово обновляем изображения только для машин с изменёнными картинками. + if update_cars_needing_images: + update_ids = [int(car.id) for car, _ in update_cars_needing_images] + for i in range(0, len(update_ids), _IN_CHUNK_SIZE): + chunk = update_ids[i:i + _IN_CHUNK_SIZE] + session.execute(delete(Image).where(Image.car_id.in_(chunk))) + for car, images in update_cars_needing_images: + self._add_images(session, int(car.id), images) + images_total += len(images) + + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]: + # Fallback на поштучный upsert. + inserted = 0 + updated = 0 + images_total = 0 + for record in records: + try: + result = self.upsert_car(record) + action = result.get("action", "inserted") + if action == "inserted": + inserted += 1 + else: + updated += 1 + images_total += int(result.get("images_upserted", 0)) + except Exception as exc: + logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) + return {"inserted": inserted, "updated": updated, "images_upserted": images_total} + + def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "encar") -> int: + """Помечает авто как проданные, если их нет в активном листинге (по origin_id). + + Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, + что кардинально быстрее при больших объёмах (200K+ IDs). + """ + if not active_origin_ids: + return 0 + with self.session_scope() as session: + if self._is_postgres(): + session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_ids (origin_id TEXT NOT NULL) ON COMMIT DROP")) + session.execute(text("TRUNCATE _active_ids")) + + id_list = list(active_origin_ids) + for i in range(0, len(id_list), _IN_CHUNK_SIZE): + chunk = id_list[i:i + _IN_CHUNK_SIZE] + values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk))) + params = {f"u{j}": oid for j, oid in enumerate(chunk)} + session.execute(text(f"INSERT INTO _active_ids (origin_id) VALUES {values}"), params) + + session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_ids ON _active_ids (origin_id)")) + + result = session.execute(text(""" + UPDATE cars + SET is_sold = TRUE + FROM ( + SELECT c.id + FROM cars c + LEFT JOIN _active_ids a ON c.origin_id = a.origin_id + WHERE a.origin_id IS NULL + AND c.is_sold = FALSE + AND c.origin_id LIKE 'encar:%%' + ) sub + WHERE cars.id = sub.id + """)) + count = result.rowcount or 0 + else: + stmt = ( + update(Car) + .where(Car.origin_id.notin_(active_origin_ids)) + .where(Car.is_sold == False) # noqa: E712 + .where(Car.origin_id.like("encar:%")) + .values(is_sold=True) + ) + result = session.execute(stmt) + count = result.rowcount or 0 + if count: + logger.info("Marked %d cars as sold (no longer in listing)", count) + return count \ No newline at end of file diff --git a/encar_scraper/storage/enums.py b/encar_scraper/storage/enums.py new file mode 100644 index 0000000..2b8ee39 --- /dev/null +++ b/encar_scraper/storage/enums.py @@ -0,0 +1,36 @@ +CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD") +DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA") +GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA") +STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA") +BODY_TYPE_ENUM_VALUES = ( + "COUPE", + "SUV", + "HATCHBACK", + "MINIVAN", + "SEDAN", + "STATION_WAGON", + "PICKUP", + "TRUCK", + "OPEN", + "RV", + "VAN", + "CONVERTIBLE", + "BUS", + "COMPACT", + "MID_SIZE", + "FULL_SIZE", + "SPORTS", + "CLASSIC", + "ELECTRIC", + "HYBRID", + "SPECIAL", + "WAGON", + "OTHER", + "NA", +) +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") +ORIGIN_ENUM_VALUES = ( + "ENCAR", + "NA", +) +SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA") diff --git a/encar_scraper/storage/models.py b/encar_scraper/storage/models.py new file mode 100644 index 0000000..1242e4b --- /dev/null +++ b/encar_scraper/storage/models.py @@ -0,0 +1,82 @@ +from datetime import datetime + +from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + +from .enums import ( + BODY_TYPE_ENUM_VALUES, + COUNTRY_ENUM_VALUES, + CURRENCY_ENUM_VALUES, + DRIVE_ENUM_VALUES, + GEARBOX_ENUM_VALUES, + ORIGIN_ENUM_VALUES, + SELLING_TYPE_ENUM_VALUES, + STEERING_WHEEL_ENUM_VALUES, +) + + +class Base(DeclarativeBase): + pass + + +class Car(Base): + __tablename__ = "cars" + __table_args__ = ( + Index("ix_cars_brand_model", "brand", "model"), + Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"), + ) + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + parser_id: Mapped[str] = mapped_column(String(255), nullable=False, unique=True) + brand: Mapped[str] = mapped_column(String(100), nullable=False, index=True) + model: Mapped[str] = mapped_column(String(500), nullable=False) + year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True) + price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) + currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD") + mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) + color: Mapped[str] = mapped_column(String(), nullable=False, default="other") + drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True) + gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True) + steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True) + body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER") + engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True) + selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True) + origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True) + is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + evaluation: Mapped[str | None] = mapped_column(String(), nullable=True) + non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True) + rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False) + slug: Mapped[str] = mapped_column(String(), nullable=False) + last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True) + images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan") + + +class Image(Base): + __tablename__ = "images" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + fullres_image: Mapped[str] = mapped_column(String(), nullable=False) + preview_image: Mapped[str] = mapped_column(String(), nullable=False) + order_index: Mapped[int] = mapped_column(Integer, nullable=False) + car_id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True) + car: Mapped[Car] = relationship("Car", back_populates="images") + + +class SyncRun(Base): + __tablename__ = "sync_runs" + id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) + status: Mapped[str] = mapped_column(Text, nullable=False, index=True) + lane: Mapped[str] = mapped_column(Text, nullable=False) + ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0) + error_summary: Mapped[str | None] = mapped_column(Text, nullable=True) diff --git a/encar_scraper/storage/schemas.py b/encar_scraper/storage/schemas.py new file mode 100644 index 0000000..7af7c43 --- /dev/null +++ b/encar_scraper/storage/schemas.py @@ -0,0 +1,87 @@ +from datetime import datetime, timezone +from pydantic import BaseModel, ConfigDict, Field + + +class ImageRecord(BaseModel): + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRecord(BaseModel): + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str + origin_id: str + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str + last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + images: list[ImageRecord] = Field(default_factory=list) + + +class ImageRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + fullres_image: str + preview_image: str + order_index: int = 0 + + +class CarRead(BaseModel): + model_config = ConfigDict(from_attributes=True) + + id: int + parser_id: str + brand: str + model: str + year: int | None = None + price: int | None = None + currency: str = "USD" + mileage: int = 0 + country: str = "US" + is_sold: bool = False + color: str = "other" + drive: str | None = None + gearbox: str | None = None + steering_wheel: str | None = None + body_type: str = "OTHER" + engine_volume: int | None = None + selling_type: str = "AUCTION" + one_owner: bool = False + new_car: bool = False + is_hidden: bool = False + origin: str = "NA" + origin_url: str = "" + origin_id: str = "" + is_damaged: bool = False + evaluation: str | None = None + non_smoking: bool = True + rental: bool = False + repair_history: bool = False + slug: str = "" + last_seen_at: datetime | None = None + images: list[ImageRead] = Field(default_factory=list) + diff --git a/encar_scraper/worker/__init__.py b/encar_scraper/worker/__init__.py new file mode 100644 index 0000000..841be13 --- /dev/null +++ b/encar_scraper/worker/__init__.py @@ -0,0 +1,3 @@ +from .celery_app import celery_app + +__all__ = ["celery_app"] diff --git a/encar_scraper/worker/celery_app.py b/encar_scraper/worker/celery_app.py new file mode 100644 index 0000000..9a373a3 --- /dev/null +++ b/encar_scraper/worker/celery_app.py @@ -0,0 +1,71 @@ +# Инициализация Celery-приложения и периодических задач для Encar. + +from celery import Celery + +from ..core.config import settings + + +def _broker_url() -> str: + return settings.celery.broker_url or settings.redis.url + + +def _result_backend() -> str: + return settings.celery.result_backend or settings.redis.url + + +celery_app = Celery( + "encar_scraper", + broker=_broker_url(), + backend=_result_backend(), +) + +celery_app.conf.update( + task_serializer="json", + accept_content=["json"], + result_serializer="json", + timezone="UTC", + enable_utc=True, + task_soft_time_limit=settings.celery.task_soft_time_limit, + task_time_limit=settings.celery.task_time_limit, + task_acks_late=True, + task_reject_on_worker_lost=True, + task_track_started=True, + worker_concurrency=settings.celery.worker_concurrency, + worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child, + worker_pool="prefork", + worker_prefetch_multiplier=1, + broker_connection_retry_on_startup=True, + broker_transport_options={ + "visibility_timeout": settings.celery.broker_visibility_timeout, + }, + result_expires=86400, + worker_hijack_root_logger=False, + beat_schedule={ + "periodic-encar-sync": { + "task": "encar_scraper.worker.tasks.encar_sync_listing_task", + "schedule": settings.celery.encar_beat_interval_minutes * 60.0, + "args": (), + "kwargs": { + "car_type": "all", + }, + "options": {"queue": "encar"}, + }, + }, + task_routes={ + "encar_scraper.worker.tasks.*": {"queue": "encar"}, + }, +) + +celery_app.autodiscover_tasks(["encar_scraper.worker"]) + + +from celery.signals import setup_logging as celery_setup_logging + + +@celery_setup_logging.connect +def _configure_logging(loglevel, logfile, format, colorize, **kwargs): + """Перехватываем настройку логирования Celery, чтобы использовать свой формат.""" + from ..core.logs import setup_logging + import logging + level_name = logging.getLevelName(loglevel) if isinstance(loglevel, int) else str(loglevel) + setup_logging(level=level_name, log_file=logfile) \ No newline at end of file diff --git a/encar_scraper/worker/tasks.py b/encar_scraper/worker/tasks.py new file mode 100644 index 0000000..fdb1e11 --- /dev/null +++ b/encar_scraper/worker/tasks.py @@ -0,0 +1,244 @@ +# Celery tasks для синхронизации автомобилей Encar. + +import logging +from threading import Event, Thread +import uuid + +from celery import shared_task +from redis import Redis + +from ..core.config import Settings +from ..core.runtime_config import load_runtime_config +from ..encar import EncarScraper, EncarFilters, expand_allowed_brands +from ..storage.db import PersistenceService + +logger = logging.getLogger("encar_scraper.worker.tasks") + +ENCAR_SYNC_LOCK_KEY = "encar:locks:sync_listing" + +_persistence: PersistenceService | None = None +_redis: Redis | None = None + + +def _sync_listing_lock_ttl_seconds() -> int: + settings = Settings() + return max(settings.celery.task_time_limit + 120, 300) + + +def _get_persistence() -> PersistenceService: + global _persistence + if _persistence is None: + _persistence = PersistenceService(Settings()) + return _persistence + + +def _get_redis() -> Redis: + global _redis + if _redis is None: + settings = Settings() + _redis = Redis.from_url( + settings.redis.url, + decode_responses=True, + socket_timeout=10, + socket_connect_timeout=5, + ) + return _redis + + +def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool: + try: + return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds)) + except Exception as exc: + logger.warning("Failed to acquire lock %s", key, exc_info=True) + return False + + +_REFRESH_LOCK_SCRIPT = """ +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) +end +return 0 +""" + +_RELEASE_LOCK_SCRIPT = """ +if redis.call('GET', KEYS[1]) == ARGV[1] then + return redis.call('DEL', KEYS[1]) +end +return 0 +""" + + +def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None: + try: + script = redis_client.register_script(_REFRESH_LOCK_SCRIPT) + refreshed = script(keys=[key], args=[owner_token, int(ttl_seconds)]) + return bool(refreshed) + except Exception as exc: + logger.warning("Failed to refresh lock %s", key, exc_info=True) + return None + + +def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None: + try: + script = redis_client.register_script(_RELEASE_LOCK_SCRIPT) + script(keys=[key], args=[owner_token]) + except Exception as exc: + logger.warning("Failed to release lock %s", key, exc_info=True) + + +def _start_lock_heartbeat( + redis_client: Redis, + key: str, + owner_token: str, + ttl_seconds: int, +) -> tuple[Event, Thread]: + stop_event = Event() + interval_seconds = max(5.0, min(30.0, ttl_seconds / 3)) + + def _heartbeat() -> None: + while not stop_event.wait(interval_seconds): + refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds) + if refreshed is False: + logger.warning("Lost sync_listing lock ownership for %s", owner_token) + return + + thread = Thread(target=_heartbeat, name="encar-lock-heartbeat", daemon=True) + thread.start() + return stop_event, thread + + +@shared_task( + name="encar_scraper.worker.tasks.encar_sync_listing_task", + bind=True, + max_retries=2, + default_retry_delay=60, + acks_late=True, +) +def encar_sync_listing_task( + self, + car_type: str = "all", + manufacturer: str | None = None, + year_from: int | None = None, + year_to: int | None = None, + price_max: int | None = None, + lane: str = "encar", + limit: int | None = None, +): + """ + Полная синхронизация листинга Encar. + Собирает ВСЕ авто, обновляет цены/данные, помечает проданные. + """ + persistence = _get_persistence() + persistence.create_tables() + task_id = self.request.id or "unknown" + owner_token = f"{task_id}:{uuid.uuid4().hex}" + redis_client = _get_redis() + + lock_ttl = _sync_listing_lock_ttl_seconds() + lock_acquired = _acquire_lock(redis_client, ENCAR_SYNC_LOCK_KEY, owner_token, lock_ttl) + + if not lock_acquired: + logger.info("encar_sync_listing_task skipped: another sync is already running") + return { + "status": "skipped", + "reason": "sync_already_running", + "task_id": task_id, + } + + heartbeat_stop: Event | None = None + heartbeat_thread: Thread | None = None + + try: + heartbeat_stop, heartbeat_thread = _start_lock_heartbeat( + redis_client, + ENCAR_SYNC_LOCK_KEY, + owner_token, + lock_ttl, + ) + self.update_state(state="STARTED", meta={"stage": "encar_sync_started", "task_id": task_id}) + + # --- Загрузка runtime_config --- + rc = load_runtime_config() + effective_limit = limit or rc.sync.limit + effective_lane = lane or rc.sync.lane or "encar" + + car_type_map = {"all": "A", "domestic": "Y", "import": "N"} + filters = EncarFilters( + car_type=car_type_map.get(car_type, "A"), + manufacturer=manufacturer, + year_from=year_from, + year_to=year_to, + price_to=price_max, + ) + + # Набор разрешённых брендов (lowercased) из runtime_config + # Расширяем английские имена alias'ами (корейский + русский перевод) + allowed_brands = expand_allowed_brands( + set(rc.filters.brands) if rc.filters.brands else None + ) + excluded_brands = expand_allowed_brands( + set(rc.filters.exclude_brands) if rc.filters.exclude_brands else None + ) + + scraper = EncarScraper() + result = scraper.sync_listing( + limit=effective_limit, + filters=filters, + lane=effective_lane, + redis_client=redis_client, + allowed_brands=allowed_brands, + excluded_brands=excluded_brands, + probe_all_photos=rc.sync.probe_all_photos, + ) + + summary = { + "task_id": task_id, + "source": "encar", + "total_available": result.get("total_available", 0), + "items_collected": result.get("items_collected", 0), + "cars_synced": result.get("cars_synced", 0), + "cars_failed": result.get("cars_failed", 0), + "cars_marked_sold": result.get("cars_marked_sold", 0), + } + logger.info( + "encar_sync_listing_task completed: %d synced, %d failed, %d marked sold", + summary["cars_synced"], summary["cars_failed"], summary["cars_marked_sold"], + ) + return {"status": "success", **summary} + + except Exception as exc: + logger.error("encar_sync_listing_task failed: %s", exc, exc_info=True) + raise self.retry(exc=exc) + finally: + if heartbeat_stop is not None: + heartbeat_stop.set() + if heartbeat_thread is not None: + heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10))) + if lock_acquired: + _release_lock_if_owner(redis_client, ENCAR_SYNC_LOCK_KEY, owner_token) + + +@shared_task( + name="encar_scraper.worker.tasks.encar_sync_vehicle_task", + bind=True, + max_retries=2, + default_retry_delay=30, + acks_late=True, +) +def encar_sync_vehicle_task(self, vehicle_url: str, lane: str = "encar"): + """Синхронизация одного авто Encar.""" + persistence = _get_persistence() + persistence.create_tables() + + try: + scraper = EncarScraper() + result = scraper.sync_vehicle(vehicle_url, lane=lane) + logger.info("encar_sync_vehicle_task completed: %s", vehicle_url) + return { + "status": "success", + "vehicle_url": vehicle_url, + "origin_id": result.get("origin_id"), + } + except Exception as exc: + logger.error("encar_sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True) + raise self.retry(exc=exc) diff --git a/pyproject.toml b/pyproject.toml index 7b5357a..2dda42f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,16 +3,15 @@ requires = ["setuptools>=68", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "iaai-scraper" +name = "encar-scraper" version = "0.1.0" -description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL" +description = "Encar scraper service with FastAPI, Celery and PostgreSQL" readme = "README.md" requires-python = ">=3.11" dependencies = [ "alembic>=1.14.0", "celery>=5.4.0", "fastapi>=0.115.0", - "playwright>=1.53.0", "psycopg2-binary>=2.9.9", "pydantic>=2.8.2", "python-dotenv>=1.0.1", @@ -29,13 +28,13 @@ dev = [ ] [project.scripts] -iaai = "iaai_scraper.cli:main" +encar = "encar_scraper.cli:main" [tool.setuptools] include-package-data = true [tool.setuptools.packages.find] -include = ["iaai_scraper*"] +include = ["encar_scraper*"] [tool.pytest.ini_options] addopts = "-q --disable-warnings" diff --git a/runtime_config.json b/runtime_config.json index a64bd1a..94f6375 100644 --- a/runtime_config.json +++ b/runtime_config.json @@ -5,9 +5,10 @@ "ids_next_size": null, "ids_max_pages": null, "condition_check_enabled": false, - "lane": "iaai_cars", - "only_new": true, - "limit": null + "lane": "encar_cars", + "only_new": false, + "limit": null, + "probe_all_photos": true }, "filters": { "price": { diff --git a/uv.lock b/uv.lock index e2820a9..8276e2e 100644 --- a/uv.lock +++ b/uv.lock @@ -259,6 +259,46 @@ toml = [ { name = "tomli", marker = "python_full_version <= '3.11'" }, ] +[[package]] +name = "encar-scraper" +version = "0.1.0" +source = { editable = "." } +dependencies = [ + { name = "alembic" }, + { name = "celery" }, + { name = "fastapi" }, + { name = "psycopg2-binary" }, + { name = "pydantic" }, + { name = "python-dotenv" }, + { name = "redis" }, + { name = "sqlalchemy" }, + { name = "urllib3" }, + { name = "uvicorn" }, +] + +[package.optional-dependencies] +dev = [ + { name = "pytest" }, + { name = "pytest-cov" }, +] + +[package.metadata] +requires-dist = [ + { name = "alembic", specifier = ">=1.14.0" }, + { name = "celery", specifier = ">=5.4.0" }, + { name = "fastapi", specifier = ">=0.115.0" }, + { name = "psycopg2-binary", specifier = ">=2.9.9" }, + { name = "pydantic", specifier = ">=2.8.2" }, + { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, + { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, + { name = "python-dotenv", specifier = ">=1.0.1" }, + { name = "redis", specifier = ">=5.2.0" }, + { name = "sqlalchemy", specifier = ">=2.0.32" }, + { name = "urllib3", specifier = ">=2.0.0" }, + { name = "uvicorn", specifier = ">=0.34.0" }, +] +provides-extras = ["dev"] + [[package]] name = "fastapi" version = "0.135.3" @@ -284,9 +324,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fb/c6/dba32cab7e3a625b011aa5647486e2d28423a48845a2998c126dd69c85e1/greenlet-3.4.0-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:805bebb4945094acbab757d34d6e1098be6de8966009ab9ca54f06ff492def58", size = 285504, upload-time = "2026-04-08T15:52:14.071Z" }, { url = "https://files.pythonhosted.org/packages/54/f4/7cb5c2b1feb9a1f50e038be79980dfa969aa91979e5e3a18fdbcfad2c517/greenlet-3.4.0-cp311-cp311-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:439fc2f12b9b512d9dfa681c5afe5f6b3232c708d13e6f02c845e0d9f4c2d8c6", size = 605476, upload-time = "2026-04-08T16:24:37.064Z" }, { url = "https://files.pythonhosted.org/packages/d6/af/b66ab0b2f9a4c5a867c136bf66d9599f34f21a1bcca26a2884a29c450bd9/greenlet-3.4.0-cp311-cp311-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:a70ed1cb0295bee1df57b63bf7f46b4e56a5c93709eea769c1fec1bb23a95875", size = 618336, upload-time = "2026-04-08T16:30:56.59Z" }, - { url = "https://files.pythonhosted.org/packages/6d/31/56c43d2b5de476f77d36ceeec436328533bff960a4cba9a07616e93063ab/greenlet-3.4.0-cp311-cp311-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:8c5696c42e6bb5cfb7c6ff4453789081c66b9b91f061e5e9367fa15792644e76", size = 625045, upload-time = "2026-04-08T16:40:37.111Z" }, { url = "https://files.pythonhosted.org/packages/e5/5c/8c5633ece6ba611d64bf2770219a98dd439921d6424e4e8cf16b0ac74ea5/greenlet-3.4.0-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:c660bce1940a1acae5f51f0a064f1bc785d07ea16efcb4bc708090afc4d69e83", size = 613515, upload-time = "2026-04-08T15:56:32.478Z" }, - { url = "https://files.pythonhosted.org/packages/80/ca/704d4e2c90acb8bdf7ae593f5cbc95f58e82de95cc540fb75631c1054533/greenlet-3.4.0-cp311-cp311-manylinux_2_39_riscv64.whl", hash = "sha256:89995ce5ddcd2896d89615116dd39b9703bfa0c07b583b85b89bf1b5d6eddf81", size = 419745, upload-time = "2026-04-08T16:43:04.022Z" }, { url = "https://files.pythonhosted.org/packages/a9/df/950d15bca0d90a0e7395eb777903060504cdb509b7b705631e8fb69ff415/greenlet-3.4.0-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee407d4d1ca9dc632265aee1c8732c4a2d60adff848057cdebfe5fe94eb2c8a2", size = 1574623, upload-time = "2026-04-08T16:26:18.596Z" }, { url = "https://files.pythonhosted.org/packages/1a/e7/0839afab829fcb7333c9ff6d80c040949510055d2d4d63251f0d1c7c804e/greenlet-3.4.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:956215d5e355fffa7c021d168728321fd4d31fd730ac609b1653b450f6a4bc71", size = 1639579, upload-time = "2026-04-08T15:57:29.231Z" }, { url = "https://files.pythonhosted.org/packages/d9/2b/b4482401e9bcaf9f5c97f67ead38db89c19520ff6d0d6699979c6efcc200/greenlet-3.4.0-cp311-cp311-win_amd64.whl", hash = "sha256:5cb614ace7c27571270354e9c9f696554d073f8aa9319079dcba466bbdead711", size = 238233, upload-time = "2026-04-08T17:02:54.286Z" }, @@ -294,9 +332,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/65/8b/3669ad3b3f247a791b2b4aceb3aa5a31f5f6817bf547e4e1ff712338145a/greenlet-3.4.0-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:1a54a921561dd9518d31d2d3db4d7f80e589083063ab4d3e2e950756ef809e1a", size = 286902, upload-time = "2026-04-08T15:52:12.138Z" }, { url = "https://files.pythonhosted.org/packages/38/3e/3c0e19b82900873e2d8469b590a6c4b3dfd2b316d0591f1c26b38a4879a5/greenlet-3.4.0-cp312-cp312-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:16dec271460a9a2b154e3b1c2fa1050ce6280878430320e85e08c166772e3f97", size = 606099, upload-time = "2026-04-08T16:24:38.408Z" }, { url = "https://files.pythonhosted.org/packages/b5/33/99fef65e7754fc76a4ed14794074c38c9ed3394a5bd129d7f61b705f3168/greenlet-3.4.0-cp312-cp312-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:90036ce224ed6fe75508c1907a77e4540176dcf0744473627785dd519c6f9996", size = 618837, upload-time = "2026-04-08T16:30:58.298Z" }, - { url = "https://files.pythonhosted.org/packages/44/57/eae2cac10421feae6c0987e3dc106c6d86262b1cb379e171b017aba893a6/greenlet-3.4.0-cp312-cp312-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:6f0def07ec9a71d72315cf26c061aceee53b306c36ed38c35caba952ea1b319d", size = 624901, upload-time = "2026-04-08T16:40:38.981Z" }, { url = "https://files.pythonhosted.org/packages/36/f7/229f3aed6948faa20e0616a0b8568da22e365ede6a54d7d369058b128afd/greenlet-3.4.0-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:a1c4f6b453006efb8310affb2d132832e9bbb4fc01ce6df6b70d810d38f1f6dc", size = 615062, upload-time = "2026-04-08T15:56:33.766Z" }, - { url = "https://files.pythonhosted.org/packages/6a/8a/0e73c9b94f31d1cc257fe79a0eff621674141cdae7d6d00f40de378a1e42/greenlet-3.4.0-cp312-cp312-manylinux_2_39_riscv64.whl", hash = "sha256:0e1254cf0cbaa17b04320c3a78575f29f3c161ef38f59c977108f19ffddaf077", size = 423927, upload-time = "2026-04-08T16:43:05.293Z" }, { url = "https://files.pythonhosted.org/packages/08/97/d988180011aa40135c46cd0d0cf01dd97f7162bae14139b4a3ef54889ba5/greenlet-3.4.0-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:9b2d9a138ffa0e306d0e2b72976d2fb10b97e690d40ab36a472acaab0838e2de", size = 1573511, upload-time = "2026-04-08T16:26:20.058Z" }, { url = "https://files.pythonhosted.org/packages/d4/0f/a5a26fe152fb3d12e6a474181f6e9848283504d0afd095f353d85726374b/greenlet-3.4.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:8424683caf46eb0eb6f626cb95e008e8cc30d0cb675bdfa48200925c79b38a08", size = 1640396, upload-time = "2026-04-08T15:57:30.88Z" }, { url = "https://files.pythonhosted.org/packages/42/cf/bb2c32d9a100e36ee9f6e38fad6b1e082b8184010cb06259b49e1266ca01/greenlet-3.4.0-cp312-cp312-win_amd64.whl", hash = "sha256:a0a53fb071531d003b075c444014ff8f8b1a9898d36bb88abd9ac7b3524648a2", size = 238892, upload-time = "2026-04-08T17:03:10.094Z" }, @@ -304,9 +340,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7a/75/7e9cd1126a1e1f0cd67b0eda02e5221b28488d352684704a78ed505bd719/greenlet-3.4.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:43748988b097f9c6f09364f260741aa73c80747f63389824435c7a50bfdfd5c1", size = 285856, upload-time = "2026-04-08T15:52:45.82Z" }, { url = "https://files.pythonhosted.org/packages/9d/c4/3e2df392e5cb199527c4d9dbcaa75c14edcc394b45040f0189f649631e3c/greenlet-3.4.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5566e4e2cd7a880e8c27618e3eab20f3494452d12fd5129edef7b2f7aa9a36d1", size = 610208, upload-time = "2026-04-08T16:24:39.674Z" }, { url = "https://files.pythonhosted.org/packages/da/af/750cdfda1d1bd30a6c28080245be8d0346e669a98fdbae7f4102aa95fff3/greenlet-3.4.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:1054c5a3c78e2ab599d452f23f7adafef55062a783a8e241d24f3b633ba6ff82", size = 621269, upload-time = "2026-04-08T16:30:59.767Z" }, - { url = "https://files.pythonhosted.org/packages/e0/93/c8c508d68ba93232784bbc1b5474d92371f2897dfc6bc281b419f2e0d492/greenlet-3.4.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:98eedd1803353daf1cd9ef23eef23eda5a4d22f99b1f998d273a8b78b70dd47f", size = 628455, upload-time = "2026-04-08T16:40:40.698Z" }, { url = "https://files.pythonhosted.org/packages/54/78/0cbc693622cd54ebe25207efbb3a0eb07c2639cb8594f6e3aaaa0bb077a8/greenlet-3.4.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:f82cb6cddc27dd81c96b1506f4aa7def15070c3b2a67d4e46fd19016aacce6cf", size = 617549, upload-time = "2026-04-08T15:56:34.893Z" }, - { url = "https://files.pythonhosted.org/packages/7f/46/cfaaa0ade435a60550fd83d07dfd5c41f873a01da17ede5c4cade0b9bab8/greenlet-3.4.0-cp313-cp313-manylinux_2_39_riscv64.whl", hash = "sha256:b7857e2202aae67bc5725e0c1f6403c20a8ff46094ece015e7d474f5f7020b55", size = 426238, upload-time = "2026-04-08T16:43:06.865Z" }, { url = "https://files.pythonhosted.org/packages/ba/c0/8966767de01343c1ff47e8b855dc78e7d1a8ed2b7b9c83576a57e289f81d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:227a46251ecba4ff46ae742bc5ce95c91d5aceb4b02f885487aff269c127a729", size = 1575310, upload-time = "2026-04-08T16:26:21.671Z" }, { url = "https://files.pythonhosted.org/packages/b8/38/bcdc71ba05e9a5fda87f63ffc2abcd1f15693b659346df994a48c968003d/greenlet-3.4.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5b99e87be7eba788dd5b75ba1cde5639edffdec5f91fe0d734a249535ec3408c", size = 1640435, upload-time = "2026-04-08T15:57:32.572Z" }, { url = "https://files.pythonhosted.org/packages/a1/c2/19b664b7173b9e4ef5f77e8cef9f14c20ec7fce7920dc1ccd7afd955d093/greenlet-3.4.0-cp313-cp313-win_amd64.whl", hash = "sha256:849f8bc17acd6295fcb5de8e46d55cc0e52381c56eaf50a2afd258e97bc65940", size = 238760, upload-time = "2026-04-08T17:04:03.878Z" }, @@ -314,9 +348,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/78/02/bde66806e8f169cf90b14d02c500c44cdbe02c8e224c9c67bafd1b8cadd1/greenlet-3.4.0-cp314-cp314-macosx_11_0_universal2.whl", hash = "sha256:10a07aca6babdd18c16a3f4f8880acfffc2b88dfe431ad6aa5f5740759d7d75e", size = 286291, upload-time = "2026-04-08T17:09:34.307Z" }, { url = "https://files.pythonhosted.org/packages/05/1f/39da1c336a87d47c58352fb8a78541ce63d63ae57c5b9dae1fe02801bbc2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:076e21040b3a917d3ce4ad68fb5c3c6b32f1405616c4a57aa83120979649bd3d", size = 656749, upload-time = "2026-04-08T16:24:41.721Z" }, { url = "https://files.pythonhosted.org/packages/d3/6c/90ee29a4ee27af7aa2e2ec408799eeb69ee3fcc5abcecac6ddd07a5cd0f2/greenlet-3.4.0-cp314-cp314-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:e82689eea4a237e530bb5cb41b180ef81fa2160e1f89422a67be7d90da67f615", size = 669084, upload-time = "2026-04-08T16:31:01.372Z" }, - { url = "https://files.pythonhosted.org/packages/d2/4a/74078d3936712cff6d3c91a930016f476ce4198d84e224fe6d81d3e02880/greenlet-3.4.0-cp314-cp314-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:06c2d3b89e0c62ba50bd7adf491b14f39da9e7e701647cb7b9ff4c99bee04b19", size = 673405, upload-time = "2026-04-08T16:40:42.527Z" }, { url = "https://files.pythonhosted.org/packages/07/49/d4cad6e5381a50947bb973d2f6cf6592621451b09368b8c20d9b8af49c5b/greenlet-3.4.0-cp314-cp314-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4df3b0b2289ec686d3c821a5fee44259c05cfe824dd5e6e12c8e5f5df23085cf", size = 665621, upload-time = "2026-04-08T15:56:35.995Z" }, - { url = "https://files.pythonhosted.org/packages/79/3e/df8a83ab894751bc31e1106fdfaa80ca9753222f106b04de93faaa55feb7/greenlet-3.4.0-cp314-cp314-manylinux_2_39_riscv64.whl", hash = "sha256:070b8bac2ff3b4d9e0ff36a0d19e42103331d9737e8504747cd1e659f76297bd", size = 471670, upload-time = "2026-04-08T16:43:08.512Z" }, { url = "https://files.pythonhosted.org/packages/37/31/d1edd54f424761b5d47718822f506b435b6aab2f3f93b465441143ea5119/greenlet-3.4.0-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:8bff29d586ea415688f4cec96a591fcc3bf762d046a796cdadc1fdb6e7f2d5bf", size = 1622259, upload-time = "2026-04-08T16:26:23.201Z" }, { url = "https://files.pythonhosted.org/packages/b0/c6/6d3f9cdcb21c4e12a79cb332579f1c6aa1af78eb68059c5a957c7812d95e/greenlet-3.4.0-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:8a569c2fb840c53c13a2b8967c63621fafbd1a0e015b9c82f408c33d626a2fda", size = 1686916, upload-time = "2026-04-08T15:57:34.282Z" }, { url = "https://files.pythonhosted.org/packages/63/45/c1ca4a1ad975de4727e52d3ffe641ae23e1d7a8ffaa8ff7a0477e1827b92/greenlet-3.4.0-cp314-cp314-win_amd64.whl", hash = "sha256:207ba5b97ea8b0b60eb43ffcacf26969dd83726095161d676aac03ff913ee50d", size = 239821, upload-time = "2026-04-08T17:03:48.423Z" }, @@ -324,9 +356,7 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/8f/18d72b629783f5e8d045a76f5325c1e938e659a9e4da79c7dcd10169a48d/greenlet-3.4.0-cp314-cp314t-macosx_11_0_universal2.whl", hash = "sha256:d70012e51df2dbbccfaf63a40aaf9b40c8bed37c3e3a38751c926301ce538ece", size = 294681, upload-time = "2026-04-08T15:52:35.778Z" }, { url = "https://files.pythonhosted.org/packages/9e/ad/5fa86ec46769c4153820d58a04062285b3b9e10ba3d461ee257b68dcbf53/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a58bec0751f43068cd40cff31bb3ca02ad6000b3a51ca81367af4eb5abc480c8", size = 658899, upload-time = "2026-04-08T16:24:43.32Z" }, { url = "https://files.pythonhosted.org/packages/43/f0/4e8174ca0e87ae748c409f055a1ba161038c43cc0a5a6f1433a26ac2e5bf/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:05fa0803561028f4b2e3b490ee41216a842eaee11aed004cc343a996d9523aa2", size = 665284, upload-time = "2026-04-08T16:31:02.833Z" }, - { url = "https://files.pythonhosted.org/packages/ef/92/466b0d9afd44b8af623139a3599d651c7564fa4152f25f117e1ee5949ffb/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:c4cd56a9eb7a6444edbc19062f7b6fbc8f287c663b946e3171d899693b1c19fa", size = 665872, upload-time = "2026-04-08T16:40:43.912Z" }, { url = "https://files.pythonhosted.org/packages/19/da/991cf7cd33662e2df92a1274b7eb4d61769294d38a1bba8a45f31364845e/greenlet-3.4.0-cp314-cp314t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e60d38719cb80b3ab5e85f9f1aed4960acfde09868af6762ccb27b260d68f4ed", size = 661861, upload-time = "2026-04-08T15:56:37.269Z" }, - { url = "https://files.pythonhosted.org/packages/0d/14/3395a7ef3e260de0325152ddfe19dffb3e49fe10873b94654352b53ad48e/greenlet-3.4.0-cp314-cp314t-manylinux_2_39_riscv64.whl", hash = "sha256:1f85f204c4d54134ae850d401fa435c89cd667d5ce9dc567571776b45941af72", size = 489237, upload-time = "2026-04-08T16:43:09.993Z" }, { url = "https://files.pythonhosted.org/packages/36/c5/6c2c708e14db3d9caea4b459d8464f58c32047451142fe2cfd90e7458f41/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:7f50c804733b43eded05ae694691c9aa68bca7d0a867d67d4a3f514742a2d53f", size = 1622182, upload-time = "2026-04-08T16:26:24.777Z" }, { url = "https://files.pythonhosted.org/packages/7a/4c/50c5fed19378e11a29fabab1f6be39ea95358f4a0a07e115a51ca93385d8/greenlet-3.4.0-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:2d4f0635dc4aa638cda4b2f5a07ae9a2cff9280327b581a3fcb6f317b4fbc38a", size = 1685050, upload-time = "2026-04-08T15:57:36.453Z" }, { url = "https://files.pythonhosted.org/packages/db/72/85ae954d734703ab48e622c59d4ce35d77ce840c265814af9c078cacc7aa/greenlet-3.4.0-cp314-cp314t-win_amd64.whl", hash = "sha256:1a4a48f24681300c640f143ba7c404270e1ebbbcf34331d7104a4ff40f8ea705", size = 245554, upload-time = "2026-04-08T17:03:50.044Z" }, @@ -341,46 +371,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] -[[package]] -name = "iaai-scraper" -version = "0.1.0" -source = { editable = "." } -dependencies = [ - { name = "alembic" }, - { name = "celery" }, - { name = "fastapi" }, - { name = "playwright" }, - { name = "psycopg2-binary" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "redis" }, - { name = "sqlalchemy" }, - { name = "uvicorn" }, -] - -[package.optional-dependencies] -dev = [ - { name = "pytest" }, - { name = "pytest-cov" }, -] - -[package.metadata] -requires-dist = [ - { name = "alembic", specifier = ">=1.14.0" }, - { name = "celery", specifier = ">=5.4.0" }, - { name = "fastapi", specifier = ">=0.115.0" }, - { name = "playwright", specifier = ">=1.53.0" }, - { name = "psycopg2-binary", specifier = ">=2.9.9" }, - { name = "pydantic", specifier = ">=2.8.2" }, - { name = "pytest", marker = "extra == 'dev'", specifier = ">=8.3.0" }, - { name = "pytest-cov", marker = "extra == 'dev'", specifier = ">=5.0.0" }, - { name = "python-dotenv", specifier = ">=1.0.1" }, - { name = "redis", specifier = ">=5.2.0" }, - { name = "sqlalchemy", specifier = ">=2.0.32" }, - { name = "uvicorn", specifier = ">=0.34.0" }, -] -provides-extras = ["dev"] - [[package]] name = "idna" version = "3.11" @@ -509,25 +499,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/b9/c538f279a4e237a006a2c98387d081e9eb060d203d8ed34467cc0f0b9b53/packaging-26.0-py3-none-any.whl", hash = "sha256:b36f1fef9334a5588b4166f8bcd26a14e521f2b55e6b9de3aaa80d3ff7a37529", size = 74366, upload-time = "2026-01-21T20:50:37.788Z" }, ] -[[package]] -name = "playwright" -version = "1.58.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet" }, - { name = "pyee" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098, upload-time = "2026-01-30T15:09:24.028Z" }, - { url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625, upload-time = "2026-01-30T15:09:27.558Z" }, - { url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098, upload-time = "2026-01-30T15:09:30.461Z" }, - { url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268, upload-time = "2026-01-30T15:09:33.787Z" }, - { url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214, upload-time = "2026-01-30T15:09:36.751Z" }, - { url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998, upload-time = "2026-01-30T15:09:39.627Z" }, - { url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005, upload-time = "2026-01-30T15:09:42.449Z" }, - { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" }, -] - [[package]] name = "pluggy" version = "1.6.0" @@ -713,18 +684,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/c7/cfc8e811f061c841d7990b0201912c3556bfeb99cdcb7ed24adc8d6f8704/pydantic_core-2.41.5-pp311-pypy311_pp73-win_amd64.whl", hash = "sha256:56121965f7a4dc965bff783d70b907ddf3d57f6eba29b6d2e5dabfaf07799c51", size = 2145302, upload-time = "2025-11-04T13:43:46.64Z" }, ] -[[package]] -name = "pyee" -version = "13.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, -] - [[package]] name = "pygments" version = "2.20.0" @@ -968,6 +927,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c2/14/e2a54fabd4f08cd7af1c07030603c3356b74da07f7cc056e600436edfa17/tzlocal-5.3.1-py3-none-any.whl", hash = "sha256:eb1a66c3ef5847adf7a834f1be0800581b683b5608e74f86ecbcef8ab91bb85d", size = 18026, upload-time = "2025-03-05T21:17:39.857Z" }, ] +[[package]] +name = "urllib3" +version = "2.6.3" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/c7/24/5f1b3bdffd70275f6661c76461e25f024d5a38a46f04aaca912426a2b1d3/urllib3-2.6.3.tar.gz", hash = "sha256:1b62b6884944a57dbe321509ab94fd4d3b307075e0c2eae991ac71ee15ad38ed", size = 435556, upload-time = "2026-01-07T16:24:43.925Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/39/08/aaaad47bc4e9dc8c725e68f9d04865dbcb2052843ff09c97b08904852d84/urllib3-2.6.3-py3-none-any.whl", hash = "sha256:bf272323e553dfb2e87d9bfd225ca7b0f467b919d7bbd355436d3fd37cb0acd4", size = 131584, upload-time = "2026-01-07T16:24:42.685Z" }, +] + [[package]] name = "uvicorn" version = "0.44.0"