mobile.de/mobilede_scraper/api/routes/__init__.py

# Роуты запуска задач синхронизации и просмотра истории sync-runs.

import json

from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel
from redis import Redis
from sqlalchemy import select, func

from ...core.config import Settings
from ..deps import get_persistence
from ...storage.db import PersistenceService
from ...storage.models import SyncRun
from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task

router = APIRouter()


class SyncVehicleRequest(BaseModel):
    vehicle_url: str
    lane: str = "MOBILEDE"


class SyncListingRequest(BaseModel):
    make: str | None = None
    model: str | None = None
    lane: str = "MOBILEDE_cars"
    limit: int | None = None
    only_new: bool | None = None


class MobileDeSyncSearchRequest(BaseModel):
    start_page: int = 1
    max_pages: int = 5
    lane: str = "mobile_de_cars"
    search_url: str | None = None
    make_id: str | None = None
    model_id: str | None = None
    price_min: str | None = None
    price_max: str | None = None
    year_min: str | None = None
    year_max: str | None = None
    delay_seconds: float = 0.7
    use_cursor: bool = False
    continuous: bool = False


class MobileDeSyncDetailRequest(BaseModel):
    listing_id: str
    lane: str = "mobile_de_cars"


class MobileDeRuntimeSegmentsRequest(BaseModel):
    lane: str = "mobile_de_cars"
    delay_seconds: float = 0.7
    use_cursor: bool = True
    continuous: bool = False


@router.post("/mobilede/tasks/sync-search")
def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
    result = mobilede_sync_search_task.apply_async(
        kwargs=body.model_dump(),
        queue=MOBILEDE_SYNC_QUEUE,
    )
    return {
        "task_id": result.id,
        "status": "queued",
        "queue": MOBILEDE_SYNC_QUEUE,
    }


@router.post("/mobilede/tasks/sync-detail")
def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
    result = mobilede_sync_detail_task.apply_async(
        kwargs=body.model_dump(),
        queue=MOBILEDE_SYNC_QUEUE,
    )
    return {
        "task_id": result.id,
        "status": "queued",
        "queue": MOBILEDE_SYNC_QUEUE,
        "listing_id": body.listing_id,
    }


@router.post("/mobilede/tasks/sync-runtime-segments")
def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
    result = mobilede_sync_runtime_segments_task.apply_async(
        kwargs=body.model_dump(),
        queue=MOBILEDE_SYNC_QUEUE,
    )
    return {
        "task_id": result.id,
        "status": "queued",
        "queue": MOBILEDE_SYNC_QUEUE,
    }


@router.post("/tasks/sync-vehicle")
def start_sync_vehicle(
    body: SyncVehicleRequest,
):
    # Запустить задачу скрапинга одного автомобиля через Celery
    result = sync_vehicle_task.apply_async(
        kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
        queue=MOBILEDE_SYNC_QUEUE,
    )

    return {
        "task_id": result.id,
        "status": "queued",
        "vehicle_url": body.vehicle_url,
    }


@router.post("/tasks/sync-listing")
def start_sync_listing(
    body: SyncListingRequest,
):
    # Запустить задачу полного цикла листинга через Celery
    result = sync_listing_task.apply_async(
        kwargs={
            "make": body.make,
            "model": body.model,
            "lane": body.lane,
            "limit": body.limit,
            "only_new": body.only_new,
        },
        queue=MOBILEDE_SYNC_QUEUE,
    )

    return {
        "task_id": result.id,
        "status": "queued",
    }


@router.get("/tasks/{task_id}")
def get_task_status(task_id: str):
    result = celery_app.AsyncResult(task_id)

    payload: dict = {
        "task_id": task_id,
        "state": result.state,
    }

    if result.successful():
        payload["result"] = result.result
    elif result.failed():
        payload["error"] = str(result.result)
    elif result.info is not None:
        payload["meta"] = result.info

    progress = _read_task_progress(task_id)
    if progress is not None:
        payload["progress"] = progress

    return payload


def _read_task_progress(task_id: str) -> dict | None:
    redis_client = None
    try:
        settings = Settings()
        redis_client = Redis.from_url(
            settings.redis.url,
            decode_responses=True,
            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
            socket_timeout=settings.redis.socket_timeout_seconds,
            health_check_interval=settings.redis.health_check_interval_seconds,
            retry_on_timeout=True,
        )
        raw = redis_client.get(f"mobilede:state:task_progress:{task_id}")
        if not raw:
            return None
        data = json.loads(raw)
        return data if isinstance(data, dict) else None
    except Exception:
        return None
    finally:
        if redis_client is not None:
            try:
                redis_client.close()
            except Exception:
                pass


@router.get("/sync-runs")
def list_sync_runs(
    page: int = Query(1, ge=1),
    per_page: int = Query(20, ge=1, le=100),
    persistence: PersistenceService = Depends(get_persistence),
):
    # История запусков синхронизации
    with persistence.session_scope() as session:
        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
        offset = (page - 1) * per_page
        runs = session.execute(
            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
        ).scalars().all()

        return {
            "total": total,
            "page": page,
            "per_page": per_page,
            "items": [
                {
                    "id": r.id,
                    "started_at": r.started_at.isoformat() if r.started_at else None,
                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
                    "status": r.status,
                    "lane": r.lane,
                    "ids_fetched": r.ids_fetched,
                    "cars_upserted": r.cars_upserted,
                    "cars_failed": r.cars_failed,
                    "images_upserted": r.images_upserted,
                    "error_summary": r.error_summary,
                }
                for r in runs
            ],
        }