add mobilede scraper

mobilede_scraper/api/__init__.py

@@ -0,0 +1,3 @@
+from .app import create_app
+
+__all__ = ["create_app"]

mobilede_scraper/api/app.py

@@ -0,0 +1,40 @@
+# Создание FastAPI-приложения и настройка его жизненного цикла.
+
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from ..core.config import Settings
+from ..storage.db import PersistenceService
+from .routes import cars, health, tasks
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    # Жизненный цикл.
+    # Таблицы создаются через Alembic-миграции (сервис migrate).
+    yield
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    _settings = settings or Settings()
+
+    app = FastAPI(
+        title="mobile.de Scraper API",
+        description="REST API для управления задачами скрапинга mobile.de и просмотра данных",
+        version="1.0.0",
+        lifespan=lifespan,
+    )
+
+    app.state.settings = _settings
+    app.state.persistence = PersistenceService(_settings)
+
+    app.include_router(health.router, tags=["health"])
+    app.include_router(cars.router, prefix="/api/v1", tags=["cars"])
+    app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"])
+
+    return app
+
+
+# Экземпляр приложения для запуска через uvicorn.
+app = create_app()

mobilede_scraper/api/deps.py

@@ -0,0 +1,9 @@
+# Вспомогательные зависимости для FastAPI-роутов.
+
+from fastapi import Request
+
+from ..storage.db import PersistenceService
+
+
+def get_persistence(request: Request) -> PersistenceService:
+    return request.app.state.persistence

mobilede_scraper/api/routes/__init__.py

@@ -0,0 +1,224 @@
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.
+
+import json
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from redis import Redis
+from sqlalchemy import select, func
+
+from ...core.config import Settings
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import SyncRun
+from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
+from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
+
+router = APIRouter()
+
+
+class SyncVehicleRequest(BaseModel):
+    vehicle_url: str
+    lane: str = "MOBILEDE"
+
+
+class SyncListingRequest(BaseModel):
+    make: str | None = None
+    model: str | None = None
+    lane: str = "MOBILEDE_cars"
+    limit: int | None = None
+    only_new: bool | None = None
+
+
+class MobileDeSyncSearchRequest(BaseModel):
+    start_page: int = 1
+    max_pages: int = 5
+    lane: str = "mobile_de_cars"
+    search_url: str | None = None
+    make_id: str | None = None
+    model_id: str | None = None
+    price_min: str | None = None
+    price_max: str | None = None
+    year_min: str | None = None
+    year_max: str | None = None
+    delay_seconds: float = 0.7
+    use_cursor: bool = False
+    continuous: bool = False
+
+
+class MobileDeSyncDetailRequest(BaseModel):
+    listing_id: str
+    lane: str = "mobile_de_cars"
+
+
+class MobileDeRuntimeSegmentsRequest(BaseModel):
+    lane: str = "mobile_de_cars"
+    delay_seconds: float = 0.7
+    use_cursor: bool = True
+    continuous: bool = False
+
+
+@router.post("/mobilede/tasks/sync-search")
+def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
+    result = mobilede_sync_search_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/mobilede/tasks/sync-detail")
+def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
+    result = mobilede_sync_detail_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+        "listing_id": body.listing_id,
+    }
+
+
+@router.post("/mobilede/tasks/sync-runtime-segments")
+def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
+    result = mobilede_sync_runtime_segments_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/tasks/sync-vehicle")
+def start_sync_vehicle(
+    body: SyncVehicleRequest,
+):
+    # Запустить задачу скрапинга одного автомобиля через Celery
+    result = sync_vehicle_task.apply_async(
+        kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "vehicle_url": body.vehicle_url,
+    }
+
+
+@router.post("/tasks/sync-listing")
+def start_sync_listing(
+    body: SyncListingRequest,
+):
+    # Запустить задачу полного цикла листинга через Celery
+    result = sync_listing_task.apply_async(
+        kwargs={
+            "make": body.make,
+            "model": body.model,
+            "lane": body.lane,
+            "limit": body.limit,
+            "only_new": body.only_new,
+        },
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+    }
+
+
+@router.get("/tasks/{task_id}")
+def get_task_status(task_id: str):
+    result = celery_app.AsyncResult(task_id)
+
+    payload: dict = {
+        "task_id": task_id,
+        "state": result.state,
+    }
+
+    if result.successful():
+        payload["result"] = result.result
+    elif result.failed():
+        payload["error"] = str(result.result)
+    elif result.info is not None:
+        payload["meta"] = result.info
+
+    progress = _read_task_progress(task_id)
+    if progress is not None:
+        payload["progress"] = progress
+
+    return payload
+
+
+def _read_task_progress(task_id: str) -> dict | None:
+    redis_client = None
+    try:
+        settings = Settings()
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        raw = redis_client.get(f"mobilede:state:task_progress:{task_id}")
+        if not raw:
+            return None
+        data = json.loads(raw)
+        return data if isinstance(data, dict) else None
+    except Exception:
+        return None
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+
+@router.get("/sync-runs")
+def list_sync_runs(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # История запусков синхронизации
+    with persistence.session_scope() as session:
+        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+        offset = (page - 1) * per_page
+        runs = session.execute(
+            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": r.id,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
+                    "status": r.status,
+                    "lane": r.lane,
+                    "ids_fetched": r.ids_fetched,
+                    "cars_upserted": r.cars_upserted,
+                    "cars_failed": r.cars_failed,
+                    "images_upserted": r.images_upserted,
+                    "error_summary": r.error_summary,
+                }
+                for r in runs
+            ],
+        }
+

mobilede_scraper/api/routes/cars.py

@@ -0,0 +1,103 @@
+# Роуты для просмотра автомобилей и агрегированной статистики.
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import func, select
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import Car, Image
+from ...storage.schemas import CarRead
+
+router = APIRouter()
+
+
+@router.get("/cars")
+def list_cars(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    brand: str | None = None,
+    model: str | None = None,
+    year_min: int | None = None,
+    year_max: int | None = None,
+    is_sold: bool | None = None,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Список автомобилей с пагинацией и фильтрами
+    with persistence.session_scope() as session:
+        query = select(Car)
+
+        if brand:
+            query = query.where(Car.brand.ilike(f"%{brand}%"))
+        if model:
+            query = query.where(Car.model.ilike(f"%{model}%"))
+        if year_min is not None:
+            query = query.where(Car.year >= year_min)
+        if year_max is not None:
+            query = query.where(Car.year <= year_max)
+        if is_sold is not None:
+            query = query.where(Car.is_sold == is_sold)
+
+        count_query = select(func.count()).select_from(query.subquery())
+        total = session.execute(count_query).scalar() or 0
+
+        offset = (page - 1) * per_page
+        cars = session.execute(
+            query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "pages": (total + per_page - 1) // per_page if per_page else 0,
+            "items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars],
+        }
+
+
+@router.get("/cars/{car_id}")
+def get_car(
+    car_id: int,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Детальная информация об автомобиле с изображениями
+    with persistence.session_scope() as session:
+        car = session.get(Car, car_id)
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return CarRead.model_validate(car).model_dump(mode="json")
+
+
+@router.get("/cars/by-origin/{origin_id}")
+def get_car_by_origin(
+    origin_id: str,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # Поиск автомобиля по origin_id
+    with persistence.session_scope() as session:
+        car = session.execute(
+            select(Car).where(Car.origin_id == origin_id)
+        ).scalars().first()
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return CarRead.model_validate(car).model_dump(mode="json")
+
+
+@router.get("/stats")
+def get_stats(persistence: PersistenceService = Depends(get_persistence)):
+    # Общая статистика по БД
+    with persistence.session_scope() as session:
+        total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
+        total_images = session.execute(select(func.count(Image.id))).scalar() or 0
+        brands = session.execute(
+            select(Car.brand, func.count(Car.id))
+            .group_by(Car.brand)
+            .order_by(func.count(Car.id).desc())
+            .limit(20)
+        ).all()
+
+        return {
+            "total_cars": total_cars,
+            "total_images": total_images,
+            "top_brands": [{"brand": b, "count": c} for b, c in brands],
+        }
+

mobilede_scraper/api/routes/health.py

@@ -0,0 +1,30 @@
+# Роут проверки доступности сервиса и соединения с БД.
+
+import logging
+
+from fastapi import APIRouter, Depends
+from sqlalchemy import text
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+
+router = APIRouter()
+logger = logging.getLogger("MOBILEDE_scraper.api.health")
+
+
+@router.get("/health")
+def health_check(persistence: PersistenceService = Depends(get_persistence)):
+    # Проверка API и БД
+    db_ok = False
+    try:
+        with persistence.session_scope() as session:
+            session.execute(text("SELECT 1"))
+            db_ok = True
+    except Exception:
+        logger.warning("Health DB check failed", exc_info=True)
+
+    return {
+        "status": "ok" if db_ok else "degraded",
+        "service": "mobilede-scraper-api",
+        "database": "connected" if db_ok else "unavailable",
+    }

mobilede_scraper/api/routes/tasks.py

@@ -0,0 +1,223 @@
+# Роуты запуска задач синхронизации и просмотра истории sync-runs.
+
+import json
+
+from fastapi import APIRouter, Depends, Query
+from pydantic import BaseModel
+from redis import Redis
+from sqlalchemy import select, func
+
+from ...core.config import Settings
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import SyncRun
+from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
+from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
+
+router = APIRouter()
+
+
+class SyncVehicleRequest(BaseModel):
+    vehicle_url: str
+    lane: str = "MOBILEDE"
+
+
+class SyncListingRequest(BaseModel):
+    make: str | None = None
+    model: str | None = None
+    lane: str = "MOBILEDE_cars"
+    limit: int | None = None
+    only_new: bool | None = None
+
+
+class MobileDeSyncSearchRequest(BaseModel):
+    start_page: int = 1
+    max_pages: int = 5
+    lane: str = "mobile_de_cars"
+    search_url: str | None = None
+    make_id: str | None = None
+    model_id: str | None = None
+    price_min: str | None = None
+    price_max: str | None = None
+    year_min: str | None = None
+    year_max: str | None = None
+    delay_seconds: float = 0.7
+    use_cursor: bool = False
+    continuous: bool | None = None
+
+
+class MobileDeSyncDetailRequest(BaseModel):
+    listing_id: str
+    lane: str = "mobile_de_cars"
+
+
+class MobileDeRuntimeSegmentsRequest(BaseModel):
+    lane: str = "mobile_de_cars"
+    delay_seconds: float = 0.7
+    use_cursor: bool = True
+    continuous: bool | None = None
+
+
+@router.post("/mobilede/tasks/sync-search")
+def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
+    result = mobilede_sync_search_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/mobilede/tasks/sync-detail")
+def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
+    result = mobilede_sync_detail_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+        "listing_id": body.listing_id,
+    }
+
+
+@router.post("/mobilede/tasks/sync-runtime-segments")
+def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
+    result = mobilede_sync_runtime_segments_task.apply_async(
+        kwargs=body.model_dump(),
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "queue": MOBILEDE_SYNC_QUEUE,
+    }
+
+
+@router.post("/tasks/sync-vehicle")
+def start_sync_vehicle(
+    body: SyncVehicleRequest,
+):
+    # Запустить задачу скрапинга одного автомобиля через Celery
+    result = sync_vehicle_task.apply_async(
+        kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "vehicle_url": body.vehicle_url,
+    }
+
+
+@router.post("/tasks/sync-listing")
+def start_sync_listing(
+    body: SyncListingRequest,
+):
+    # Запустить задачу полного цикла листинга через Celery
+    result = sync_listing_task.apply_async(
+        kwargs={
+            "make": body.make,
+            "model": body.model,
+            "lane": body.lane,
+            "limit": body.limit,
+            "only_new": body.only_new,
+        },
+        queue=MOBILEDE_SYNC_QUEUE,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+    }
+
+
+@router.get("/tasks/{task_id}")
+def get_task_status(task_id: str):
+    result = celery_app.AsyncResult(task_id)
+
+    payload: dict = {
+        "task_id": task_id,
+        "state": result.state,
+    }
+
+    if result.successful():
+        payload["result"] = result.result
+    elif result.failed():
+        payload["error"] = str(result.result)
+    elif result.info is not None:
+        payload["meta"] = result.info
+
+    progress = _read_task_progress(task_id)
+    if progress is not None:
+        payload["progress"] = progress
+
+    return payload
+
+
+def _read_task_progress(task_id: str) -> dict | None:
+    redis_client = None
+    try:
+        settings = Settings()
+        redis_client = Redis.from_url(
+            settings.redis.url,
+            decode_responses=True,
+            socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
+            socket_timeout=settings.redis.socket_timeout_seconds,
+            health_check_interval=settings.redis.health_check_interval_seconds,
+            retry_on_timeout=True,
+        )
+        raw = redis_client.get(f"mobilede:state:task_progress:{task_id}")
+        if not raw:
+            return None
+        data = json.loads(raw)
+        return data if isinstance(data, dict) else None
+    except Exception:
+        return None
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
+
+
+@router.get("/sync-runs")
+def list_sync_runs(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    # История запусков синхронизации
+    with persistence.session_scope() as session:
+        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+        offset = (page - 1) * per_page
+        runs = session.execute(
+            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": r.id,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
+                    "status": r.status,
+                    "lane": r.lane,
+                    "ids_fetched": r.ids_fetched,
+                    "cars_upserted": r.cars_upserted,
+                    "cars_failed": r.cars_failed,
+                    "images_upserted": r.images_upserted,
+                    "error_summary": r.error_summary,
+                }
+                for r in runs
+            ],
+        }