add fastapi rest api and celery workers

iaai_scraper/api/__init__.py

@@ -0,0 +1,3 @@
+from .app import create_app
+
+__all__ = ["create_app"]

iaai_scraper/api/app.py

@@ -0,0 +1,41 @@
+"""FastAPI app."""
+
+from contextlib import asynccontextmanager
+
+from fastapi import FastAPI
+
+from ..core.config import Settings
+from ..storage.db import PersistenceService
+from .routes import cars, health, tasks
+
+
+@asynccontextmanager
+async def lifespan(app: FastAPI):
+    """Жизненный цикл."""
+    persistence: PersistenceService = app.state.persistence
+    persistence.create_tables()
+    yield
+
+
+def create_app(settings: Settings | None = None) -> FastAPI:
+    _settings = settings or Settings()
+
+    app = FastAPI(
+        title="IAAI Scraper API",
+        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
+        version="1.0.0",
+        lifespan=lifespan,
+    )
+
+    app.state.settings = _settings
+    app.state.persistence = PersistenceService(_settings)
+
+    app.include_router(health.router, tags=["health"])
+    app.include_router(cars.router, prefix="/api/v1", tags=["cars"])
+    app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"])
+
+    return app
+
+
+# Для запуска через uvicorn.
+app = create_app()

iaai_scraper/api/deps.py

@@ -0,0 +1,9 @@
+"""FastAPI deps."""
+
+from fastapi import Request
+
+from ..storage.db import PersistenceService
+
+
+def get_persistence(request: Request) -> PersistenceService:
+    return request.app.state.persistence

iaai_scraper/api/routes/cars.py

@@ -0,0 +1,144 @@
+"""Cars endpoints."""
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from sqlalchemy import func, select
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import Car, Image
+
+router = APIRouter()
+
+
+@router.get("/cars")
+def list_cars(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    brand: str | None = None,
+    model: str | None = None,
+    year_min: int | None = None,
+    year_max: int | None = None,
+    is_sold: bool | None = None,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    """Список автомобилей с пагинацией и фильтрами."""
+    with persistence.session_scope() as session:
+        query = select(Car)
+
+        if brand:
+            query = query.where(Car.brand.ilike(f"%{brand}%"))
+        if model:
+            query = query.where(Car.model.ilike(f"%{model}%"))
+        if year_min is not None:
+            query = query.where(Car.year >= year_min)
+        if year_max is not None:
+            query = query.where(Car.year <= year_max)
+        if is_sold is not None:
+            query = query.where(Car.is_sold == is_sold)
+
+        # Общее количество.
+        count_query = select(func.count()).select_from(query.subquery())
+        total = session.execute(count_query).scalar() or 0
+
+        # Пагинация.
+        offset = (page - 1) * per_page
+        cars = session.execute(
+            query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "pages": (total + per_page - 1) // per_page if per_page else 0,
+            "items": [_car_to_dict(car) for car in cars],
+        }
+
+
+@router.get("/cars/{car_id}")
+def get_car(
+    car_id: int,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    """Детальная информация об автомобиле с изображениями."""
+    with persistence.session_scope() as session:
+        car = session.get(Car, car_id)
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return _car_to_dict(car, include_images=True)
+
+
+@router.get("/cars/by-origin/{origin_id}")
+def get_car_by_origin(
+    origin_id: str,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    """Поиск автомобиля по origin_id."""
+    with persistence.session_scope() as session:
+        car = session.execute(
+            select(Car).where(Car.origin_id == origin_id)
+        ).scalars().first()
+        if car is None:
+            raise HTTPException(status_code=404, detail="Car not found")
+        return _car_to_dict(car, include_images=True)
+
+
+@router.get("/stats")
+def get_stats(persistence: PersistenceService = Depends(get_persistence)):
+    """Общая статистика по БД."""
+    with persistence.session_scope() as session:
+        total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
+        total_images = session.execute(select(func.count(Image.id))).scalar() or 0
+        brands = session.execute(
+            select(Car.brand, func.count(Car.id))
+            .group_by(Car.brand)
+            .order_by(func.count(Car.id).desc())
+            .limit(20)
+        ).all()
+
+        return {
+            "total_cars": total_cars,
+            "total_images": total_images,
+            "top_brands": [{"brand": b, "count": c} for b, c in brands],
+        }
+
+
+def _car_to_dict(car: Car, include_images: bool = False) -> dict:
+    """Сериализация Car в dict."""
+    result = {
+        "id": car.id,
+        "parser_id": car.parser_id,
+        "brand": car.brand,
+        "model": car.model,
+        "year": car.year,
+        "price": car.price,
+        "currency": car.currency,
+        "mileage": car.mileage,
+        "country": car.country,
+        "is_sold": car.is_sold,
+        "color": car.color,
+        "drive": car.drive,
+        "gearbox": car.gearbox,
+        "steering_wheel": car.steering_wheel,
+        "body_type": car.body_type,
+        "engine_volume": car.engine_volume,
+        "selling_type": car.selling_type,
+        "origin": car.origin,
+        "origin_url": car.origin_url,
+        "origin_id": car.origin_id,
+        "is_damaged": car.is_damaged,
+        "slug": car.slug,
+        "last_seen_at": car.last_seen_at.isoformat() if car.last_seen_at else None,
+        "content_hash": car.content_hash,
+    }
+    if include_images:
+        result["images"] = [
+            {
+                "id": img.id,
+                "fullres_image": img.fullres_image,
+                "preview_image": img.preview_image,
+                "order_index": img.order_index,
+            }
+            for img in sorted(car.images, key=lambda i: i.order_index)
+        ]
+    return result

iaai_scraper/api/routes/health.py

@@ -0,0 +1,27 @@
+"""Health endpoint."""
+
+from fastapi import APIRouter, Depends
+from sqlalchemy import text
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+
+router = APIRouter()
+
+
+@router.get("/health")
+def health_check(persistence: PersistenceService = Depends(get_persistence)):
+    """Проверка API и БД."""
+    db_ok = False
+    try:
+        with persistence.session_scope() as session:
+            session.execute(text("SELECT 1"))
+            db_ok = True
+    except Exception:
+        pass
+
+    return {
+        "status": "ok" if db_ok else "degraded",
+        "service": "iaai-scraper-api",
+        "database": "connected" if db_ok else "unavailable",
+    }

iaai_scraper/api/routes/tasks.py

@@ -0,0 +1,174 @@
+"""Task endpoints."""
+
+from datetime import datetime, timezone
+
+from fastapi import APIRouter, Depends, HTTPException, Query
+from pydantic import BaseModel
+from sqlalchemy import select, func
+
+from ..deps import get_persistence
+from ...storage.db import PersistenceService
+from ...storage.models import ScrapeTask, SyncRun
+from ...worker.tasks import sync_vehicle_task, sync_listing_task
+
+router = APIRouter()
+
+
+# --- Схемы.
+
+class SyncVehicleRequest(BaseModel):
+    vehicle_url: str
+    lane: str = "iaai"
+
+
+class SyncListingRequest(BaseModel):
+    make: str | None = None
+    model: str | None = None
+    lane: str = "iaai_cars"
+    limit: int | None = None
+    only_new: bool | None = None
+
+
+# --- Эндпоинты.
+
+@router.post("/tasks/sync-vehicle")
+def start_sync_vehicle(
+    body: SyncVehicleRequest,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    """Запустить задачу скрапинга одного автомобиля через Celery."""
+    result = sync_vehicle_task.delay(body.vehicle_url, lane=body.lane)
+
+    # Регистрируем задачу в БД.
+    persistence.create_scrape_task(
+        celery_task_id=result.id,
+        task_type="sync_vehicle",
+        vehicle_url=body.vehicle_url,
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+        "vehicle_url": body.vehicle_url,
+    }
+
+
+@router.post("/tasks/sync-listing")
+def start_sync_listing(
+    body: SyncListingRequest,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    """Запустить задачу полного цикла листинга через Celery."""
+    result = sync_listing_task.delay(
+        make=body.make,
+        model=body.model,
+        lane=body.lane,
+        limit=body.limit,
+        only_new=body.only_new,
+    )
+
+    persistence.create_scrape_task(
+        celery_task_id=result.id,
+        task_type="sync_listing",
+    )
+
+    return {
+        "task_id": result.id,
+        "status": "queued",
+    }
+
+
+@router.get("/tasks/{task_id}")
+def get_task_status(
+    task_id: str,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    """Статус Celery-задачи."""
+    task_info = persistence.get_scrape_task(task_id)
+    if task_info is None:
+        raise HTTPException(status_code=404, detail="Task not found")
+    return task_info
+
+
+@router.get("/tasks")
+def list_tasks(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    status: str | None = None,
+    task_type: str | None = None,
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    """Список задач с пагинацией."""
+    with persistence.session_scope() as session:
+        query = select(ScrapeTask)
+        if status:
+            query = query.where(ScrapeTask.status == status)
+        if task_type:
+            query = query.where(ScrapeTask.task_type == task_type)
+
+        total = session.execute(
+            select(func.count()).select_from(query.subquery())
+        ).scalar() or 0
+
+        offset = (page - 1) * per_page
+        tasks = session.execute(
+            query.order_by(ScrapeTask.created_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": t.id,
+                    "celery_task_id": t.celery_task_id,
+                    "task_type": t.task_type,
+                    "status": t.status,
+                    "vehicle_url": t.vehicle_url,
+                    "created_at": t.created_at.isoformat() if t.created_at else None,
+                    "started_at": t.started_at.isoformat() if t.started_at else None,
+                    "finished_at": t.finished_at.isoformat() if t.finished_at else None,
+                    "result_summary": t.result_summary,
+                    "error_message": t.error_message,
+                }
+                for t in tasks
+            ],
+        }
+
+
+@router.get("/sync-runs")
+def list_sync_runs(
+    page: int = Query(1, ge=1),
+    per_page: int = Query(20, ge=1, le=100),
+    persistence: PersistenceService = Depends(get_persistence),
+):
+    """История запусков синхронизации."""
+    with persistence.session_scope() as session:
+        total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
+
+        offset = (page - 1) * per_page
+        runs = session.execute(
+            select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
+        ).scalars().all()
+
+        return {
+            "total": total,
+            "page": page,
+            "per_page": per_page,
+            "items": [
+                {
+                    "id": r.id,
+                    "started_at": r.started_at.isoformat() if r.started_at else None,
+                    "finished_at": r.finished_at.isoformat() if r.finished_at else None,
+                    "status": r.status,
+                    "lane": r.lane,
+                    "ids_fetched": r.ids_fetched,
+                    "cars_upserted": r.cars_upserted,
+                    "cars_failed": r.cars_failed,
+                    "images_upserted": r.images_upserted,
+                    "error_summary": r.error_summary,
+                }
+                for r in runs
+            ],
+        }