Prepare mobile de parser preview
This commit is contained in:
qananasikq
0
iaai_scraper/api/routes/__init__.py
Normal file
0
iaai_scraper/api/routes/__init__.py
Normal file
103
iaai_scraper/api/routes/cars.py
Normal file
103
iaai_scraper/api/routes/cars.py
Normal file
@@ -0,0 +1,103 @@
|
||||
# Роуты для просмотра автомобилей и агрегированной статистики.
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..deps import get_persistence
|
||||
from ...storage.db import PersistenceService
|
||||
from ...storage.models import Car, Image
|
||||
from ...storage.schemas import CarRead
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/cars")
|
||||
def list_cars(
|
||||
page: int = Query(1, ge=1),
|
||||
per_page: int = Query(20, ge=1, le=100),
|
||||
brand: str | None = None,
|
||||
model: str | None = None,
|
||||
year_min: int | None = None,
|
||||
year_max: int | None = None,
|
||||
is_sold: bool | None = None,
|
||||
persistence: PersistenceService = Depends(get_persistence),
|
||||
):
|
||||
# Список автомобилей с пагинацией и фильтрами
|
||||
with persistence.session_scope() as session:
|
||||
query = select(Car)
|
||||
|
||||
if brand:
|
||||
query = query.where(Car.brand.ilike(f"%{brand}%"))
|
||||
if model:
|
||||
query = query.where(Car.model.ilike(f"%{model}%"))
|
||||
if year_min is not None:
|
||||
query = query.where(Car.year >= year_min)
|
||||
if year_max is not None:
|
||||
query = query.where(Car.year <= year_max)
|
||||
if is_sold is not None:
|
||||
query = query.where(Car.is_sold == is_sold)
|
||||
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total = session.execute(count_query).scalar() or 0
|
||||
|
||||
offset = (page - 1) * per_page
|
||||
cars = session.execute(
|
||||
query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
|
||||
).scalars().all()
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"pages": (total + per_page - 1) // per_page if per_page else 0,
|
||||
"items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/cars/{car_id}")
|
||||
def get_car(
|
||||
car_id: int,
|
||||
persistence: PersistenceService = Depends(get_persistence),
|
||||
):
|
||||
# Детальная информация об автомобиле с изображениями
|
||||
with persistence.session_scope() as session:
|
||||
car = session.get(Car, car_id)
|
||||
if car is None:
|
||||
raise HTTPException(status_code=404, detail="Car not found")
|
||||
return CarRead.model_validate(car).model_dump(mode="json")
|
||||
|
||||
|
||||
@router.get("/cars/by-origin/{origin_id}")
|
||||
def get_car_by_origin(
|
||||
origin_id: str,
|
||||
persistence: PersistenceService = Depends(get_persistence),
|
||||
):
|
||||
# Поиск автомобиля по origin_id
|
||||
with persistence.session_scope() as session:
|
||||
car = session.execute(
|
||||
select(Car).where(Car.origin_id == origin_id)
|
||||
).scalars().first()
|
||||
if car is None:
|
||||
raise HTTPException(status_code=404, detail="Car not found")
|
||||
return CarRead.model_validate(car).model_dump(mode="json")
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
def get_stats(persistence: PersistenceService = Depends(get_persistence)):
|
||||
# Общая статистика по БД
|
||||
with persistence.session_scope() as session:
|
||||
total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
|
||||
total_images = session.execute(select(func.count(Image.id))).scalar() or 0
|
||||
brands = session.execute(
|
||||
select(Car.brand, func.count(Car.id))
|
||||
.group_by(Car.brand)
|
||||
.order_by(func.count(Car.id).desc())
|
||||
.limit(20)
|
||||
).all()
|
||||
|
||||
return {
|
||||
"total_cars": total_cars,
|
||||
"total_images": total_images,
|
||||
"top_brands": [{"brand": b, "count": c} for b, c in brands],
|
||||
}
|
||||
|
||||
30
iaai_scraper/api/routes/health.py
Normal file
30
iaai_scraper/api/routes/health.py
Normal file
@@ -0,0 +1,30 @@
|
||||
# Роут проверки доступности сервиса и соединения с БД.
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import text
|
||||
|
||||
from ..deps import get_persistence
|
||||
from ...storage.db import PersistenceService
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger("iaai_scraper.api.health")
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
def health_check(persistence: PersistenceService = Depends(get_persistence)):
|
||||
# Проверка API и БД
|
||||
db_ok = False
|
||||
try:
|
||||
with persistence.session_scope() as session:
|
||||
session.execute(text("SELECT 1"))
|
||||
db_ok = True
|
||||
except Exception:
|
||||
logger.warning("Health DB check failed", exc_info=True)
|
||||
|
||||
return {
|
||||
"status": "ok" if db_ok else "degraded",
|
||||
"service": "mobilede-scraper-api",
|
||||
"database": "connected" if db_ok else "unavailable",
|
||||
}
|
||||
223
iaai_scraper/api/routes/tasks.py
Normal file
223
iaai_scraper/api/routes/tasks.py
Normal file
@@ -0,0 +1,223 @@
|
||||
# Роуты запуска задач синхронизации и просмотра истории sync-runs.
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel
|
||||
from redis import Redis
|
||||
from sqlalchemy import select, func
|
||||
|
||||
from ...core.config import Settings
|
||||
from ..deps import get_persistence
|
||||
from ...storage.db import PersistenceService
|
||||
from ...storage.models import SyncRun
|
||||
from ...worker.celery_app import IAAI_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
|
||||
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class SyncVehicleRequest(BaseModel):
|
||||
vehicle_url: str
|
||||
lane: str = "iaai"
|
||||
|
||||
|
||||
class SyncListingRequest(BaseModel):
|
||||
make: str | None = None
|
||||
model: str | None = None
|
||||
lane: str = "iaai_cars"
|
||||
limit: int | None = None
|
||||
only_new: bool | None = None
|
||||
|
||||
|
||||
class MobileDeSyncSearchRequest(BaseModel):
|
||||
start_page: int = 1
|
||||
max_pages: int = 5
|
||||
lane: str = "mobile_de_cars"
|
||||
search_url: str | None = None
|
||||
make_id: str | None = None
|
||||
model_id: str | None = None
|
||||
price_min: str | None = None
|
||||
price_max: str | None = None
|
||||
year_min: str | None = None
|
||||
year_max: str | None = None
|
||||
delay_seconds: float = 0.7
|
||||
use_cursor: bool = False
|
||||
continuous: bool = True
|
||||
|
||||
|
||||
class MobileDeSyncDetailRequest(BaseModel):
|
||||
listing_id: str
|
||||
lane: str = "mobile_de_cars"
|
||||
|
||||
|
||||
class MobileDeRuntimeSegmentsRequest(BaseModel):
|
||||
lane: str = "mobile_de_cars"
|
||||
delay_seconds: float = 0.7
|
||||
use_cursor: bool = True
|
||||
continuous: bool = True
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-search")
|
||||
def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
|
||||
result = mobilede_sync_search_task.apply_async(
|
||||
kwargs=body.model_dump(),
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-detail")
|
||||
def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
|
||||
result = mobilede_sync_detail_task.apply_async(
|
||||
kwargs=body.model_dump(),
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
"listing_id": body.listing_id,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-runtime-segments")
|
||||
def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
|
||||
result = mobilede_sync_runtime_segments_task.apply_async(
|
||||
kwargs=body.model_dump(),
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tasks/sync-vehicle")
|
||||
def start_sync_vehicle(
|
||||
body: SyncVehicleRequest,
|
||||
):
|
||||
# Запустить задачу скрапинга одного автомобиля через Celery
|
||||
result = sync_vehicle_task.apply_async(
|
||||
kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
|
||||
queue=IAAI_SYNC_QUEUE,
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"vehicle_url": body.vehicle_url,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tasks/sync-listing")
|
||||
def start_sync_listing(
|
||||
body: SyncListingRequest,
|
||||
):
|
||||
# Запустить задачу полного цикла листинга через Celery
|
||||
result = sync_listing_task.apply_async(
|
||||
kwargs={
|
||||
"make": body.make,
|
||||
"model": body.model,
|
||||
"lane": body.lane,
|
||||
"limit": body.limit,
|
||||
"only_new": body.only_new,
|
||||
},
|
||||
queue=IAAI_SYNC_QUEUE,
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/tasks/{task_id}")
|
||||
def get_task_status(task_id: str):
|
||||
result = celery_app.AsyncResult(task_id)
|
||||
|
||||
payload: dict = {
|
||||
"task_id": task_id,
|
||||
"state": result.state,
|
||||
}
|
||||
|
||||
if result.successful():
|
||||
payload["result"] = result.result
|
||||
elif result.failed():
|
||||
payload["error"] = str(result.result)
|
||||
elif result.info is not None:
|
||||
payload["meta"] = result.info
|
||||
|
||||
progress = _read_task_progress(task_id)
|
||||
if progress is not None:
|
||||
payload["progress"] = progress
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def _read_task_progress(task_id: str) -> dict | None:
|
||||
redis_client = None
|
||||
try:
|
||||
settings = Settings()
|
||||
redis_client = Redis.from_url(
|
||||
settings.redis.url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
|
||||
socket_timeout=settings.redis.socket_timeout_seconds,
|
||||
health_check_interval=settings.redis.health_check_interval_seconds,
|
||||
retry_on_timeout=True,
|
||||
)
|
||||
raw = redis_client.get(f"iaai:state:task_progress:{task_id}")
|
||||
if not raw:
|
||||
return None
|
||||
data = json.loads(raw)
|
||||
return data if isinstance(data, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if redis_client is not None:
|
||||
try:
|
||||
redis_client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@router.get("/sync-runs")
|
||||
def list_sync_runs(
|
||||
page: int = Query(1, ge=1),
|
||||
per_page: int = Query(20, ge=1, le=100),
|
||||
persistence: PersistenceService = Depends(get_persistence),
|
||||
):
|
||||
# История запусков синхронизации
|
||||
with persistence.session_scope() as session:
|
||||
total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
|
||||
offset = (page - 1) * per_page
|
||||
runs = session.execute(
|
||||
select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
|
||||
).scalars().all()
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"items": [
|
||||
{
|
||||
"id": r.id,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||
"status": r.status,
|
||||
"lane": r.lane,
|
||||
"ids_fetched": r.ids_fetched,
|
||||
"cars_upserted": r.cars_upserted,
|
||||
"cars_failed": r.cars_failed,
|
||||
"images_upserted": r.images_upserted,
|
||||
"error_summary": r.error_summary,
|
||||
}
|
||||
for r in runs
|
||||
],
|
||||
}
|
||||
Reference in New Issue
Block a user