Prepare mobile de parser release
This commit is contained in:
@@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
|
||||
|
||||
return {
|
||||
"status": "ok" if db_ok else "degraded",
|
||||
"service": "iaai-scraper-api",
|
||||
"service": "mobilede-scraper-api",
|
||||
"database": "connected" if db_ok else "unavailable",
|
||||
}
|
||||
|
||||
@@ -1,14 +1,18 @@
|
||||
# Роуты запуска задач синхронизации и просмотра истории sync-runs.
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel
|
||||
from redis import Redis
|
||||
from sqlalchemy import select, func
|
||||
|
||||
from ...core.config import Settings
|
||||
from ..deps import get_persistence
|
||||
from ...storage.db import PersistenceService
|
||||
from ...storage.models import SyncRun
|
||||
from ...worker.celery_app import IAAI_SYNC_QUEUE, celery_app
|
||||
from ...worker.tasks import sync_vehicle_task, sync_listing_task
|
||||
from ...worker.celery_app import IAAI_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
|
||||
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
@@ -26,6 +30,74 @@ class SyncListingRequest(BaseModel):
|
||||
only_new: bool | None = None
|
||||
|
||||
|
||||
class MobileDeSyncSearchRequest(BaseModel):
|
||||
start_page: int = 1
|
||||
max_pages: int = 5
|
||||
lane: str = "mobile_de_cars"
|
||||
search_url: str | None = None
|
||||
make_id: str | None = None
|
||||
model_id: str | None = None
|
||||
price_min: str | None = None
|
||||
price_max: str | None = None
|
||||
year_min: str | None = None
|
||||
year_max: str | None = None
|
||||
delay_seconds: float = 0.7
|
||||
use_cursor: bool = False
|
||||
continuous: bool = True
|
||||
|
||||
|
||||
class MobileDeSyncDetailRequest(BaseModel):
|
||||
listing_id: str
|
||||
lane: str = "mobile_de_cars"
|
||||
|
||||
|
||||
class MobileDeRuntimeSegmentsRequest(BaseModel):
|
||||
lane: str = "mobile_de_cars"
|
||||
delay_seconds: float = 0.7
|
||||
use_cursor: bool = True
|
||||
continuous: bool = True
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-search")
|
||||
def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
|
||||
result = mobilede_sync_search_task.apply_async(
|
||||
kwargs=body.model_dump(),
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-detail")
|
||||
def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
|
||||
result = mobilede_sync_detail_task.apply_async(
|
||||
kwargs=body.model_dump(),
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
"listing_id": body.listing_id,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-runtime-segments")
|
||||
def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
|
||||
result = mobilede_sync_runtime_segments_task.apply_async(
|
||||
kwargs=body.model_dump(),
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tasks/sync-vehicle")
|
||||
def start_sync_vehicle(
|
||||
body: SyncVehicleRequest,
|
||||
@@ -81,9 +153,40 @@ def get_task_status(task_id: str):
|
||||
elif result.info is not None:
|
||||
payload["meta"] = result.info
|
||||
|
||||
progress = _read_task_progress(task_id)
|
||||
if progress is not None:
|
||||
payload["progress"] = progress
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def _read_task_progress(task_id: str) -> dict | None:
|
||||
redis_client = None
|
||||
try:
|
||||
settings = Settings()
|
||||
redis_client = Redis.from_url(
|
||||
settings.redis.url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
|
||||
socket_timeout=settings.redis.socket_timeout_seconds,
|
||||
health_check_interval=settings.redis.health_check_interval_seconds,
|
||||
retry_on_timeout=True,
|
||||
)
|
||||
raw = redis_client.get(f"iaai:state:task_progress:{task_id}")
|
||||
if not raw:
|
||||
return None
|
||||
data = json.loads(raw)
|
||||
return data if isinstance(data, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if redis_client is not None:
|
||||
try:
|
||||
redis_client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@router.get("/sync-runs")
|
||||
def list_sync_runs(
|
||||
page: int = Query(1, ge=1),
|
||||
|
||||
Reference in New Issue
Block a user