qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor mobile.de parser, fix country mapping, update README

Browse Source
This commit is contained in:
qananasikq
2026-05-04 18:12:10 +03:00
parent edddfc884c
commit 800867b4e6
46 changed files with 2887 additions and 10900 deletions
Download Patch File Download Diff File Expand all files Collapse all files

227
mobilede_scraper/api/routes/__init__.py
View File

@@ -1,224 +1,7 @@
# Роуты запуска задач синхронизации и просмотра истории sync-runs.
"""API route package.
import json
from fastapi import APIRouter, Depends, Query
from pydantic import BaseModel
from redis import Redis
from sqlalchemy import select, func
from ...core.config import Settings
from ..deps import get_persistence
from ...storage.db import PersistenceService
from ...storage.models import SyncRun
from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
router = APIRouter()
class SyncVehicleRequest(BaseModel):
vehicle_url: str
lane: str = "MOBILEDE"
class SyncListingRequest(BaseModel):
make: str | None = None
model: str | None = None
lane: str = "MOBILEDE_cars"
limit: int | None = None
only_new: bool | None = None
class MobileDeSyncSearchRequest(BaseModel):
start_page: int = 1
max_pages: int = 5
lane: str = "mobile_de_cars"
search_url: str | None = None
make_id: str | None = None
model_id: str | None = None
price_min: str | None = None
price_max: str | None = None
year_min: str | None = None
year_max: str | None = None
delay_seconds: float = 0.7
use_cursor: bool = False
continuous: bool = False
class MobileDeSyncDetailRequest(BaseModel):
listing_id: str
lane: str = "mobile_de_cars"
class MobileDeRuntimeSegmentsRequest(BaseModel):
lane: str = "mobile_de_cars"
delay_seconds: float = 0.7
use_cursor: bool = True
continuous: bool = False
@router.post("/mobilede/tasks/sync-search")
def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
result = mobilede_sync_search_task.apply_async(
kwargs=body.model_dump(),
queue=MOBILEDE_SYNC_QUEUE,
)
return {
"task_id": result.id,
"status": "queued",
"queue": MOBILEDE_SYNC_QUEUE,
}
@router.post("/mobilede/tasks/sync-detail")
def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
result = mobilede_sync_detail_task.apply_async(
kwargs=body.model_dump(),
queue=MOBILEDE_SYNC_QUEUE,
)
return {
"task_id": result.id,
"status": "queued",
"queue": MOBILEDE_SYNC_QUEUE,
"listing_id": body.listing_id,
}
@router.post("/mobilede/tasks/sync-runtime-segments")
def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
result = mobilede_sync_runtime_segments_task.apply_async(
kwargs=body.model_dump(),
queue=MOBILEDE_SYNC_QUEUE,
)
return {
"task_id": result.id,
"status": "queued",
"queue": MOBILEDE_SYNC_QUEUE,
}
@router.post("/tasks/sync-vehicle")
def start_sync_vehicle(
body: SyncVehicleRequest,
):
# Запустить задачу скрапинга одного автомобиля через Celery
result = sync_vehicle_task.apply_async(
kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
queue=MOBILEDE_SYNC_QUEUE,
)
return {
"task_id": result.id,
"status": "queued",
"vehicle_url": body.vehicle_url,
}
@router.post("/tasks/sync-listing")
def start_sync_listing(
body: SyncListingRequest,
):
# Запустить задачу полного цикла листинга через Celery
result = sync_listing_task.apply_async(
kwargs={
"make": body.make,
"model": body.model,
"lane": body.lane,
"limit": body.limit,
"only_new": body.only_new,
},
queue=MOBILEDE_SYNC_QUEUE,
)
return {
"task_id": result.id,
"status": "queued",
}
@router.get("/tasks/{task_id}")
def get_task_status(task_id: str):
result = celery_app.AsyncResult(task_id)
payload: dict = {
"task_id": task_id,
"state": result.state,
}
if result.successful():
payload["result"] = result.result
elif result.failed():
payload["error"] = str(result.result)
elif result.info is not None:
payload["meta"] = result.info
progress = _read_task_progress(task_id)
if progress is not None:
payload["progress"] = progress
return payload
def _read_task_progress(task_id: str) -> dict | None:
redis_client = None
try:
settings = Settings()
redis_client = Redis.from_url(
settings.redis.url,
decode_responses=True,
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
socket_timeout=settings.redis.socket_timeout_seconds,
health_check_interval=settings.redis.health_check_interval_seconds,
retry_on_timeout=True,
)
raw = redis_client.get(f"mobilede:state:task_progress:{task_id}")
if not raw:
return None
data = json.loads(raw)
return data if isinstance(data, dict) else None
except Exception:
return None
finally:
if redis_client is not None:
try:
redis_client.close()
except Exception:
pass
@router.get("/sync-runs")
def list_sync_runs(
page: int = Query(1, ge=1),
per_page: int = Query(20, ge=1, le=100),
persistence: PersistenceService = Depends(get_persistence),
):
# История запусков синхронизации
with persistence.session_scope() as session:
total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
offset = (page - 1) * per_page
runs = session.execute(
select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
).scalars().all()
return {
"total": total,
"page": page,
"per_page": per_page,
"items": [
{
"id": r.id,
"started_at": r.started_at.isoformat() if r.started_at else None,
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
"status": r.status,
"lane": r.lane,
"ids_fetched": r.ids_fetched,
"cars_upserted": r.cars_upserted,
"cars_failed": r.cars_failed,
"images_upserted": r.images_upserted,
"error_summary": r.error_summary,
}
for r in runs
],
}
Routers are defined in dedicated modules (`cars`, `health`, `tasks`). This
package intentionally does not create routes to avoid duplicate task endpoints
and accidental imports of legacy parser channels during FastAPI startup.
"""

60
mobilede_scraper/api/routes/tasks.py
View File

@@ -11,25 +11,12 @@ from ...core.config import Settings
from ..deps import get_persistence
from ...storage.db import PersistenceService
from ...storage.models import SyncRun
from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, celery_app
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task
router = APIRouter()
class SyncVehicleRequest(BaseModel):
vehicle_url: str
lane: str = "MOBILEDE"
class SyncListingRequest(BaseModel):
make: str | None = None
model: str | None = None
lane: str = "MOBILEDE_cars"
limit: int | None = None
only_new: bool | None = None
class MobileDeSyncSearchRequest(BaseModel):
start_page: int = 1
max_pages: int = 5
@@ -43,7 +30,7 @@ class MobileDeSyncSearchRequest(BaseModel):
year_max: str | None = None
delay_seconds: float = 0.7
use_cursor: bool = False
continuous: bool | None = None
continuous: bool | None = False
class MobileDeSyncDetailRequest(BaseModel):
@@ -55,7 +42,7 @@ class MobileDeRuntimeSegmentsRequest(BaseModel):
lane: str = "mobile_de_cars"
delay_seconds: float = 0.7
use_cursor: bool = True
continuous: bool | None = None
continuous: bool | None = False
@router.post("/mobilede/tasks/sync-search")
@@ -98,45 +85,6 @@ def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
}
@router.post("/tasks/sync-vehicle")
def start_sync_vehicle(
body: SyncVehicleRequest,
):
# Запустить задачу скрапинга одного автомобиля через Celery
result = sync_vehicle_task.apply_async(
kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
queue=MOBILEDE_SYNC_QUEUE,
)
return {
"task_id": result.id,
"status": "queued",
"vehicle_url": body.vehicle_url,
}
@router.post("/tasks/sync-listing")
def start_sync_listing(
body: SyncListingRequest,
):
# Запустить задачу полного цикла листинга через Celery
result = sync_listing_task.apply_async(
kwargs={
"make": body.make,
"model": body.model,
"lane": body.lane,
"limit": body.limit,
"only_new": body.only_new,
},
queue=MOBILEDE_SYNC_QUEUE,
)
return {
"task_id": result.id,
"status": "queued",
}
@router.get("/tasks/{task_id}")
def get_task_status(task_id: str):
result = celery_app.AsyncResult(task_id)