qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepare mobile de parser release

Browse Source
This commit is contained in:
qananasikq
2026-04-27 21:10:17 +03:00
parent 31f87a9bdf
commit b86bcd04b8
21 changed files with 2835 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
iaai_scraper/worker/celery_app.py
View File

@@ -15,6 +15,7 @@ from ..core.logs import setup_logging
logger = logging.getLogger("iaai_scraper.worker.celery_app")
STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
IAAI_SYNC_QUEUE = "iaai_sync"
MOBILEDE_SYNC_QUEUE = "mobilede_sync"
PROGRESS_KEY_PREFIX = "iaai:state:task_progress:"
@@ -108,18 +109,25 @@ celery_app.conf.update(
worker_redirect_stdouts=False,
worker_hijack_root_logger=False,
beat_schedule={
"periodic-sync-listing": {
"task": "iaai.sync_cars_feed",
"periodic-mobilede-sync-search": {
"task": "mobilede.sync_runtime_segments",
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
"args": (),
"kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
"kwargs": {
"delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
"use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
"continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
},
"options": {
"queue": IAAI_SYNC_QUEUE,
"queue": MOBILEDE_SYNC_QUEUE,
"expires": settings.celery.beat_sync_interval_minutes * 60.0,
},
}
},
task_routes={
"mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
"mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
"mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE},
"iaai.sync_cars_feed": {"queue": IAAI_SYNC_QUEUE},
"iaai_scraper.worker.tasks.*": {"queue": IAAI_SYNC_QUEUE},
},
@@ -186,10 +194,14 @@ def _on_worker_ready(**kwargs):
logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
return
logger.info("Worker ready — dispatching initial sync_listing task")
logger.info("Worker ready — dispatching initial mobile.de sync_search task")
celery_app.send_task(
"iaai.sync_cars_feed",
kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
queue=IAAI_SYNC_QUEUE,
"mobilede.sync_runtime_segments",
kwargs={
"delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
"use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
"continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
},
queue=MOBILEDE_SYNC_QUEUE,
expires=settings.celery.beat_sync_interval_minutes * 60.0,
)