qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Schedule stale gallery enrichment

Browse Source
This commit is contained in:
qananasikq
2026-08-18 10:40:26 +03:00
parent 19007dafc8
commit ca1fab1034
15 changed files with 460 additions and 82 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobilede_scraper/worker/celery_app.py
+27 -12
View File
@@ -15,6 +15,7 @@ from .constants import (
GLOBAL_DB_PROGRESS_TS_KEY,
GLOBAL_PROGRESS_TS_KEY,
MOBILEDE_RUNTIME_SEGMENTS_TASK,
MOBILEDE_IMAGES_QUEUE,
MOBILEDE_SYNC_QUEUE,
MOBILEDE_SYNC_TASK_NAME,
)
@@ -35,6 +36,7 @@ def _env_bool(name: str, default: bool) -> bool:
MOBILEDE_BEAT_SYNC_ENABLED = _env_bool("MOBILEDE_BEAT_SYNC_ENABLED", True)
MOBILEDE_BEAT_ENRICH_ENABLED = _env_bool("MOBILEDE_BEAT_ENRICH_ENABLED", True)
def _runtime_sync_kwargs() -> dict[str, bool | float]:
@@ -49,6 +51,10 @@ def _runtime_sync_expires_seconds() -> float:
return settings.celery.beat_sync_interval_minutes * 60.0
def _runtime_enrich_interval_seconds() -> float:
return max(60.0, float(os.getenv("MOBILEDE_ENRICH_INTERVAL_SECONDS", "900")))
def _has_fresh_active_progress(
redis_client: Redis,
*,
@@ -155,17 +161,26 @@ if _hard > _max_hard:
beat_schedule = {}
if MOBILEDE_BEAT_SYNC_ENABLED:
beat_schedule = {
"periodic-mobilede-sync-search": {
"task": MOBILEDE_RUNTIME_SEGMENTS_TASK,
"schedule": _runtime_sync_expires_seconds(),
"args": (),
"kwargs": _runtime_sync_kwargs(),
"options": {
"queue": MOBILEDE_SYNC_QUEUE,
"expires": _runtime_sync_expires_seconds(),
},
}
beat_schedule["periodic-mobilede-sync-search"] = {
"task": MOBILEDE_RUNTIME_SEGMENTS_TASK,
"schedule": _runtime_sync_expires_seconds(),
"args": (),
"kwargs": _runtime_sync_kwargs(),
"options": {
"queue": MOBILEDE_SYNC_QUEUE,
"expires": _runtime_sync_expires_seconds(),
},
}
if MOBILEDE_BEAT_ENRICH_ENABLED:
beat_schedule["periodic-mobilede-enrich-images"] = {
"task": MOBILEDE_ENRICH_IMAGES_TASK,
"schedule": _runtime_enrich_interval_seconds(),
"args": (),
"kwargs": {},
"options": {
"queue": MOBILEDE_IMAGES_QUEUE,
"expires": _runtime_enrich_interval_seconds(),
},
}
celery_app.conf.update(
@@ -195,7 +210,7 @@ celery_app.conf.update(
MOBILEDE_RUNTIME_SEGMENTS_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
MOBILEDE_SYNC_TASK_NAME: {"queue": MOBILEDE_SYNC_QUEUE},
MOBILEDE_SYNC_DETAIL_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
MOBILEDE_ENRICH_IMAGES_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
MOBILEDE_ENRICH_IMAGES_TASK: {"queue": MOBILEDE_IMAGES_QUEUE},
"mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE},
},
)