qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor mobile.de parser, fix country mapping, update README

Browse Source
This commit is contained in:
qananasikq
2026-05-04 18:12:10 +03:00
parent edddfc884c
commit 800867b4e6
46 changed files with 2887 additions and 10900 deletions
Download Patch File Download Diff File Expand all files Collapse all files

21
mobilede_scraper/worker/self_heal.py
View File

@@ -7,17 +7,14 @@ import time
from redis import Redis
from .constants import DB_PROGRESS_STAGES
logger = logging.getLogger("MOBILEDE_scraper.worker.self_heal")
MOBILEDE_SYNC_QUEUE = "MOBILEDE_sync"
MOBILEDE_SYNC_QUEUE = "mobilede_sync"
GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts"
GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts"
SELF_HEAL_RESTART_LOCK_KEY = "mobilede:state:self_heal_restart_in_progress"
SYNC_LISTING_LOCK_KEY = "mobilede:locks:sync_listing"
SYNC_FULL_SCAN_DONE_KEY = "mobilede:state:sync_full_scan_done"
SYNC_LISTING_CHECKPOINT_KEY = "mobilede:state:sync_listing_checkpoint"
SYNC_LISTING_FOLLOWUP_PENDING_KEY = "mobilede:state:sync_listing_followup_pending"
SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
@@ -97,7 +94,7 @@ def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
if not payload:
continue
data = json.loads(payload)
if str(data.get("stage") or "") == "fast_db_progress":
if str(data.get("stage") or "") in DB_PROGRESS_STAGES:
ts = _safe_int(data.get("ts"), 0)
else:
ts = _safe_int(data.get("last_db_progress_ts"), 0)
@@ -110,28 +107,28 @@ def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None:
pipe = redis_client.pipeline()
pipe.delete(SYNC_LISTING_CHECKPOINT_KEY)
pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
pipe.delete(GLOBAL_PROGRESS_TS_KEY)
pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY)
pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0")
pipe.execute()
def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
queue_len = _safe_int(redis_client.llen(queue_name), 0)
has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0
has_segment_lock = 0
for _ in redis_client.scan_iter(match="mobilede:locks:segment:*"):
has_segment_lock = 1
break
has_task_progress = 0
for _ in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
has_task_progress = 1
break
flags = {
"queue_len": queue_len,
"has_lock": has_lock,
"has_segment_lock": has_segment_lock,
"has_task_progress": has_task_progress,
}
return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags
return (queue_len > 0 or has_segment_lock == 1 or has_task_progress == 1), flags
def _kill_worker_process() -> None: