qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Build complete runtime segments

Browse Source
This commit is contained in:
qananasikq
2026-08-19 17:11:00 +03:00
parent 8297742a64
commit 7068b7e8fa
5 changed files with 1054 additions and 232 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobilede_scraper/worker/refresh_cycle.py
+31 -9
View File
@@ -69,17 +69,28 @@ def _mobilede_redis_text(value: object) -> str:
def _mobilede_claim_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> bool:
ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
lease_seconds = min(300, max(60, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)))
return bool(
redis_client.set(
_mobilede_refresh_cycle_finalized_key(cycle_id),
"1",
"in_progress",
nx=True,
ex=ttl,
ex=lease_seconds,
)
)
def _mobilede_release_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> None:
key = _mobilede_refresh_cycle_finalized_key(cycle_id)
if _mobilede_redis_text(redis_client.get(key)) == "in_progress":
redis_client.delete(key)
def _mobilede_complete_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> None:
ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
redis_client.set(_mobilede_refresh_cycle_finalized_key(cycle_id), "completed", ex=ttl)
def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None:
cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
keys = [
@@ -217,6 +228,7 @@ def _mobilede_finalize_refresh_cycle_sold_marking(
post_refresh_probe_enabled: bool,
post_refresh_probe_batch_size: int,
schedule_post_refresh_probe: Callable[[], None],
scope_brands: tuple[str, ...] = (),
) -> int:
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
@@ -231,10 +243,12 @@ def _mobilede_finalize_refresh_cycle_sold_marking(
total_now,
sorted(unsafe_reasons),
)
_mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
return 0
started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
if not started_at_raw:
logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id)
_mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
return 0
try:
cutoff = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
@@ -246,18 +260,26 @@ def _mobilede_finalize_refresh_cycle_sold_marking(
started_at_raw,
cycle_id,
)
_mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
return 0
sold_marked = get_persistence().mark_sold_not_seen_since(
cutoff,
prefix=origin_prefixes,
safety_ratio=0.8,
)
try:
sold_marked = get_persistence().mark_sold_not_seen_since(
cutoff,
prefix=origin_prefixes,
safety_ratio=0.8,
brands=scope_brands,
)
except Exception:
_mobilede_release_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
raise
_mobilede_complete_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
logger.info(
"mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s sold_marked=%s",
"mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s brands=%s sold_marked=%s",
cycle_id,
done_now,
total_now,
cutoff.isoformat(),
list(scope_brands) if scope_brands else "all",
sold_marked,
)
if post_refresh_probe_enabled and post_refresh_probe_batch_size > 0: