qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Queue gallery jobs in Redis

Browse Source
This commit is contained in:
qananasikq
2026-08-20 11:34:00 +03:00
parent 1891f84291
commit cb49420bb2
3 changed files with 574 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_worker_runtime_tasks.py
+256 -61
View File
@@ -11,40 +11,27 @@ from mobilede_scraper.worker import planner, refresh_cycle, search_sync, tasks
class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
class _FakeDetailPersistence:
def __init__(self) -> None:
self.enqueued: list[str] = []
self.released: list[tuple[str, str | None]] = []
def enqueue_detail_jobs(self, origin_ids, *, priority=3, reset=False): # noqa: ARG002
self.enqueued = list(origin_ids)
return {"ready": len(self.enqueued), "deferred": 0, "missing": 0}
def reserve_detail_jobs(self, *, limit, lease_seconds=900, listing_ids=None): # noqa: ARG002
return [
{
"listing_id": listing_id,
"lease_owner": f"lease-{listing_id}",
"attempts": 1,
"priority": 9,
}
for listing_id in list(listing_ids or [])[:limit]
]
def release_detail_job(self, listing_id, *, lease_owner=None):
self.released.append((listing_id, lease_owner))
return True
class _FakeRedis:
def __init__(self) -> None:
self.store: dict[str, str] = {}
self.sets: dict[str, set[str]] = {}
self.lists: dict[str, list[str]] = {}
self.zsets: dict[str, dict[str, float]] = {}
def get(self, key: str):
return self.store.get(key)
def set(self, key: str, value: str, nx: bool = False, ex: int | None = None): # noqa: ARG002
def pttl(self, key: str):
return 1_000 if key in self.store else -2
def set(
self,
key: str,
value: str,
nx: bool = False,
ex: int | None = None,
px: int | None = None,
): # noqa: ARG002
if nx and key in self.store:
return False
self.store[key] = str(value)
@@ -72,12 +59,65 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
def scard(self, key: str):
return len(self.sets.get(key, set()))
def sismember(self, key: str, member: str):
return member in self.sets.get(key, set())
def smembers(self, key: str):
return self.sets.get(key, set())
def llen(self, key: str):
return len(self.lists.get(key, []))
def zcard(self, key: str):
return len(self.zsets.get(key, {}))
def zrem(self, key: str, member: str):
bucket = self.zsets.setdefault(key, {})
return 1 if bucket.pop(str(member), None) is not None else 0
def zremrangebyscore(self, key: str, minimum, maximum): # noqa: ANN001
bucket = self.zsets.setdefault(key, {})
upper = float(maximum)
removed = [member for member, score in bucket.items() if score <= upper]
for member in removed:
del bucket[member]
return len(removed)
def eval(self, script: str, numkeys: int, *args): # noqa: ARG002
if "ZCARD" in script and "queued:" in script:
active_key, claim_key, now, max_active, token, ttl, listing_id = args
self.zremrangebyscore(active_key, "-inf", now)
if claim_key in self.store:
return 0
if self.zcard(active_key) >= int(max_active):
return -1
self.store[claim_key] = f"queued:{token}"
self.zsets.setdefault(active_key, {})[str(listing_id)] = float(now) + float(ttl)
return 1
if "~= 'queued:'" in script:
active_key, claim_key, claim_token, owner, ttl, now, listing_id = args
if self.store.get(claim_key) != f"queued:{claim_token}":
return 0
self.store[claim_key] = f"running:{owner}"
self.zsets.setdefault(active_key, {})[str(listing_id)] = float(now) + float(ttl)
return 1
if numkeys == 3 and "return 0" in script:
active_key, claim_key, retry_key, owner, listing_id = args
if self.store.get(claim_key) != f"running:{owner}":
return 0
self.store.pop(claim_key, None)
self.zrem(active_key, str(listing_id))
self.store.pop(retry_key, None)
return 1
if numkeys == 2 and "ARGV[2]" in script:
active_key, claim_key, owner, status, _cooldown, listing_id = args
if self.store.get(claim_key) != f"running:{owner}":
return 0
self.store[claim_key] = str(status)
self.zrem(active_key, str(listing_id))
return 1
raise AssertionError("Unsupported fake Redis Lua script")
def delete(self, *keys: str):
removed = 0
for key in keys:
@@ -90,6 +130,9 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
if key in self.lists:
del self.lists[key]
removed += 1
if key in self.zsets:
del self.zsets[key]
removed += 1
return removed
def expire(self, key: str, ttl: int): # noqa: ARG002
@@ -169,6 +212,97 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(key1, key2)
self.assertEqual(len(key1), 16)
def test_detail_rate_limit_keys_rotate_across_parallel_slots(self) -> None:
redis_client = self._FakeRedis()
keys = [tasks._mobilede_detail_rate_limit_key(redis_client, 2) for _ in range(4)]
self.assertEqual(
keys,
[
f"{tasks.MOBILEDE_DETAIL_RATE_LIMIT_KEY}:0",
f"{tasks.MOBILEDE_DETAIL_RATE_LIMIT_KEY}:1",
f"{tasks.MOBILEDE_DETAIL_RATE_LIMIT_KEY}:0",
f"{tasks.MOBILEDE_DETAIL_RATE_LIMIT_KEY}:1",
],
)
self.assertEqual(
tasks._mobilede_detail_rate_limit_key(redis_client, 1),
tasks.MOBILEDE_DETAIL_RATE_LIMIT_KEY,
)
def test_sold_probe_circuit_breaker_opens_after_configured_403_429_streak(self) -> None:
redis_client = self._FakeRedis()
with patch.object(tasks, "MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT", 3):
self.assertFalse(tasks._mobilede_record_sold_probe_antibot_result(
redis_client,
verdict="blocked",
status_code=403,
))
self.assertFalse(tasks._mobilede_record_sold_probe_antibot_result(
redis_client,
verdict="blocked",
status_code=429,
))
self.assertTrue(tasks._mobilede_record_sold_probe_antibot_result(
redis_client,
verdict="blocked",
status_code=403,
))
self.assertEqual(
redis_client.store[tasks.MOBILEDE_SOLD_PROBE_ANTIBOT_BLOCK_KEY],
"403",
)
self.assertNotIn(tasks.MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_KEY, redis_client.store)
def test_sold_probe_circuit_breaker_resets_streak_after_definitive_result(self) -> None:
redis_client = self._FakeRedis()
tasks._mobilede_record_sold_probe_antibot_result(
redis_client,
verdict="blocked",
status_code=403,
)
tasks._mobilede_record_sold_probe_antibot_result(
redis_client,
verdict="available",
status_code=200,
)
self.assertNotIn(tasks.MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_KEY, redis_client.store)
def test_sold_probe_stops_batch_when_antibot_circuit_breaker_opens(self) -> None:
redis_client = self._FakeRedis()
persistence = MagicMock()
persistence.get_active_cars_batch_for_sold_probe.return_value = [
(index, f"https://example.test/{index}", datetime.now(timezone.utc))
for index in range(1, 6)
]
client = MagicMock()
with (
patch.object(tasks, "_get_redis", return_value=redis_client),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.MobileDeClient, "for_worker", return_value=client),
patch.object(
tasks,
"_mobilede_probe_active_listing_status",
return_value=("blocked", 403),
) as probe,
patch.object(tasks, "MOBILEDE_SOLD_PROBE_ANTIBOT_STREAK_LIMIT", 3),
):
result = tasks.mobilede_verify_active_sold_batch_task.run(limit=5)
self.assertEqual(result["status"], "completed")
self.assertEqual(result["checked"], 3)
self.assertEqual(result["blocked"], 3)
self.assertEqual(result["skipped"], 2)
self.assertEqual(probe.call_count, 3)
persistence.mark_cars_sold_by_ids.assert_not_called()
self.assertIn(tasks.MOBILEDE_SOLD_PROBE_ANTIBOT_BLOCK_KEY, redis_client.store)
def test_runtime_numeric_filters_intersect_planner_segment(self) -> None:
self.assertEqual(
search_sync._intersect_numeric_bounds("10000", "30000", 15000, 25000),
@@ -201,11 +335,9 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
def test_detail_queue_prioritizes_new_unique_listings(self) -> None:
redis_client = self._FakeRedis()
persistence = self._FakeDetailPersistence()
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true", "MOBILEDE_DETAIL_QUEUE_MAX_PENDING": "3"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async") as apply_async,
):
result = tasks._queue_mobilede_detail_listing_ids(
@@ -215,26 +347,21 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
)
self.assertEqual(result, {"queued": 2, "duplicate": 0, "full": 0})
self.assertEqual(redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY], {"100", "101"})
self.assertEqual(redis_client.sets, {})
self.assertEqual(apply_async.call_count, 2)
self.assertTrue(all(call.kwargs["queue"] == tasks.MOBILEDE_IMAGES_QUEUE for call in apply_async.call_args_list))
self.assertTrue(all(call.kwargs["priority"] == 9 for call in apply_async.call_args_list))
self.assertEqual(
[call.kwargs["kwargs"] for call in apply_async.call_args_list],
[
{"listing_id": "100", "lane": "mobile_de_cars", "lease_owner": "lease-100", "attempt": 1},
{"listing_id": "101", "lane": "mobile_de_cars", "lease_owner": "lease-101", "attempt": 1},
],
)
payloads = [call.kwargs["kwargs"] for call in apply_async.call_args_list]
self.assertEqual([payload["listing_id"] for payload in payloads], ["100", "101"])
self.assertTrue(all(payload["claim_token"] for payload in payloads))
self.assertTrue(all(payload["force_refresh"] is False for payload in payloads))
def test_detail_queue_respects_pending_limit(self) -> None:
redis_client = self._FakeRedis()
redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY] = {"100", "101"}
persistence = self._FakeDetailPersistence()
redis_client.zsets[tasks.MOBILEDE_DETAIL_ACTIVE_KEY] = {"100": 9999999999, "101": 9999999999}
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true", "MOBILEDE_DETAIL_QUEUE_MAX_PENDING": "2"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async") as apply_async,
):
result = tasks._queue_mobilede_detail_listing_ids(
@@ -245,15 +372,12 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(result, {"queued": 0, "duplicate": 0, "full": 2})
apply_async.assert_not_called()
def test_detail_queue_skips_recently_completed_listing(self) -> None:
def test_detail_queue_skips_existing_redis_claim(self) -> None:
redis_client = self._FakeRedis()
persistence = self._FakeDetailPersistence()
completed_key = tasks.MOBILEDE_DETAIL_COMPLETED_KEY_FMT.format(listing_id="100")
redis_client.store[completed_key] = "1"
redis_client.store[tasks.MOBILEDE_DETAIL_CLAIM_KEY_FMT.format(listing_id="100")] = "running:task-a"
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async") as apply_async,
):
result = tasks._queue_mobilede_detail_listing_ids(
@@ -262,18 +386,15 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
)
self.assertEqual(result, {"queued": 1, "duplicate": 1, "full": 0})
self.assertEqual(redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY], {"101"})
apply_async.assert_called_once()
def test_explicit_gallery_refresh_invalidates_completed_listing(self) -> None:
redis_client = self._FakeRedis()
persistence = self._FakeDetailPersistence()
completed_key = tasks.MOBILEDE_DETAIL_COMPLETED_KEY_FMT.format(listing_id="100")
redis_client.store[completed_key] = "1"
claim_key = tasks.MOBILEDE_DETAIL_CLAIM_KEY_FMT.format(listing_id="100")
redis_client.store[claim_key] = "exhausted"
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async") as apply_async,
):
result = tasks._queue_mobilede_detail_listing_ids(
@@ -283,24 +404,78 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
)
self.assertEqual(result, {"queued": 1, "duplicate": 0, "full": 0})
self.assertNotIn(completed_key, redis_client.store)
self.assertEqual(redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY], {"100"})
apply_async.assert_called_once()
self.assertTrue(redis_client.store[claim_key].startswith("queued:"))
def test_detail_queue_releases_lease_when_dispatch_fails(self) -> None:
def test_detail_queue_preserves_redis_claim_when_dispatch_is_ambiguous(self) -> None:
redis_client = self._FakeRedis()
persistence = self._FakeDetailPersistence()
with (
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_ENABLED": "true"}),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks.mobilede_sync_detail_task, "apply_async", side_effect=RuntimeError("broker down")),
self.assertRaisesRegex(RuntimeError, "broker down"),
):
tasks._queue_mobilede_detail_listing_ids(redis_client, ["mobile.de:100"])
self.assertEqual(persistence.released, [("100", "lease-100")])
self.assertNotIn("100", redis_client.sets[tasks.MOBILEDE_DETAIL_PENDING_SET_KEY])
self.assertIn(tasks.MOBILEDE_DETAIL_CLAIM_KEY_FMT.format(listing_id="100"), redis_client.store)
self.assertEqual(redis_client.zcard(tasks.MOBILEDE_DETAIL_ACTIVE_KEY), 1)
def test_detail_claim_fences_duplicate_delivery_and_finishes_by_owner(self) -> None:
redis_client = self._FakeRedis()
claim_token = tasks._mobilede_claim_detail_dispatch(redis_client, "100", max_active=10)
self.assertTrue(claim_token)
self.assertTrue(tasks._mobilede_start_detail_execution(
redis_client,
"100",
claim_token=str(claim_token),
execution_owner="task-a",
))
self.assertFalse(tasks._mobilede_start_detail_execution(
redis_client,
"100",
claim_token=str(claim_token),
execution_owner="task-b",
))
self.assertFalse(tasks._mobilede_finish_detail_claim(
redis_client,
"100",
execution_owner="task-b",
))
self.assertTrue(tasks._mobilede_finish_detail_claim(
redis_client,
"100",
execution_owner="task-a",
))
self.assertEqual(redis_client.zcard(tasks.MOBILEDE_DETAIL_ACTIVE_KEY), 0)
def test_stale_detail_owner_cannot_clear_current_cooldown_claim(self) -> None:
redis_client = self._FakeRedis()
claim_token = tasks._mobilede_claim_detail_dispatch(redis_client, "100", max_active=10)
self.assertTrue(claim_token)
self.assertTrue(tasks._mobilede_start_detail_execution(
redis_client,
"100",
claim_token=str(claim_token),
execution_owner="current-task",
))
tasks._mobilede_cooldown_detail_claim(
redis_client,
"100",
execution_owner="stale-task",
status="retry",
cooldown_seconds=60,
)
claim_key = tasks.MOBILEDE_DETAIL_CLAIM_KEY_FMT.format(listing_id="100")
self.assertEqual(redis_client.store[claim_key], "running:current-task")
self.assertEqual(redis_client.zcard(tasks.MOBILEDE_DETAIL_ACTIVE_KEY), 1)
def test_detail_claim_ttl_is_short_and_configurable(self) -> None:
with patch.dict("os.environ", {"MOBILEDE_DETAIL_TIMEOUT_SECONDS": "10"}, clear=False):
self.assertEqual(tasks._mobilede_detail_claim_ttl_seconds(), 300)
with patch.dict("os.environ", {"MOBILEDE_DETAIL_CLAIM_TTL_SECONDS": "420"}, clear=False):
self.assertEqual(tasks._mobilede_detail_claim_ttl_seconds(), 420)
def test_only_new_segment_with_updates_only_enters_cooldown(self) -> None:
redis_client = self._FakeRedis()
@@ -467,16 +642,20 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(finalized[0]["total_results"], 900)
self.assertEqual(finalized[1]["make_id"], "20100")
def test_complete_runtime_plan_rejects_unknown_and_dense_leaves(self) -> None:
with self.assertRaisesRegex(RuntimeError, "unknown=1 dense=1"):
def test_complete_runtime_plan_rejects_unknown_dense_and_under_capacity_leaves(self) -> None:
with self.assertRaisesRegex(RuntimeError, "unknown=1 dense=1 under_capacity=2"):
planner._mobilede_assert_complete_runtime_plan(
[
{"label": "unknown", "total_results": None},
{"label": "dense", "total_results": 1001},
{"label": "valid", "total_results": 950},
{"label": "unknown", "total_results": None, "max_pages": 50},
{"label": "dense", "total_results": 1001, "max_pages": 50},
{"label": "under capacity", "total_results": 950, "max_pages": 12},
]
)
def test_segment_pages_cover_known_total_even_when_parent_was_shorter(self) -> None:
self.assertEqual(planner._mobilede_segment_pages_for_total(611, 12), 31)
self.assertEqual(planner._mobilede_segment_pages_for_total(944, 12), 48)
def test_complete_dense_refine_ignores_soft_probe_limit(self) -> None:
parent = {
"label": "BMW dense",
@@ -512,6 +691,23 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual([item["total_results"] for item in refined], [800, 800, 800])
planner._mobilede_assert_complete_runtime_plan(refined)
def test_partial_transport_failure_does_not_reject_overflow_split(self) -> None:
parent = {
"label": "Lexus dense",
"search_url": "https://www.mobile.de/ru/search.html?ms=13200&p=40001:50000",
"make_id": "13200",
"total_results": 2452,
"max_pages": 50,
}
children = [
{**parent, "label": "unresolved child", "total_results": None},
{**parent, "label": "tiny child", "total_results": 1},
]
self.assertTrue(
planner._mobilede_overflow_candidate_group_is_useful(parent, "mileage", children)
)
def test_dense_refine_recursively_completes_tree_in_one_call(self) -> None:
parent = {
"label": "BMW parent",
@@ -708,7 +904,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(finalized)
redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
def test_force_full_scan_only_new_stays_full_pass_for_continuous_hourly_cycle(self) -> None:
redis_client = self._FakeRedis()