Stabilize worker sync flow
This commit is contained in:
qananasikq
@@ -150,7 +150,11 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
# Roundtrip: save → load.
|
||||
redis_client = MagicMock()
|
||||
storage: dict[str, str] = {}
|
||||
redis_client.set.side_effect = lambda key, value: storage.__setitem__(key, value)
|
||||
def _fake_set(key, value, ex=None):
|
||||
storage[key] = value
|
||||
return True
|
||||
|
||||
redis_client.set.side_effect = _fake_set
|
||||
redis_client.get.side_effect = lambda key: storage.get(key)
|
||||
|
||||
tasks._save_sync_checkpoint(
|
||||
@@ -165,7 +169,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
@@ -204,6 +208,311 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
clear_checkpoint.assert_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: json.dumps({
|
||||
"status": "in_progress", "last_successful_page": 25,
|
||||
"make": None, "model": None, "lane": "iaai_cars",
|
||||
}) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
get_redis.return_value = redis_client
|
||||
stop_event = MagicMock()
|
||||
heartbeat_thread = MagicMock()
|
||||
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
||||
|
||||
sync_listing_mock = MagicMock(return_value={
|
||||
"run_id": 14, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-792")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 1)
|
||||
self.assertIsNone(sync_listing_mock.call_args.kwargs["progress_callback"])
|
||||
clear_checkpoint.assert_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_checkpoint_zero_page_restarts_from_first_page(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: json.dumps({
|
||||
"status": "in_progress", "last_successful_page": 0,
|
||||
"make": None, "model": None, "lane": "iaai_cars",
|
||||
}) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
get_redis.return_value = redis_client
|
||||
stop_event = MagicMock()
|
||||
heartbeat_thread = MagicMock()
|
||||
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
||||
|
||||
sync_listing_mock = MagicMock(return_value={
|
||||
"run_id": 12, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-790")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 1)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_save_sync_checkpoint_sets_ttl(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
|
||||
tasks._save_sync_checkpoint(
|
||||
redis_client,
|
||||
task_id="task-1",
|
||||
page_number=3,
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
)
|
||||
|
||||
redis_client.set.assert_called_once()
|
||||
self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
|
||||
|
||||
def test_load_sync_checkpoint_clears_invalid_payload(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = "{broken-json"
|
||||
|
||||
checkpoint = tasks._load_sync_checkpoint(redis_client)
|
||||
|
||||
self.assertIsNone(checkpoint)
|
||||
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
|
||||
|
||||
def test_sync_listing_task_clears_checkpoint_on_partial_success_when_full_scan_completed(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 13,
|
||||
"status": "partial_success",
|
||||
"full_scan_completed": True,
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 1,
|
||||
"images_upserted": 3,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 4.0,
|
||||
"failures": [{"vehicle_url": "v", "error": "e"}],
|
||||
}), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
stop_event = MagicMock()
|
||||
heartbeat_thread = MagicMock()
|
||||
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-791")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(make="Toyota")
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "partial_success")
|
||||
clear_checkpoint.assert_called_once_with(redis_client)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_try_set_followup_pending_deduplicates(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.set.side_effect = [True, False]
|
||||
|
||||
self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
|
||||
self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
|
||||
|
||||
def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
|
||||
|
||||
streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
|
||||
redis_client,
|
||||
reason="max_retries_exceeded",
|
||||
)
|
||||
|
||||
self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
|
||||
self.assertFalse(should_enqueue)
|
||||
redis_client.expire.assert_called_once_with(
|
||||
tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
|
||||
tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
|
||||
)
|
||||
|
||||
def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.incr.return_value = 1
|
||||
|
||||
streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
|
||||
redis_client,
|
||||
reason="bootstrap_not_completed",
|
||||
)
|
||||
|
||||
self.assertEqual(streak, 1)
|
||||
self.assertTrue(should_enqueue)
|
||||
|
||||
def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
|
||||
patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
|
||||
patch.object(tasks, "_clear_followup_pending") as clear_pending, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 99,
|
||||
"status": "failed",
|
||||
"full_scan_completed": False,
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [{"vehicle_url": "listing", "error": "bad resume"}],
|
||||
"listing": {"vehicles_collected": 0},
|
||||
}):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
stop_event = MagicMock()
|
||||
heartbeat_thread = MagicMock()
|
||||
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
||||
|
||||
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
|
||||
tasks.sync_listing_task.push_request(id="task-900")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "failed")
|
||||
bump_streak.assert_called_once()
|
||||
clear_pending.assert_called()
|
||||
task_app.send_task.assert_not_called()
|
||||
|
||||
def test_bump_checkpoint_resume_failure_deletes_after_second_failure(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
checkpoint = {
|
||||
"status": "in_progress",
|
||||
"last_successful_page": 9,
|
||||
"resume_failures": 1,
|
||||
"lane": "iaai_cars",
|
||||
}
|
||||
|
||||
failures, deleted = tasks._bump_checkpoint_resume_failure(
|
||||
redis_client,
|
||||
checkpoint,
|
||||
reason="resume failed",
|
||||
)
|
||||
|
||||
self.assertEqual(failures, 2)
|
||||
self.assertTrue(deleted)
|
||||
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
|
||||
|
||||
def test_bump_checkpoint_resume_failure_persists_first_failure(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
checkpoint = {
|
||||
"status": "in_progress",
|
||||
"last_successful_page": 9,
|
||||
"resume_failures": 0,
|
||||
"lane": "iaai_cars",
|
||||
}
|
||||
|
||||
failures, deleted = tasks._bump_checkpoint_resume_failure(
|
||||
redis_client,
|
||||
checkpoint,
|
||||
reason="resume failed",
|
||||
)
|
||||
|
||||
self.assertEqual(failures, 1)
|
||||
self.assertFalse(deleted)
|
||||
redis_client.set.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_deletes_checkpoint_after_second_resume_failure(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_bump_checkpoint_resume_failure", return_value=(2, True)) as bump_failures, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks.sync_listing_task, "retry", side_effect=tasks.sync_listing_task.MaxRetriesExceededError()), \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: json.dumps({
|
||||
"status": "in_progress",
|
||||
"last_successful_page": 9,
|
||||
"resume_failures": 1,
|
||||
"make": None,
|
||||
"model": None,
|
||||
"lane": "iaai_cars",
|
||||
}) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
get_redis.return_value = redis_client
|
||||
stop_event = MagicMock()
|
||||
heartbeat_thread = MagicMock()
|
||||
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
||||
|
||||
with patch.object(tasks, "IAAIScraper") as scraper_cls:
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing = MagicMock(side_effect=RuntimeError("Failed to resume listing at page 10"))
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
scraper_cls.return_value = scraper_ctx
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-793")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "failed")
|
||||
bump_failures.assert_called_once()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user