from __future__ import annotations import json from dataclasses import replace import unittest from unittest.mock import MagicMock, patch from iaai_scraper.worker import tasks from iaai_scraper.core.config import settings as base_settings class TestWorkerTaskLockHelpers(unittest.TestCase): def test_sync_vehicle_task_skips_empty_vehicle_url(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence: get_persistence.return_value = MagicMock() tasks.sync_vehicle_task.push_request(id="task-empty-vehicle") try: result = tasks.sync_vehicle_task.run(vehicle_url=" ", lane=" ") finally: tasks.sync_vehicle_task.pop_request() self.assertEqual(result["status"], "skipped") self.assertEqual(result["reason"], "empty_vehicle_url") get_persistence.assert_not_called() def test_lock_acquire_refresh_release(self) -> None: redis_client = MagicMock() # Acquire. redis_client.set.return_value = True self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)) redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) # Refresh. redis_client.eval.return_value = 1 self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)) # Release. redis_client.eval.reset_mock() tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") args = redis_client.eval.call_args[0] self.assertEqual(args[2], "lock:key") self.assertEqual(args[3], "owner-token") def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=False): persistence = MagicMock() get_persistence.return_value = persistence get_redis.return_value = MagicMock() tasks.sync_listing_task.push_request(id="task-123") try: result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() persistence.create_tables.assert_called_once() self.assertEqual(result["status"], "skipped") self.assertEqual(result["reason"], "sync_already_running") def test_sync_listing_task_releases_owned_lock(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 7, "cars_upserted": 2, "cars_failed": 0, "images_upserted": 4, "skipped_existing": 1, "elapsed_seconds": 1.25, }): persistence = MagicMock() get_persistence.return_value = persistence redis_client = MagicMock() redis_client.get.return_value = None get_redis.return_value = redis_client stop_event = MagicMock() heartbeat_thread = MagicMock() start_heartbeat.return_value = (stop_event, heartbeat_thread) tasks.sync_listing_task.push_request(id="task-123") try: result = tasks.sync_listing_task.run(make="Toyota") finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") stop_event.set.assert_called_once() heartbeat_thread.join.assert_called_once() release_lock.assert_called_once() def test_clear_orphan_sync_listing_lock(self) -> None: # Нет запущенных задач → удаляет. redis_client = MagicMock() redis_client.get.return_value = "owner-token" redis_client.ttl.return_value = 120 celery_app = MagicMock() inspector = MagicMock() inspector.active.return_value = {"worker@node": []} inspector.reserved.return_value = {"worker@node": []} inspector.scheduled.return_value = {"worker@node": []} celery_app.control.inspect.return_value = inspector self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)) redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY) # Задача активна → не удаляет. redis_client2 = MagicMock() redis_client2.get.return_value = "owner-token" inspector2 = MagicMock() inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]} inspector2.reserved.return_value = {"worker@node": []} inspector2.scheduled.return_value = {"worker@node": []} celery_app2 = MagicMock() celery_app2.control.inspect.return_value = inspector2 self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2)) redis_client2.delete.assert_not_called() def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \ patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 9, "cars_upserted": 3, "cars_failed": 0, "images_upserted": 5, "skipped_existing": 0, "elapsed_seconds": 2.0, }): persistence = MagicMock() get_persistence.return_value = persistence redis_client = MagicMock() redis_client.get.return_value = None get_redis.return_value = redis_client stop_event = MagicMock() heartbeat_thread = MagicMock() start_heartbeat.return_value = (stop_event, heartbeat_thread) tasks.sync_listing_task.push_request(id="task-456") try: result = tasks.sync_listing_task.run(make="Honda") finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") clear_orphan.assert_called_once() self.assertEqual(acquire_lock.call_count, 2) release_lock.assert_called_once() def test_checkpoint_save_load_roundtrip(self) -> None: storage: dict[str, str] = {} def _fake_set(key, value, ex=None): storage[key] = value return True redis_client = MagicMock() redis_client.set.side_effect = _fake_set redis_client.get.side_effect = lambda key: storage.get(key) tasks._save_last_completed_segment(redis_client, 12) self.assertEqual(tasks._load_last_completed_segment(redis_client), 12) self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS) def test_load_checkpoint_clears_invalid_payload(self) -> None: redis_client = MagicMock() redis_client.get.return_value = "{not-a-number" self.assertIsNone(tasks._load_last_completed_segment(redis_client)) redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY) def test_load_checkpoint_empty_when_missing(self) -> None: redis_client = MagicMock() redis_client.get.return_value = None self.assertIsNone(tasks._load_last_completed_segment(redis_client)) redis_client.delete.assert_not_called() def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None: segments = [ {"make": "ACURA"}, {"make": "AUDI"}, {"make": "BMW"}, {"make": "EAGLE"}, ] with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.side_effect = lambda key: ( "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None ) get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) sync_segmented_mock = MagicMock(return_value={ "run_id": 11, "status": "success", "full_scan_completed": True, "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], }) scraper_ctx = MagicMock() scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__exit__.return_value = None with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-resume-seg") try: result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") # last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW). self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2) self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) release_lock.assert_called_once() def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None: segments = [{"make": "ACURA"}, {"make": "AUDI"}] with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object( tasks, "Settings", return_value=replace( base_settings, discovery=replace( base_settings.discovery, always_full_scan=False, mode="listing", ), celery=replace( base_settings.celery, parallel_segments=False, ), ), ), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() # Оставшийся чекпоинт не должен использоваться. redis_client.get.side_effect = lambda key: ( "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None ) get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) sync_segmented_mock = MagicMock(return_value={ "run_id": 14, "status": "success", "full_scan_completed": True, "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], }) scraper_ctx = MagicMock() scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__exit__.return_value = None with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-792") try: result = tasks.sync_listing_task.run(limit=1) finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"]) clear_checkpoint.assert_called() release_lock.assert_called_once() def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None: segments = [{"make": "ACURA"}, {"make": "AUDI"}] with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.side_effect = lambda key: ( "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None ) get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) sync_segmented_mock = MagicMock(return_value={ "run_id": 15, "status": "success", "full_scan_completed": True, "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], }) scraper_ctx = MagicMock() scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__exit__.return_value = None with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-beyond") try: result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0) release_lock.assert_called_once() def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 13, "status": "partial_success", "full_scan_completed": True, "cars_upserted": 2, "cars_failed": 1, "images_upserted": 3, "skipped_existing": 0, "elapsed_seconds": 4.0, "failures": [{"vehicle_url": "v", "error": "e"}], }), \ patch.object(tasks.sync_listing_task, "update_state"): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) tasks.sync_listing_task.push_request(id="task-791") try: result = tasks.sync_listing_task.run(make="Toyota") finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "partial_success") # Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт # до браузерного job + после. Главное — вызов произошёл. clear_checkpoint.assert_called() release_lock.assert_called_once() def test_try_set_followup_pending_deduplicates(self) -> None: redis_client = MagicMock() redis_client.set.side_effect = [True, False] self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120)) def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None: redis_client = MagicMock() redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT streak, should_enqueue = tasks._bump_bootstrap_failure_streak( redis_client, reason="max_retries_exceeded", ) self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT) self.assertFalse(should_enqueue) redis_client.expire.assert_called_once_with( tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS, ) def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None: redis_client = MagicMock() redis_client.incr.return_value = 1 streak, should_enqueue = tasks._bump_bootstrap_failure_streak( redis_client, reason="bootstrap_not_completed", ) self.assertEqual(streak, 1) self.assertTrue(should_enqueue) def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_try_set_followup_pending", return_value=True), \ patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \ patch.object(tasks, "_clear_followup_pending") as clear_pending, \ patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 99, "status": "failed", "full_scan_completed": False, "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [{"vehicle_url": "listing", "error": "bad resume"}], "listing": {"vehicles_collected": 0}, }): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: tasks.sync_listing_task.push_request(id="task-900") try: result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "failed") bump_streak.assert_called_once() clear_pending.assert_called() task_app.send_task.assert_not_called() def test_sync_listing_task_stops_bootstrap_followup_after_no_progress_limit(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ patch.object(tasks, "_release_lock_if_owner"), \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 101, "status": "success", "full_scan_completed": False, "cars_upserted": 0, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "total_discovered": 0, "listing": {"vehicles_collected": 0}, "failures": [], }), \ patch.object(tasks, "_bump_bootstrap_no_progress_streak", return_value=( tasks.SYNC_LISTING_BOOTSTRAP_NO_PROGRESS_STREAK_LIMIT, False, )), \ patch.object(tasks, "_set_full_scan_done") as set_done, \ patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ patch.object(tasks.sync_listing_task, "update_state"): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None get_redis.return_value = redis_client with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: tasks.sync_listing_task.push_request(id="task-no-progress") try: result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "failed") set_done.assert_called() clear_checkpoint.assert_called() task_app.send_task.assert_not_called() def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ patch.object(tasks, "_release_lock_if_owner"), \ patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \ patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \ patch.object(tasks.sync_listing_task, "update_state"): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None get_redis.return_value = redis_client with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: tasks.sync_listing_task.push_request(id="task-soft-timeout") try: result = tasks.sync_listing_task.run(make="Toyota") finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "timed_out") set_pending.assert_called_once() task_app.send_task.assert_not_called() def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \ patch.object(tasks, "_release_lock_if_owner"), \ patch.object(tasks, "_try_set_followup_pending", return_value=True), \ patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \ patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 101, "status": "partial_success", "full_scan_completed": False, "cars_upserted": 2, "cars_failed": 0, "images_upserted": 1, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], "listing": {"vehicles_collected": 2}, }): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None get_redis.return_value = redis_client with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app: tasks.sync_listing_task.push_request(id="task-breaker-stop") try: result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "partial_success") # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly. self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list)) self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list)) clear_checkpoint.assert_called_once() task_app.send_task.assert_not_called() def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object( tasks, "Settings", return_value=replace( base_settings, discovery=replace(base_settings.discovery, always_full_scan=True), ), ), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks, "_run_browser_job") as run_job, \ patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) run_job.return_value = { "run_id": 17, "status": "success", "full_scan_completed": True, "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], } tasks.sync_listing_task.push_request(id="task-always-full") try: result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") set_full_scan_done.assert_called_with(redis_client, False) release_lock.assert_called_once() def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None: segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}] with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object( tasks, "Settings", return_value=replace( base_settings, discovery=replace(base_settings.discovery, always_full_scan=True), ), ), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.side_effect = lambda key: ( "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None ) get_redis.return_value = redis_client start_heartbeat.return_value = (MagicMock(), MagicMock()) sync_segmented_mock = MagicMock(return_value={ "run_id": 18, "status": "success", "full_scan_completed": True, "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], }) scraper_ctx = MagicMock() scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__enter__.return_value.sync_listing = MagicMock() scraper_ctx.__exit__.return_value = None with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-always-full-resume") try: result = tasks.sync_listing_task.run() finally: tasks.sync_listing_task.pop_request() self.assertEqual(result["status"], "success") self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1) self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) release_lock.assert_called_once() if __name__ == "__main__": unittest.main()