Add checkpoint logic and tests

tests/test_listing.py

@@ -11,6 +11,7 @@ class _FakePage:
     def __init__(self, counts: dict[str, int]) -> None:
         self._counts = counts
         self._evaluate_result = False
+        self._evaluate_values: list[object] = []
 
     class _Locator:
         def __init__(self, count_value: int) -> None:
@@ -23,6 +24,8 @@ class _FakePage:
         return _FakePage._Locator(self._counts.get(selector, 0))
 
     def evaluate(self, _script: str):
+        if self._evaluate_values:
+            return self._evaluate_values.pop(0)
         return self._evaluate_result
 
 
@@ -40,6 +43,11 @@ class TestListingUnit(unittest.TestCase):
         page._evaluate_result = True
         self.assertTrue(ListingCollector._has_next_page(page))
 
+    def test_get_current_page_number_from_text_counter(self) -> None:
+        page = _FakePage({})
+        page._evaluate_values = [5]
+        self.assertEqual(ListingCollector._get_current_page_number(page), 5)
+
     def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
         collector = ListingCollector(Settings(), HumanPacer(Settings()))
         html = """

tests/test_scraper.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
 
 import unittest
+from types import SimpleNamespace
 from unittest.mock import MagicMock
 
 from iaai_scraper.core.config import Settings
@@ -169,6 +170,50 @@ class TestScraperSync(unittest.TestCase):
         self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
         self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
 
+    def test_recover_empty_listing_page_reload_recovers_links(self) -> None:
+        scraper = self._make_scraper()
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
+        )
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+
+        all_raw_urls: list[str] = []
+        seen_urls: set[str] = set()
+
+        recovered_result, recovered_urls = scraper._recover_empty_listing_page(
+            page,
+            page_number=5,
+            all_raw_urls=all_raw_urls,
+            seen_urls=seen_urls,
+        )
+
+        page.reload.assert_called_once()
+        self.assertIs(recovered_result, page_result)
+        self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/123~US"])
+        self.assertEqual(all_raw_urls, ["https://www.iaai.com/VehicleDetail/123~US"])
+
+    def test_recover_empty_listing_page_reopens_listing_for_first_page(self) -> None:
+        scraper = self._make_scraper()
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/456~US")],
+        )
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+
+        recovered_result, recovered_urls = scraper._recover_empty_listing_page(
+            page,
+            page_number=1,
+            all_raw_urls=[],
+            seen_urls=set(),
+        )
+
+        scraper.listing_collector.open_cars_listing.assert_called_once_with(page)
+        page.reload.assert_not_called()
+        self.assertIs(recovered_result, page_result)
+        self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/456~US"])
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_worker_tasks.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import unittest
 from unittest.mock import MagicMock, patch
 
@@ -72,6 +73,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             persistence = MagicMock()
             get_persistence.return_value = persistence
             redis_client = MagicMock()
+            redis_client.get.return_value = None
             get_redis.return_value = redis_client
             stop_event = MagicMock()
             heartbeat_thread = MagicMock()
@@ -140,6 +142,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             persistence = MagicMock()
             get_persistence.return_value = persistence
             redis_client = MagicMock()
+            redis_client.get.return_value = None
             get_redis.return_value = redis_client
             stop_event = MagicMock()
             heartbeat_thread = MagicMock()
@@ -156,6 +159,78 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         self.assertEqual(acquire_lock.call_count, 2)
         release_lock.assert_called_once()
 
+    def test_sync_listing_checkpoint_roundtrip(self) -> None:
+        redis_client = MagicMock()
+        storage: dict[str, str] = {}
+        redis_client.set.side_effect = lambda key, value: storage.__setitem__(key, value)
+        redis_client.get.side_effect = lambda key: storage.get(key)
+
+        tasks._save_sync_checkpoint(
+            redis_client,
+            task_id="task-1",
+            page_number=12,
+            make=None,
+            model=None,
+            lane="iaai_cars",
+        )
+
+        checkpoint = tasks._load_sync_checkpoint(redis_client)
+
+        self.assertIsNotNone(checkpoint)
+        self.assertEqual(checkpoint["last_successful_page"], 12)
+        self.assertEqual(checkpoint["status"], "in_progress")
+
+    def test_sync_listing_task_resumes_from_checkpoint_page(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: json.dumps({
+                "status": "in_progress",
+                "last_successful_page": 9,
+                "make": None,
+                "model": None,
+                "lane": "iaai_cars",
+            }) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 11,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-789")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 10)
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
 
 if __name__ == "__main__":
     unittest.main()