Add checkpoint logic and tests

tests/test_worker_tasks.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import unittest
 from unittest.mock import MagicMock, patch
 
@@ -72,6 +73,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             persistence = MagicMock()
             get_persistence.return_value = persistence
             redis_client = MagicMock()
+            redis_client.get.return_value = None
             get_redis.return_value = redis_client
             stop_event = MagicMock()
             heartbeat_thread = MagicMock()
@@ -140,6 +142,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             persistence = MagicMock()
             get_persistence.return_value = persistence
             redis_client = MagicMock()
+            redis_client.get.return_value = None
             get_redis.return_value = redis_client
             stop_event = MagicMock()
             heartbeat_thread = MagicMock()
@@ -156,6 +159,78 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         self.assertEqual(acquire_lock.call_count, 2)
         release_lock.assert_called_once()
 
+    def test_sync_listing_checkpoint_roundtrip(self) -> None:
+        redis_client = MagicMock()
+        storage: dict[str, str] = {}
+        redis_client.set.side_effect = lambda key, value: storage.__setitem__(key, value)
+        redis_client.get.side_effect = lambda key: storage.get(key)
+
+        tasks._save_sync_checkpoint(
+            redis_client,
+            task_id="task-1",
+            page_number=12,
+            make=None,
+            model=None,
+            lane="iaai_cars",
+        )
+
+        checkpoint = tasks._load_sync_checkpoint(redis_client)
+
+        self.assertIsNotNone(checkpoint)
+        self.assertEqual(checkpoint["last_successful_page"], 12)
+        self.assertEqual(checkpoint["status"], "in_progress")
+
+    def test_sync_listing_task_resumes_from_checkpoint_page(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()):
+            persistence = MagicMock()
+            get_persistence.return_value = persistence
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: json.dumps({
+                "status": "in_progress",
+                "last_successful_page": 9,
+                "make": None,
+                "model": None,
+                "lane": "iaai_cars",
+            }) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            get_redis.return_value = redis_client
+            stop_event = MagicMock()
+            heartbeat_thread = MagicMock()
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
+
+            sync_listing_mock = MagicMock(return_value={
+                "run_id": 11,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-789")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 10)
+        clear_checkpoint.assert_called()
+        release_lock.assert_called_once()
+
 
 if __name__ == "__main__":
     unittest.main()