Stabilize worker sync flow

tests/test_scraper.py

@@ -5,7 +5,7 @@ from types import SimpleNamespace
 from unittest.mock import MagicMock
 
 from iaai_scraper.core.config import Settings
-from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from iaai_scraper.core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError
 from iaai_scraper.scraper import IAAIScraper
 from iaai_scraper.storage.schemas import CarRecord
 
@@ -121,6 +121,41 @@ class TestScraperSync(unittest.TestCase):
         self.assertIn("FORD", call_args_log[0]["listing_url"])
         self.assertIn("HONDA", call_args_log[1]["listing_url"])
 
+    def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None:
+        scraper = self._make_scraper()
+        scraper.sync_listing = MagicMock(side_effect=[
+            {
+                "status": "failed",
+                "full_scan_completed": False,
+                "cars_upserted": 0,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 0},
+                "failures": [{"vehicle_url": "segment", "error": "resume failed"}],
+            },
+            {
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "listing": {"vehicles_collected": 10},
+                "failures": [],
+            },
+        ])
+
+        result = scraper.sync_listing_segmented(
+            segments=[
+                {"make": "EAGLE", "year_min": None, "year_max": None},
+                {"make": "FORD", "year_min": None, "year_max": None},
+            ]
+        )
+
+        self.assertFalse(result["full_scan_completed"])
+        self.assertEqual(result["status"], "partial_success")
+
     def test_build_segment_listing_url(self) -> None:
         base = "https://www.iaai.com/Vehiclelisting/Cars"
         self.assertEqual(
@@ -188,6 +223,55 @@ class TestScraperSync(unittest.TestCase):
         scraper.listing_collector.open_cars_listing.assert_called_once()
         page.reload.assert_not_called()
 
+    def test_sync_listing_resets_stale_resume_checkpoint_and_restarts_from_page_one(self) -> None:
+        scraper = self._make_scraper()
+        scraper.settings.celery.batch_size = 1
+
+        page = MagicMock()
+        page_result = SimpleNamespace(
+            page_number=1,
+            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            next_page_detected=False,
+        )
+
+        progress_calls: list[int] = []
+
+        scraper._get_page_with_warmup = MagicMock(return_value=page)
+        scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("checkpoint page is no longer reachable"))
+        scraper.listing_collector.open_cars_listing = MagicMock()
+        scraper.listing_collector.apply_filters = MagicMock(return_value={
+            "make": None,
+            "model": None,
+            "year_min": None,
+            "year_max": None,
+        })
+        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1,
+            "cars_failed": 0,
+            "images_upserted": 0,
+            "failures": [],
+        })
+
+        result = scraper._sync_listing_streaming(
+            make=None,
+            model=None,
+            lane="iaai_cars",
+            limit=None,
+            effective_only_new=False,
+            started_at=0.0,
+            start_page=50,
+            progress_callback=progress_calls.append,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
+        )
+
+        self.assertEqual(progress_calls, [0, 1])
+        scraper.listing_collector.open_cars_listing.assert_called_once()
+        scraper.sync_batch.assert_called_once()
+        self.assertEqual(result["cars_upserted"], 1)
+        self.assertEqual(result["total"], 1)
+
 
 if __name__ == "__main__":
     unittest.main()