tune vps config and update tests

.env.example

@@ -0,0 +1,57 @@
+IAAI_HEADLESS=true
+IAAI_LOG_LEVEL=INFO
+
+IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
+IAAI_MAX_CAPTURED_REQUESTS=40
+IAAI_MAX_CAPTURED_JSON_RESPONSES=20
+
+IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
+IAAI_LISTING_SEGMENTS=auto
+IAAI_MAX_PAGES_PER_RUN=999999
+IAAI_MAX_VEHICLES_PER_RUN=999999
+IAAI_PAGE_LINK_LIMIT=999999
+IAAI_INCLUDE_PAGINATION=true
+IAAI_COLLECT_CURRENT_PAGE_ONLY=false
+
+IAAI_HUMAN_PACE_ENABLED=true
+IAAI_PARALLEL_TABS=10
+CELERY_BATCH_SIZE=100
+IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
+IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
+IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
+IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
+IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
+IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
+IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
+IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
+IAAI_BETWEEN_VEHICLES_MIN_S=0.02
+IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
+
+IAAI_SYNC_ONLY_NEW=false
+IAAI_TOKENS_FILE=/data/tokens.json
+IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
+
+IAAI_MAX_RETRIES=5
+IAAI_RETRY_DELAY_SECONDS=4
+IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
+IAAI_RETRY_JITTER_SECONDS=0.5
+IAAI_TIMEOUT_MS=90000
+IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
+
+IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+IAAI_DATABASE_ECHO=false
+IAAI_DATABASE_POOL_SIZE=5
+IAAI_DATABASE_MAX_OVERFLOW=5
+
+IAAI_REDIS_URL=redis://redis:6379/0
+
+CELERY_BROKER_URL=redis://redis:6379/0
+CELERY_RESULT_BACKEND=redis://redis:6379/0
+CELERY_TASK_SOFT_TIME_LIMIT=86400
+CELERY_TASK_TIME_LIMIT=86520
+CELERY_BROKER_VISIBILITY_TIMEOUT=90000
+CELERY_WORKER_CONCURRENCY=1
+CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
+CELERY_BEAT_SYNC_LIMIT=0

.gitignore

@@ -3,8 +3,7 @@ __pycache__/
 .coverage
 coverage.xml
 .env*
-!.env
+!.env.example
-.env.example
 .venv/
 venv/
 *.pyc

tests/test_listing.py

@@ -30,23 +30,19 @@ class _FakePage:
 
 
 class TestListingUnit(unittest.TestCase):
-    def test_has_next_page_true_for_known_selector(self) -> None:
+    def test_pagination_detection_and_page_number(self) -> None:
-        page = _FakePage({"a[aria-label*='Next']": 1})
+        # Есть кнопка Next → True.
-        self.assertTrue(ListingCollector._has_next_page(page))
+        self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']":  1})))
-
+        # Нет селекторов → False.
-    def test_has_next_page_false_when_no_selectors(self) -> None:
+        self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
-        page = _FakePage({})
+        # Числовая пагинация через JS → True.
-        self.assertFalse(ListingCollector._has_next_page(page))
-
-    def test_has_next_page_true_for_numeric_pagination_fallback(self) -> None:
         page = _FakePage({})
         page._evaluate_result = True
         self.assertTrue(ListingCollector._has_next_page(page))
-
+        # Номер текущей страницы.
-    def test_get_current_page_number_from_text_counter(self) -> None:
+        page2 = _FakePage({})
-        page = _FakePage({})
+        page2._evaluate_values = [5]
-        page._evaluate_values = [5]
+        self.assertEqual(ListingCollector._get_current_page_number(page2), 5)
-        self.assertEqual(ListingCollector._get_current_page_number(page), 5)
 
     def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
         collector = ListingCollector(Settings(), HumanPacer(Settings()))

tests/test_mappers.py

@@ -41,30 +41,27 @@ class TestCarMapper(unittest.TestCase):
 
         self.assertFalse(record.is_damaged)
 
-    def test_unknown_and_empty_values_fallbacks(self) -> None:
+    def test_unknown_empty_values_and_normalization(self) -> None:
         record = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
             vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
-
         self.assertEqual(record.brand, "UNKNOWN")
         self.assertEqual(record.model, "UNKNOWN")
-        self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
         self.assertEqual(record.drive, "NA")
         self.assertEqual(record.gearbox, "NA")
 
-    def test_mapper_handles_case_and_spaces(self) -> None:
+        # Нормализация регистра и пробелов.
-        record = self.mapper.map_to_car_record(
+        record2 = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
+        self.assertEqual(record2.drive, "FWD")
+        self.assertEqual(record2.gearbox, "AT")
 
-        self.assertEqual(record.drive, "FWD")
+    def test_price_and_currency_parsing(self) -> None:
-        self.assertEqual(record.gearbox, "AT")
-
-    def test_price_parsing_dirty_formats(self) -> None:
         record = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
@@ -76,11 +73,9 @@ class TestCarMapper(unittest.TestCase):
                 "images": {},
             },
         )
-
         self.assertEqual(record.price, 5200)
 
-    def test_currency_detection_from_symbol(self) -> None:
+        record2 = self.mapper.map_to_car_record(
-        record = self.mapper.map_to_car_record(
             vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
             payload_insights={
@@ -91,9 +86,8 @@ class TestCarMapper(unittest.TestCase):
                 "images": {},
             },
         )
-
+        self.assertEqual(record2.currency, "EUR")
-        self.assertEqual(record.currency, "EUR")
+        self.assertEqual(record2.price, 4500)
-        self.assertEqual(record.price, 4500)
 
 
 if __name__ == "__main__":

tests/test_parser.py

@@ -9,7 +9,7 @@ class TestVehicleParserUnit(unittest.TestCase):
     def setUp(self) -> None:
         self.parser = VehicleParser()
 
-    def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
+    def test_parse_dom_pairs_and_title(self) -> None:
         dom_text = """
         Stock #:
         45089484
@@ -23,41 +23,27 @@ class TestVehicleParserUnit(unittest.TestCase):
         self.assertEqual(result.get("primary_damage"), "Front End")
         self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
 
-    def test_parse_title_for_year_make_model(self) -> None:
         parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
         self.assertEqual(parsed["year"], "2014")
         self.assertEqual(parsed["make"], "TOYOTA")
         self.assertEqual(parsed["model"], "CAMRY")
 
-    def test_extract_image_urls_filters_other_vehicle(self) -> None:
+    def test_extract_image_urls_filters_and_deduplicates(self) -> None:
-        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
-        payloads = [
-            {
-                "imageUrls": [
-                    "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-                    "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
-                ]
-            }
-        ]
-        urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
-        self.assertEqual(len(urls), 1)
-        self.assertIn("45089484", urls[0])
-
-    def test_extract_image_urls_deduplicates_same_url(self) -> None:
         vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
         payloads = [{"imageUrls": [
             "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
             "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
         ]}]
         urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
         self.assertEqual(len(urls), 1)
+        self.assertIn("45089484", urls[0])
 
-    def test_dom_hints_detect_captcha_and_antibot(self) -> None:
+    def test_dom_hints_and_access_notes_detect_antibot(self) -> None:
         hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
         self.assertTrue(hints["has_captcha_text"])
         self.assertTrue(hints["has_antibot_text"])
 
-    def test_access_notes_reflect_antibot_signals(self) -> None:
         summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
         notes = self.parser._build_access_notes(summary, [])
         self.assertTrue(notes["possible_captcha"])

tests/test_scraper.py

@@ -28,104 +28,129 @@ class TestScraperSync(unittest.TestCase):
         s.database.url = "sqlite://"
         return IAAIScraper(s)
 
-    def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
+    def test_sync_vehicle_uses_db_record(self) -> None:
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=1)
         scraper.persistence.finish_sync_run = MagicMock()
         scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-
         scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
 
         result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
 
         self.assertEqual(result["status"], "success")
         self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
+        scraper.persistence.upsert_car.assert_called_once()
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
 
-    def test_sync_listing_uses_db_record_without_remapping(self) -> None:
+    def test_only_new_routing_legacy_and_streaming(self) -> None:
+        # Legacy path: only_new без listing_url.
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=2)
         scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set()))
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
-
+            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
-        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
+        ))
-        scraper.sync_batch = MagicMock(return_value={
+        scraper.collect_listing = MagicMock(return_value={
-            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
+            "vehicle_urls": [
+                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.iaai.com/VehicleDetail/333~US",
+            ]
         })
-
+        scraper.sync_batch = MagicMock(return_value={
-        result = scraper.sync_listing()
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
-
+        })
+        result = scraper.sync_listing(only_new=True)
+        self.assertEqual(result["skipped_existing"], 2)
         self.assertEqual(result["cars_upserted"], 1)
-        self.assertEqual(result["cars_failed"], 0)
-        self.assertIn("trace_id", result)
-        self.assertIn("elapsed_seconds", result)
-        scraper.sync_batch.assert_called_once()
 
-    def test_sync_listing_respects_limit(self) -> None:
+        # Streaming path: only_new + listing_url.
+        scraper2 = self._make_scraper()
+        scraper2.persistence.create_tables = MagicMock()
+        scraper2.persistence.start_sync_run = MagicMock(return_value=6)
+        scraper2.persistence.finish_sync_run = MagicMock()
+        scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used"))
+        scraper2._sync_listing_streaming = MagicMock(return_value={
+            "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False},
+            "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0,
+            "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(),
+        })
+        result2 = scraper2.sync_listing(
+            only_new=True,
+            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            year_min=2020, year_max=2027,
+        )
+        self.assertEqual(result2["cars_upserted"], 10)
+        scraper2._sync_listing_streaming.assert_called_once()
+        scraper2.collect_listing.assert_not_called()
+
+    def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
         scraper = self._make_scraper()
-
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=3)
         scraper.persistence.finish_sync_run = MagicMock()
 
-        # Проверка пути only_new с limit.
+        call_args_log: list[dict] = []
-        scraper._collect_listing_iterative = MagicMock(return_value=(
+        def _fake_sync_listing(**kwargs):
-            ["https://www.iaai.com/VehicleDetail/222~US"],
+            call_args_log.append(kwargs)
-            ["https://www.iaai.com/VehicleDetail/222~US",
+            return {
-             "https://www.iaai.com/VehicleDetail/333~US"],
+                "status": "success", "cars_upserted": 5, "cars_failed": 0,
-            {"vehicle_urls": [], "pages_collected": 1, "early_stopped": False},
+                "images_upserted": 10, "skipped_existing": 0,
-            0,
+                "listing": {"vehicles_collected": 50}, "failures": [],
-        ))
+            }
-        scraper.sync_batch = MagicMock(return_value={
-            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
-        })
 
-        scraper.sync_listing(limit=1)
+        scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing)
+        segments = [
+            {"make": "TOYOTA", "year_min": 2020, "year_max": 2027},
+            {"make": "FORD", "year_min": None, "year_max": None},
+            {"make": "HONDA", "year_min": None, "year_max": None},
+        ]
 
-        # Должен уйти только один URL.
+        # Resume: пропускаем TOYOTA, начинаем с FORD на стр. 5.
-        scraper.sync_batch.assert_called_once()
+        result = scraper.sync_listing_segmented(
-        batch_urls = scraper.sync_batch.call_args[0][0]
+            segments=segments, start_segment=1, start_page=5,
-        self.assertEqual(len(batch_urls), 1)
+        )
 
-    def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
+        self.assertEqual(result["segments_completed"], 2)  # FORD + HONDA
-        scraper = self._make_scraper()
+        self.assertEqual(result["cars_upserted"], 10)
+        # FORD: start_page=5, HONDA: start_page=1.
+        self.assertEqual(call_args_log[0]["start_page"], 5)
+        self.assertEqual(call_args_log[1]["start_page"], 1)
+        # URL содержит бренд.
+        self.assertIn("FORD", call_args_log[0]["listing_url"])
+        self.assertIn("HONDA", call_args_log[1]["listing_url"])
 
-        scraper.persistence.create_tables = MagicMock()
+    def test_build_segment_listing_url(self) -> None:
-        scraper.persistence.start_sync_run = MagicMock(return_value=5)
+        base = "https://www.iaai.com/Vehiclelisting/Cars"
-        scraper.persistence.finish_sync_run = MagicMock()
+        self.assertEqual(
-        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
+            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
-            {"https://www.iaai.com/VehicleDetail/111~US"},
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
-            {"iaai:222"},
+        )
-        ))
+        self.assertEqual(
+            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
+            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
+        )
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
+        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
 
-        scraper.collect_listing = MagicMock(return_value={
+    def test_guard_and_protection_detection(self) -> None:
-            "vehicle_urls": [
+        with self.assertRaises(AntiBotDetectedError):
-                "https://www.iaai.com/VehicleDetail/111~US",  # exists by URL
+            IAAIScraper._raise_if_blocked_or_incomplete(
-                "https://www.iaai.com/VehicleDetail/222~US",  # exists by ID
+                {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
-                "https://www.iaai.com/VehicleDetail/333~US",  # new
+                 "access_notes": {}, "vehicle_summary": {}},
-            ]
+                "https://www.iaai.com/VehicleDetail/999~US",
-        })
+            )
-        # Возвращаем результат для одного нового авто.
+        with self.assertRaises(SiteStructureChangedError):
-        scraper.sync_batch = MagicMock(return_value={
+            IAAIScraper._raise_if_blocked_or_incomplete(
-            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
+                {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
-        })
+                 "access_notes": {"possible_captcha": False, "possible_antibot": False},
-
+                 "vehicle_summary": {}},
-        result = scraper.sync_listing(only_new=True)
+                "https://www.iaai.com/VehicleDetail/999~US",
-
+            )
-        self.assertEqual(result["skipped_existing"], 2)
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
-        self.assertEqual(result["cars_upserted"], 1)
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
-        # В batch должен попасть только новый URL.
+        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
-        scraper.sync_batch.assert_called_once()
-        batch_urls = scraper.sync_batch.call_args[0][0]
-        self.assertEqual(len(batch_urls), 1)
-        self.assertIn("333", batch_urls[0])
-        scraper.persistence.get_existing_urls_and_ids.assert_called_once()
 
     def test_close_resets_browser_state(self) -> None:
         scraper = self._make_scraper()
@@ -134,85 +159,34 @@ class TestScraperSync(unittest.TestCase):
         scraper.context = MagicMock()
         scraper.browser = MagicMock()
         scraper.playwright = MagicMock()
-
         scraper.close()
-
         http_pool.clear.assert_called_once()
-        self.assertIsNone(scraper._http_pool)
         self.assertIsNone(scraper.context)
         self.assertIsNone(scraper.browser)
-        self.assertIsNone(scraper.playwright)
 
-    def test_guard_raises_on_antibot_signals(self) -> None:
+    def test_recover_empty_listing_page(self) -> None:
-        with self.assertRaises(AntiBotDetectedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
-                {
-                    "dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
-                    "access_notes": {},
-                    "vehicle_summary": {},
-                },
-                "https://www.iaai.com/VehicleDetail/999~US",
-            )
-
-    def test_guard_raises_on_empty_vehicle_page(self) -> None:
-        with self.assertRaises(SiteStructureChangedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
-                {
-                    "dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
-                    "access_notes": {"possible_captcha": False, "possible_antibot": False},
-                    "vehicle_summary": {},
-                },
-                "https://www.iaai.com/VehicleDetail/999~US",
-            )
-
-    def test_is_protection_or_network_error_detects_known_signals(self) -> None:
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
-        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
-
-    def test_recover_empty_listing_page_reload_recovers_links(self) -> None:
         scraper = self._make_scraper()
         page = MagicMock()
         page_result = SimpleNamespace(
             vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
         )
         scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
-
-        all_raw_urls: list[str] = []
-        seen_urls: set[str] = set()
-
-        recovered_result, recovered_urls = scraper._recover_empty_listing_page(
-            page,
-            page_number=5,
-            all_raw_urls=all_raw_urls,
-            seen_urls=seen_urls,
-        )
-
-        page.reload.assert_called_once()
-        self.assertIs(recovered_result, page_result)
-        self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/123~US"])
-        self.assertEqual(all_raw_urls, ["https://www.iaai.com/VehicleDetail/123~US"])
-
-    def test_recover_empty_listing_page_reopens_listing_for_first_page(self) -> None:
-        scraper = self._make_scraper()
-        page = MagicMock()
-        page_result = SimpleNamespace(
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/456~US")],
-        )
         scraper.listing_collector.open_cars_listing = MagicMock()
-        scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
 
-        recovered_result, recovered_urls = scraper._recover_empty_listing_page(
+        # Страница > 1: reload.
-            page,
+        _, urls = scraper._recover_empty_listing_page(
-            page_number=1,
+            page, page_number=5, all_raw_urls=[], seen_urls=set(),
-            all_raw_urls=[],
-            seen_urls=set(),
         )
+        page.reload.assert_called_once()
+        self.assertEqual(len(urls), 1)
 
-        scraper.listing_collector.open_cars_listing.assert_called_once_with(page)
+        # Страница 1: переоткрытие листинга.
+        page.reset_mock()
+        _, urls = scraper._recover_empty_listing_page(
+            page, page_number=1, all_raw_urls=[], seen_urls=set(),
+        )
+        scraper.listing_collector.open_cars_listing.assert_called_once()
         page.reload.assert_not_called()
-        self.assertIs(recovered_result, page_result)
-        self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/456~US"])
 
 
 if __name__ == "__main__":

tests/test_utils.py

@@ -3,28 +3,72 @@ from __future__ import annotations
 import unittest
 
 from iaai_scraper.core.utils import deep_find_key
+from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
 
 
 class TestDeepFindKey(unittest.TestCase):
-    def test_finds_key_in_nested_structure(self) -> None:
+    def test_finds_nested_and_respects_depth(self) -> None:
         payload = {
             "root": {
                 "target": "a",
                 "nested": [{"target": "b"}, {"x": 1}],
             }
         }
+        self.assertEqual(deep_find_key(payload, {"target"}), ["a", "b"])
 
-        found = deep_find_key(payload, {"target"})
+        deep_payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
-        self.assertEqual(found, ["a", "b"])
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=2), [])
+        self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"])
 
-    def test_respects_max_depth(self) -> None:
-        payload = {"l1": {"l2": {"l3": {"target": "value"}}}}
 
-        found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2)
+class TestParseListingSegments(unittest.TestCase):
-        found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8)
+    def test_empty_and_invalid_return_empty(self) -> None:
+        self.assertEqual(parse_listing_segments(""), [])
+        self.assertEqual(parse_listing_segments("  "), [])
+        self.assertEqual(parse_listing_segments("invalid"), [])
 
-        self.assertEqual(found_too_shallow, [])
+    def test_auto_segments_complete_coverage(self) -> None:
-        self.assertEqual(found_enough_depth, ["value"])
+        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
+        segs = parse_listing_segments("auto")
+
+        # Точное число сегментов.
+        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
+        self.assertEqual(len(segs), expected)
+
+        # Все бренды из списка присутствуют.
+        makes_in_segments = {s["make"] for s in segs}
+        for make in IAAI_DEFAULT_MAKES:
+            self.assertIn(make, makes_in_segments)
+
+        # Крупные бренды разбиты на 3 сегмента с годами.
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        self.assertEqual(len(toyota), 3)
+        for s in toyota:
+            self.assertIsNotNone(s["year_min"])
+
+        # Мелкие бренды без годов.
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+
+        # Годовые диапазоны покрывают 1950-2027.
+        years = set()
+        for yr_min, yr_max in _YEAR_SPLITS:
+            years.update(range(yr_min, yr_max + 1))
+        for year in range(1950, 2027):
+            self.assertIn(year, years)
+
+    def test_json_input_formats(self) -> None:
+        # Массив строк.
+        segs = parse_listing_segments('["toyota", "ford"]')
+        self.assertEqual(len(segs), 2)
+        self.assertEqual(segs[0]["make"], "TOYOTA")
+
+        # Массив объектов с годами.
+        segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]')
+        self.assertEqual(segs[0]["make"], "BMW")
+        self.assertEqual(segs[0]["year_min"], 2020)
+        self.assertEqual(segs[0]["year_max"], 2025)
 
 
 if __name__ == "__main__":

tests/test_worker_tasks.py

@@ -8,32 +8,22 @@ from iaai_scraper.worker import tasks
 
 
 class TestWorkerTaskLockHelpers(unittest.TestCase):
-    def test_acquire_lock_returns_true_on_success(self) -> None:
+    def test_lock_acquire_refresh_release(self) -> None:
         redis_client = MagicMock()
+
+        # Acquire.
         redis_client.set.return_value = True
-
+        self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
-        acquired = tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)
-
-        self.assertTrue(acquired)
         redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
 
-    def test_refresh_lock_if_owner_extends_ttl(self) -> None:
+        # Refresh.
-        redis_client = MagicMock()
         redis_client.eval.return_value = 1
+        self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
 
-        refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)
+        # Release.
-
+        redis_client.eval.reset_mock()
-        self.assertTrue(refreshed)
-        redis_client.eval.assert_called_once()
-
-    def test_release_lock_if_owner_uses_owner_token(self) -> None:
-        redis_client = MagicMock()
-
         tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
-
-        redis_client.eval.assert_called_once()
         args = redis_client.eval.call_args[0]
-        self.assertEqual(args[1], 1)
         self.assertEqual(args[2], "lock:key")
         self.assertEqual(args[3], "owner-token")
 
@@ -62,6 +52,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_is_full_scan_done", return_value=True), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
              patch.object(tasks, "_run_browser_job", return_value={
                  "run_id": 7,
                  "cars_upserted": 2,
@@ -90,7 +81,8 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         heartbeat_thread.join.assert_called_once()
         release_lock.assert_called_once()
 
-    def test_clear_orphan_sync_listing_lock_deletes_when_no_tasks_running(self) -> None:
+    def test_clear_orphan_sync_listing_lock(self) -> None:
+        # Нет запущенных задач → удаляет.
         redis_client = MagicMock()
         redis_client.get.return_value = "owner-token"
         redis_client.ttl.return_value = 120
@@ -101,27 +93,21 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         inspector.scheduled.return_value = {"worker@node": []}
         celery_app.control.inspect.return_value = inspector
 
-        cleared = tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)
+        self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
-
-        self.assertTrue(cleared)
         redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
 
-    def test_clear_orphan_sync_listing_lock_keeps_when_task_detected(self) -> None:
+        # Задача активна → не удаляет.
-        redis_client = MagicMock()
+        redis_client2 = MagicMock()
-        redis_client.get.return_value = "owner-token"
+        redis_client2.get.return_value = "owner-token"
-        celery_app = MagicMock()
+        inspector2 = MagicMock()
-        inspector = MagicMock()
+        inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
-        inspector.active.return_value = {
+        inspector2.reserved.return_value = {"worker@node": []}
-            "worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]
+        inspector2.scheduled.return_value = {"worker@node": []}
-        }
+        celery_app2 = MagicMock()
-        inspector.reserved.return_value = {"worker@node": []}
+        celery_app2.control.inspect.return_value = inspector2
-        inspector.scheduled.return_value = {"worker@node": []}
-        celery_app.control.inspect.return_value = inspector
 
-        cleared = tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)
+        self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
-
+        redis_client2.delete.assert_not_called()
-        self.assertFalse(cleared)
-        redis_client.delete.assert_not_called()
 
     def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
         with patch.object(tasks, "_get_persistence") as get_persistence, \
@@ -131,6 +117,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_is_full_scan_done", return_value=True), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
              patch.object(tasks, "_run_browser_job", return_value={
                  "run_id": 9,
                  "cars_upserted": 3,
@@ -159,28 +146,22 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         self.assertEqual(acquire_lock.call_count, 2)
         release_lock.assert_called_once()
 
-    def test_sync_listing_checkpoint_roundtrip(self) -> None:
+    def test_sync_listing_checkpoint_save_load_and_resume(self) -> None:
+        # Roundtrip: save → load.
         redis_client = MagicMock()
         storage: dict[str, str] = {}
         redis_client.set.side_effect = lambda key, value: storage.__setitem__(key, value)
         redis_client.get.side_effect = lambda key: storage.get(key)
 
         tasks._save_sync_checkpoint(
-            redis_client,
+            redis_client, task_id="task-1", page_number=12,
-            task_id="task-1",
+            make=None, model=None, lane="iaai_cars",
-            page_number=12,
-            make=None,
-            model=None,
-            lane="iaai_cars",
         )
-
         checkpoint = tasks._load_sync_checkpoint(redis_client)
-
         self.assertIsNotNone(checkpoint)
         self.assertEqual(checkpoint["last_successful_page"], 12)
-        self.assertEqual(checkpoint["status"], "in_progress")
 
-    def test_sync_listing_task_resumes_from_checkpoint_page(self) -> None:
+        # Resume: задача стартует со страницы checkpoint + 1.
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
              patch.object(tasks, "_acquire_lock", return_value=True), \
@@ -188,32 +169,24 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
-             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()):
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
-            persistence = MagicMock()
+             patch.object(tasks.sync_listing_task, "update_state"), \
-            get_persistence.return_value = persistence
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
-            redis_client = MagicMock()
+            get_persistence.return_value = MagicMock()
-            redis_client.get.side_effect = lambda key: json.dumps({
+            redis_client2 = MagicMock()
-                "status": "in_progress",
+            redis_client2.get.side_effect = lambda key: json.dumps({
-                "last_successful_page": 9,
+                "status": "in_progress", "last_successful_page": 9,
-                "make": None,
+                "make": None, "model": None, "lane": "iaai_cars",
-                "model": None,
-                "lane": "iaai_cars",
             }) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
-            get_redis.return_value = redis_client
+            get_redis.return_value = redis_client2
             stop_event = MagicMock()
             heartbeat_thread = MagicMock()
             start_heartbeat.return_value = (stop_event, heartbeat_thread)
 
             sync_listing_mock = MagicMock(return_value={
-                "run_id": 11,
+                "run_id": 11, "status": "success", "full_scan_completed": True,
-                "status": "success",
+                "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
-                "full_scan_completed": True,
+                "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
-                "cars_upserted": 1,
-                "cars_failed": 0,
-                "images_upserted": 0,
-                "skipped_existing": 0,
-                "elapsed_seconds": 1.0,
-                "failures": [],
             })
             scraper_ctx = MagicMock()
             scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock