fix docker scraping

improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup

tests/test_scraper.py

@@ -51,15 +51,12 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=2)
         scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set()))
 
         scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
-        page = MagicMock()
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=page)
-        scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
+        })
 
         result = scraper.sync_listing()
 
@@ -67,8 +64,7 @@ class TestScraperSync(unittest.TestCase):
         self.assertEqual(result["cars_failed"], 0)
         self.assertIn("trace_id", result)
         self.assertIn("elapsed_seconds", result)
-        self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
-        page.close.assert_called_once()
+        scraper.sync_batch.assert_called_once()
 
     def test_sync_listing_respects_limit(self) -> None:
         scraper = self._make_scraper()
@@ -76,38 +72,25 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=3)
         scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
-        scraper.collect_listing = MagicMock(return_value={
-            "vehicle_urls": [
-                "https://www.iaai.com/VehicleDetail/222~US",
-                "https://www.iaai.com/VehicleDetail/333~US",
-            ]
+
+        # Проверка пути only_new с limit.
+        scraper._collect_listing_iterative = MagicMock(return_value=(
+            ["https://www.iaai.com/VehicleDetail/222~US"],
+            ["https://www.iaai.com/VehicleDetail/222~US",
+             "https://www.iaai.com/VehicleDetail/333~US"],
+            {"vehicle_urls": [], "pages_collected": 1, "early_stopped": False},
+            0,
+        ))
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
         })
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
-        scraper._get_page = MagicMock(return_value=MagicMock())
 
         scraper.sync_listing(limit=1)
 
-        self.assertEqual(scraper._scrape_on_page.call_count, 1)
-
-    def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
-        scraper = self._make_scraper()
-
-        scraper.persistence.create_tables = MagicMock()
-        scraper.persistence.start_sync_run = MagicMock(return_value=4)
-        scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
-        scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
-        scraper._get_page = MagicMock(return_value=MagicMock())
-
-        result = scraper.sync_listing()
-
-        self.assertEqual(result["cars_upserted"], 0)
+        # Должен уйти только один URL.
+        scraper.sync_batch.assert_called_once()
+        batch_urls = scraper.sync_batch.call_args[0][0]
+        self.assertEqual(len(batch_urls), 1)
 
     def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
         scraper = self._make_scraper()
@@ -115,9 +98,10 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.create_tables = MagicMock()
         scraper.persistence.start_sync_run = MagicMock(return_value=5)
         scraper.persistence.finish_sync_run = MagicMock()
-        scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
-        scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"})
-        scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"})
+        scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
+            {"https://www.iaai.com/VehicleDetail/111~US"},
+            {"iaai:222"},
+        ))
 
         scraper.collect_listing = MagicMock(return_value={
             "vehicle_urls": [
@@ -126,26 +110,34 @@ class TestScraperSync(unittest.TestCase):
                 "https://www.iaai.com/VehicleDetail/333~US",  # new
             ]
         })
-        page = MagicMock()
-        scraper._get_page = MagicMock(return_value=page)
-        scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")})
+        # Возвращаем результат для одного нового авто.
+        scraper.sync_batch = MagicMock(return_value={
+            "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
+        })
 
         result = scraper.sync_listing(only_new=True)
 
         self.assertEqual(result["skipped_existing"], 2)
         self.assertEqual(result["cars_upserted"], 1)
-        self.assertEqual(scraper._scrape_on_page.call_count, 1)
-        scraper.persistence.get_existing_origin_urls.assert_called_once()
-        scraper.persistence.get_existing_origin_ids.assert_called_once()
+        # В batch должен попасть только новый URL.
+        scraper.sync_batch.assert_called_once()
+        batch_urls = scraper.sync_batch.call_args[0][0]
+        self.assertEqual(len(batch_urls), 1)
+        self.assertIn("333", batch_urls[0])
+        scraper.persistence.get_existing_urls_and_ids.assert_called_once()
 
     def test_close_resets_browser_state(self) -> None:
         scraper = self._make_scraper()
+        http_pool = MagicMock()
+        scraper._http_pool = http_pool
         scraper.context = MagicMock()
         scraper.browser = MagicMock()
         scraper.playwright = MagicMock()
 
         scraper.close()
 
+        http_pool.clear.assert_called_once()
+        self.assertIsNone(scraper._http_pool)
         self.assertIsNone(scraper.context)
         self.assertIsNone(scraper.browser)
         self.assertIsNone(scraper.playwright)
@@ -172,6 +164,11 @@ class TestScraperSync(unittest.TestCase):
                 "https://www.iaai.com/VehicleDetail/999~US",
             )
 
+    def test_is_protection_or_network_error_detects_known_signals(self) -> None:
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
+
 
 if __name__ == "__main__":
     unittest.main()