qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

Browse Source
This commit is contained in:
qananasikq
2026-04-20 16:27:42 +03:00
parent 55203d33ea
commit 65d5f8e1eb
23 changed files with 3436 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
tests/test_scraper.py
View File

@@ -6,6 +6,8 @@ from unittest.mock import MagicMock
from iaai_scraper.core.config import Settings
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from iaai_scraper.core.exceptions import ListingResumeError
from iaai_scraper.discovery import SitemapDiscoveryError
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.storage.schemas import CarRecord
@@ -85,6 +87,51 @@ class TestScraperSync(unittest.TestCase):
scraper2._sync_listing_streaming.assert_called_once()
scraper2.collect_listing.assert_not_called()
def test_full_scan_uses_sitemap_discovery_path(self) -> None:
scraper = self._make_scraper()
scraper.settings.discovery.mode = "listing"
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=88)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
scraper._sync_listing_via_sitemap = MagicMock(return_value={
"listing": {
"vehicles_collected": 123,
"early_stopped": False,
"truncated_by_time_budget": False,
"pagination_interrupted": False,
"source": "sitemap",
},
"total": 123,
"skipped_existing": 0,
"cars_upserted": 120,
"cars_failed": 3,
"images_upserted": 500,
"failures": [{"vehicle_url": "v", "error": "e"}],
"all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"},
})
scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used"))
result = scraper.sync_listing()
scraper._sync_listing_via_sitemap.assert_called_once()
scraper._sync_listing_streaming.assert_not_called()
self.assertEqual(result["cars_upserted"], 120)
self.assertTrue(result["full_scan_completed"])
def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=89)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down"))
result = scraper.sync_listing()
scraper._sync_listing_via_sitemap.assert_called_once()
self.assertEqual(result["status"], "failed")
self.assertEqual(result["cars_upserted"], 0)
def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
@@ -268,6 +315,126 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(result["total"], 1)
def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None:
scraper = self._make_scraper()
scraper.settings.celery.batch_size = 1000
page = MagicMock()
page_result = SimpleNamespace(
page_number=1,
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
next_page_detected=True,
)
scraper._get_page_with_warmup = MagicMock(return_value=page)
scraper.listing_collector.open_cars_listing = MagicMock()
scraper.listing_collector.apply_filters = MagicMock(return_value={
"make": None,
"model": None,
"year_min": None,
"year_max": None,
})
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper.listing_collector.go_to_next_page = MagicMock(return_value=False)
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed"))
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 0,
"cars_failed": 0,
"images_upserted": 0,
"failures": [],
})
result = scraper._sync_listing_streaming(
make=None,
model=None,
lane="iaai_cars",
limit=None,
effective_only_new=False,
started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
)
self.assertTrue(result["listing"]["pagination_interrupted"])
def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None:
scraper = self._make_scraper()
scraper.settings.celery.batch_size = 1000
page = MagicMock()
first_page = SimpleNamespace(
page_number=1,
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")],
next_page_detected=False,
)
second_page = SimpleNamespace(
page_number=2,
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")],
next_page_detected=False,
)
scraper._get_page_with_warmup = MagicMock(return_value=page)
scraper.listing_collector.open_cars_listing = MagicMock()
scraper.listing_collector.apply_filters = MagicMock(return_value={
"make": None,
"model": None,
"year_min": None,
"year_max": None,
})
scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page])
scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False])
scraper._extract_page_urls = MagicMock(side_effect=[
["https://www.iaai.com/VehicleDetail/111~US"],
["https://www.iaai.com/VehicleDetail/222~US"],
])
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 2,
"cars_failed": 0,
"images_upserted": 0,
"failures": [],
})
result = scraper._sync_listing_streaming(
make=None,
model=None,
lane="iaai_cars",
limit=None,
effective_only_new=False,
started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
)
self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2)
scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2)
self.assertEqual(result["listing"]["pages_collected"], 2)
def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=77)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
scraper._sync_listing_streaming = MagicMock(return_value={
"listing": {
"vehicles_collected": 10,
"early_stopped": False,
"truncated_by_time_budget": False,
"pagination_interrupted": True,
},
"total": 10,
"skipped_existing": 0,
"cars_upserted": 10,
"cars_failed": 0,
"images_upserted": 0,
"failures": [],
"all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"},
})
result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA")
self.assertTrue(result["full_scan_completed"] is False)
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
if __name__ == "__main__":
unittest.main()