Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services
This commit is contained in:
qananasikq
@@ -6,6 +6,8 @@ from unittest.mock import MagicMock
|
||||
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
|
||||
from iaai_scraper.core.exceptions import ListingResumeError
|
||||
from iaai_scraper.discovery import SitemapDiscoveryError
|
||||
from iaai_scraper.scraper import IAAIScraper
|
||||
from iaai_scraper.storage.schemas import CarRecord
|
||||
|
||||
@@ -85,6 +87,51 @@ class TestScraperSync(unittest.TestCase):
|
||||
scraper2._sync_listing_streaming.assert_called_once()
|
||||
scraper2.collect_listing.assert_not_called()
|
||||
|
||||
def test_full_scan_uses_sitemap_discovery_path(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.discovery.mode = "listing"
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=88)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
|
||||
scraper._sync_listing_via_sitemap = MagicMock(return_value={
|
||||
"listing": {
|
||||
"vehicles_collected": 123,
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
"pagination_interrupted": False,
|
||||
"source": "sitemap",
|
||||
},
|
||||
"total": 123,
|
||||
"skipped_existing": 0,
|
||||
"cars_upserted": 120,
|
||||
"cars_failed": 3,
|
||||
"images_upserted": 500,
|
||||
"failures": [{"vehicle_url": "v", "error": "e"}],
|
||||
"all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"},
|
||||
})
|
||||
scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used"))
|
||||
|
||||
result = scraper.sync_listing()
|
||||
|
||||
scraper._sync_listing_via_sitemap.assert_called_once()
|
||||
scraper._sync_listing_streaming.assert_not_called()
|
||||
self.assertEqual(result["cars_upserted"], 120)
|
||||
self.assertTrue(result["full_scan_completed"])
|
||||
|
||||
def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=89)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
|
||||
scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down"))
|
||||
result = scraper.sync_listing()
|
||||
|
||||
scraper._sync_listing_via_sitemap.assert_called_once()
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(result["cars_upserted"], 0)
|
||||
|
||||
def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
@@ -268,6 +315,126 @@ class TestScraperSync(unittest.TestCase):
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
self.assertEqual(result["total"], 1)
|
||||
|
||||
def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.celery.batch_size = 1000
|
||||
|
||||
page = MagicMock()
|
||||
page_result = SimpleNamespace(
|
||||
page_number=1,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
|
||||
next_page_detected=True,
|
||||
)
|
||||
|
||||
scraper._get_page_with_warmup = MagicMock(return_value=page)
|
||||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||||
scraper.listing_collector.apply_filters = MagicMock(return_value={
|
||||
"make": None,
|
||||
"model": None,
|
||||
"year_min": None,
|
||||
"year_max": None,
|
||||
})
|
||||
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
|
||||
scraper.listing_collector.go_to_next_page = MagicMock(return_value=False)
|
||||
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
|
||||
scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed"))
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
})
|
||||
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
)
|
||||
|
||||
self.assertTrue(result["listing"]["pagination_interrupted"])
|
||||
|
||||
def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.celery.batch_size = 1000
|
||||
|
||||
page = MagicMock()
|
||||
first_page = SimpleNamespace(
|
||||
page_number=1,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")],
|
||||
next_page_detected=False,
|
||||
)
|
||||
second_page = SimpleNamespace(
|
||||
page_number=2,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")],
|
||||
next_page_detected=False,
|
||||
)
|
||||
|
||||
scraper._get_page_with_warmup = MagicMock(return_value=page)
|
||||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||||
scraper.listing_collector.apply_filters = MagicMock(return_value={
|
||||
"make": None,
|
||||
"model": None,
|
||||
"year_min": None,
|
||||
"year_max": None,
|
||||
})
|
||||
scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page])
|
||||
scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False])
|
||||
scraper._extract_page_urls = MagicMock(side_effect=[
|
||||
["https://www.iaai.com/VehicleDetail/111~US"],
|
||||
["https://www.iaai.com/VehicleDetail/222~US"],
|
||||
])
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
})
|
||||
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
)
|
||||
|
||||
self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2)
|
||||
scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2)
|
||||
self.assertEqual(result["listing"]["pages_collected"], 2)
|
||||
|
||||
def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=77)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
|
||||
scraper._sync_listing_streaming = MagicMock(return_value={
|
||||
"listing": {
|
||||
"vehicles_collected": 10,
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
"pagination_interrupted": True,
|
||||
},
|
||||
"total": 10,
|
||||
"skipped_existing": 0,
|
||||
"cars_upserted": 10,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
"all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"},
|
||||
})
|
||||
|
||||
result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA")
|
||||
|
||||
self.assertTrue(result["full_scan_completed"] is False)
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user