fix scraper flow
This commit is contained in:
qananasikq
@@ -1,13 +1,12 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from concurrent.futures import TimeoutError as FuturesTimeoutError
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
|
||||
from iaai_scraper.core.exceptions import ListingResumeError
|
||||
from iaai_scraper.discovery import SitemapDiscoveryError
|
||||
from iaai_scraper.scraper import IAAIScraper
|
||||
from iaai_scraper.storage.schemas import CarRecord
|
||||
|
||||
@@ -87,51 +86,6 @@ class TestScraperSync(unittest.TestCase):
|
||||
scraper2._sync_listing_streaming.assert_called_once()
|
||||
scraper2.collect_listing.assert_not_called()
|
||||
|
||||
def test_full_scan_uses_sitemap_discovery_path(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.discovery.mode = "listing"
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=88)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
|
||||
scraper._sync_listing_via_sitemap = MagicMock(return_value={
|
||||
"listing": {
|
||||
"vehicles_collected": 123,
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
"pagination_interrupted": False,
|
||||
"source": "sitemap",
|
||||
},
|
||||
"total": 123,
|
||||
"skipped_existing": 0,
|
||||
"cars_upserted": 120,
|
||||
"cars_failed": 3,
|
||||
"images_upserted": 500,
|
||||
"failures": [{"vehicle_url": "v", "error": "e"}],
|
||||
"all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"},
|
||||
})
|
||||
scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used"))
|
||||
|
||||
result = scraper.sync_listing()
|
||||
|
||||
scraper._sync_listing_via_sitemap.assert_called_once()
|
||||
scraper._sync_listing_streaming.assert_not_called()
|
||||
self.assertEqual(result["cars_upserted"], 120)
|
||||
self.assertTrue(result["full_scan_completed"])
|
||||
|
||||
def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=89)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
|
||||
scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down"))
|
||||
result = scraper.sync_listing()
|
||||
|
||||
scraper._sync_listing_via_sitemap.assert_called_once()
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(result["cars_upserted"], 0)
|
||||
|
||||
def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
@@ -315,125 +269,50 @@ class TestScraperSync(unittest.TestCase):
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
self.assertEqual(result["total"], 1)
|
||||
|
||||
def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None:
|
||||
def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.celery.batch_size = 1000
|
||||
|
||||
page = MagicMock()
|
||||
page_result = SimpleNamespace(
|
||||
page_number=1,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
|
||||
next_page_detected=True,
|
||||
)
|
||||
|
||||
scraper._get_page_with_warmup = MagicMock(return_value=page)
|
||||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||||
scraper.listing_collector.apply_filters = MagicMock(return_value={
|
||||
"make": None,
|
||||
"model": None,
|
||||
"year_min": None,
|
||||
"year_max": None,
|
||||
})
|
||||
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
|
||||
scraper.listing_collector.go_to_next_page = MagicMock(return_value=False)
|
||||
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
|
||||
scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed"))
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
scraper.persistence.upsert_cars_batch = MagicMock(return_value={
|
||||
"inserted": 1,
|
||||
"updated": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
})
|
||||
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
)
|
||||
urls = [
|
||||
"https://www.iaai.com/VehicleDetail/111~US",
|
||||
"https://www.iaai.com/VehicleDetail/222~US",
|
||||
"https://www.iaai.com/VehicleDetail/333~US",
|
||||
]
|
||||
first_record = CarRecord.model_validate(make_db_record("111"))
|
||||
|
||||
self.assertTrue(result["listing"]["pagination_interrupted"])
|
||||
class _FakeExecutor:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.celery.batch_size = 1000
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
page = MagicMock()
|
||||
first_page = SimpleNamespace(
|
||||
page_number=1,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")],
|
||||
next_page_detected=False,
|
||||
)
|
||||
second_page = SimpleNamespace(
|
||||
page_number=2,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")],
|
||||
next_page_detected=False,
|
||||
)
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
scraper._get_page_with_warmup = MagicMock(return_value=page)
|
||||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||||
scraper.listing_collector.apply_filters = MagicMock(return_value={
|
||||
"make": None,
|
||||
"model": None,
|
||||
"year_min": None,
|
||||
"year_max": None,
|
||||
})
|
||||
scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page])
|
||||
scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False])
|
||||
scraper._extract_page_urls = MagicMock(side_effect=[
|
||||
["https://www.iaai.com/VehicleDetail/111~US"],
|
||||
["https://www.iaai.com/VehicleDetail/222~US"],
|
||||
])
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
})
|
||||
def map(self, fn, iterable, timeout=None): # noqa: ARG002
|
||||
yield 0, first_record
|
||||
raise FuturesTimeoutError()
|
||||
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
)
|
||||
with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
|
||||
patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
|
||||
patch.object(scraper, "_browser_fallback_parallel", return_value={
|
||||
"records": [],
|
||||
"failures": [],
|
||||
"cars_failed": 0,
|
||||
"protection_events": 0,
|
||||
}) as fallback_mock:
|
||||
result = scraper.sync_batch(urls)
|
||||
|
||||
self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2)
|
||||
scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2)
|
||||
self.assertEqual(result["listing"]["pages_collected"], 2)
|
||||
|
||||
def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=77)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
|
||||
scraper._sync_listing_streaming = MagicMock(return_value={
|
||||
"listing": {
|
||||
"vehicles_collected": 10,
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
"pagination_interrupted": True,
|
||||
},
|
||||
"total": 10,
|
||||
"skipped_existing": 0,
|
||||
"cars_upserted": 10,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
"all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"},
|
||||
})
|
||||
|
||||
result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA")
|
||||
|
||||
self.assertTrue(result["full_scan_completed"] is False)
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
fallback_urls = fallback_mock.call_args.args[0]
|
||||
self.assertEqual(len(fallback_urls), 2)
|
||||
self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
|
||||
self.assertNotIn(urls[0], {u for u, _ in fallback_urls})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user