Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services
This commit is contained in:
qananasikq
@@ -8,26 +8,83 @@ from iaai_scraper.browser.listing import ListingCollector
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self, counts: dict[str, int]) -> None:
|
||||
def __init__(self, counts: dict[str, int], attrs: dict[str, dict[str, str]] | None = None) -> None:
|
||||
self._counts = counts
|
||||
self._attrs = attrs or {}
|
||||
self._evaluate_result = False
|
||||
self._evaluate_values: list[object] = []
|
||||
self._html = ""
|
||||
self.url = "https://www.iaai.com/Vehiclelisting/Cars"
|
||||
self._current_page_number: int | None = None
|
||||
self._vehicle_hrefs: list[str] = []
|
||||
self._wait_for_function_error: Exception | None = None
|
||||
self._clicks: dict[str, int] = {}
|
||||
self._goto_calls: list[str] = []
|
||||
|
||||
class _Locator:
|
||||
def __init__(self, count_value: int) -> None:
|
||||
def __init__(self, page: "_FakePage", selector: str, count_value: int) -> None:
|
||||
self._page = page
|
||||
self._selector = selector
|
||||
self._count_value = count_value
|
||||
|
||||
@property
|
||||
def first(self) -> "_FakePage._Locator":
|
||||
return self
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count_value
|
||||
|
||||
def locator(self, selector: str) -> "_FakePage._Locator":
|
||||
return _FakePage._Locator(self._counts.get(selector, 0))
|
||||
def is_visible(self, timeout: int | None = None) -> bool:
|
||||
_ = timeout
|
||||
return False
|
||||
|
||||
def evaluate(self, _script: str):
|
||||
def get_attribute(self, name: str, timeout: int | None = None) -> str | None:
|
||||
_ = timeout
|
||||
return self._page._attrs.get(self._selector, {}).get(name)
|
||||
|
||||
def click(self, timeout: int | None = None) -> None:
|
||||
_ = timeout
|
||||
self._page._clicks[self._selector] = self._page._clicks.get(self._selector, 0) + 1
|
||||
return None
|
||||
|
||||
class _HtmlLocator:
|
||||
def __init__(self, html: str) -> None:
|
||||
self._html = html
|
||||
|
||||
def inner_html(self, timeout: int | None = None) -> str:
|
||||
_ = timeout
|
||||
return self._html
|
||||
|
||||
def locator(self, selector: str):
|
||||
if selector == "html":
|
||||
return _FakePage._HtmlLocator(self._html)
|
||||
return _FakePage._Locator(self, selector, self._counts.get(selector, 0))
|
||||
|
||||
def evaluate(self, script: str, *args):
|
||||
_ = args
|
||||
if self._evaluate_values:
|
||||
return self._evaluate_values.pop(0)
|
||||
if "querySelectorAll" in script and "VehicleDetail" in script:
|
||||
return list(self._vehicle_hrefs)
|
||||
if "document.body?.innerText" in script and "aria-current" in script and "parseInt" in script:
|
||||
return self._current_page_number if self._current_page_number is not None else self._evaluate_result
|
||||
return self._evaluate_result
|
||||
|
||||
def wait_for_selector(self, selector: str, timeout: int | None = None) -> None:
|
||||
_ = selector, timeout
|
||||
return None
|
||||
|
||||
def wait_for_function(self, script: str, timeout: int | None = None) -> None:
|
||||
_ = script, timeout
|
||||
if self._wait_for_function_error is not None:
|
||||
raise self._wait_for_function_error
|
||||
return None
|
||||
|
||||
def goto(self, url: str, wait_until: str | None = None, timeout: int | None = None) -> None:
|
||||
_ = wait_until, timeout
|
||||
self._goto_calls.append(url)
|
||||
self.url = url
|
||||
|
||||
|
||||
class TestListingUnit(unittest.TestCase):
|
||||
def test_pagination_detection_and_page_number(self) -> None:
|
||||
@@ -63,6 +120,67 @@ class TestListingUnit(unittest.TestCase):
|
||||
],
|
||||
)
|
||||
|
||||
def test_has_next_page_from_html(self) -> None:
|
||||
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
||||
html = '<a class="pagination-next" href="/Vehiclelisting/Cars?page=43">Next</a>'
|
||||
self.assertTrue(collector._has_next_page_from_html(html, current_page_number=42))
|
||||
self.assertFalse(collector._has_next_page_from_html("<div>done</div>", current_page_number=42))
|
||||
|
||||
def test_build_listing_page_url_replaces_existing_page_parameter(self) -> None:
|
||||
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
||||
|
||||
url = collector._build_listing_page_url(
|
||||
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&page=43&foo=bar",
|
||||
44,
|
||||
)
|
||||
|
||||
self.assertEqual(
|
||||
url,
|
||||
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA&foo=bar&page=44",
|
||||
)
|
||||
|
||||
def test_collect_current_page_uses_html_first(self) -> None:
|
||||
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
||||
page = _FakePage({})
|
||||
page._html = '<a href="/VehicleDetail/555~US">Car 555</a><a class="pagination-next" href="/Vehiclelisting/Cars?page=2">Next</a>'
|
||||
|
||||
result = collector.collect_current_page(page, page_number=1)
|
||||
|
||||
self.assertEqual(len(result.vehicle_links), 1)
|
||||
self.assertEqual(result.vehicle_links[0].lot_number, "555")
|
||||
self.assertTrue(result.next_page_detected)
|
||||
|
||||
def test_wait_for_navigation_result_rejects_duplicate_content_even_when_page_number_matches(self) -> None:
|
||||
page = _FakePage({})
|
||||
page._wait_for_function_error = RuntimeError("same content")
|
||||
page._current_page_number = 41
|
||||
page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
|
||||
|
||||
self.assertFalse(
|
||||
ListingCollector._wait_for_navigation_result(
|
||||
page,
|
||||
"/VehicleDetail/111~US",
|
||||
41,
|
||||
old_fingerprint=("/VehicleDetail/111~US", "/VehicleDetail/222~US"),
|
||||
)
|
||||
)
|
||||
|
||||
def test_go_to_next_page_skips_flaky_ui_fallbacks_on_deep_pages(self) -> None:
|
||||
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
||||
page = _FakePage(
|
||||
{ListingCollector._NEXT_PAGE_SELECTORS[0]: 1, "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": 1},
|
||||
attrs={
|
||||
"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']": {"href": "/VehicleDetail/111~US"},
|
||||
},
|
||||
)
|
||||
page._wait_for_function_error = RuntimeError("same content")
|
||||
page._current_page_number = 41
|
||||
page._vehicle_hrefs = ["/VehicleDetail/111~US", "/VehicleDetail/222~US"]
|
||||
|
||||
self.assertFalse(collector.go_to_next_page(page, expected_page_number=41))
|
||||
self.assertGreaterEqual(len(page._goto_calls), 1)
|
||||
self.assertEqual(page._clicks.get(ListingCollector._NEXT_PAGE_SELECTORS[0], 0), 0)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -6,6 +6,8 @@ from unittest.mock import MagicMock
|
||||
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
|
||||
from iaai_scraper.core.exceptions import ListingResumeError
|
||||
from iaai_scraper.discovery import SitemapDiscoveryError
|
||||
from iaai_scraper.scraper import IAAIScraper
|
||||
from iaai_scraper.storage.schemas import CarRecord
|
||||
|
||||
@@ -85,6 +87,51 @@ class TestScraperSync(unittest.TestCase):
|
||||
scraper2._sync_listing_streaming.assert_called_once()
|
||||
scraper2.collect_listing.assert_not_called()
|
||||
|
||||
def test_full_scan_uses_sitemap_discovery_path(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.discovery.mode = "listing"
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=88)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
|
||||
scraper._sync_listing_via_sitemap = MagicMock(return_value={
|
||||
"listing": {
|
||||
"vehicles_collected": 123,
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
"pagination_interrupted": False,
|
||||
"source": "sitemap",
|
||||
},
|
||||
"total": 123,
|
||||
"skipped_existing": 0,
|
||||
"cars_upserted": 120,
|
||||
"cars_failed": 3,
|
||||
"images_upserted": 500,
|
||||
"failures": [{"vehicle_url": "v", "error": "e"}],
|
||||
"all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/111~US"},
|
||||
})
|
||||
scraper._sync_listing_streaming = MagicMock(side_effect=AssertionError("pagination path should not be used"))
|
||||
|
||||
result = scraper.sync_listing()
|
||||
|
||||
scraper._sync_listing_via_sitemap.assert_called_once()
|
||||
scraper._sync_listing_streaming.assert_not_called()
|
||||
self.assertEqual(result["cars_upserted"], 120)
|
||||
self.assertTrue(result["full_scan_completed"])
|
||||
|
||||
def test_full_scan_does_not_fallback_to_pagination_when_sitemap_fails(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=89)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock(return_value=0)
|
||||
scraper._sync_listing_via_sitemap = MagicMock(side_effect=SitemapDiscoveryError("sitemap down"))
|
||||
result = scraper.sync_listing()
|
||||
|
||||
scraper._sync_listing_via_sitemap.assert_called_once()
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertEqual(result["cars_upserted"], 0)
|
||||
|
||||
def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
@@ -268,6 +315,126 @@ class TestScraperSync(unittest.TestCase):
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
self.assertEqual(result["total"], 1)
|
||||
|
||||
def test_sync_listing_marks_pagination_interrupted_when_next_page_resume_fails(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.celery.batch_size = 1000
|
||||
|
||||
page = MagicMock()
|
||||
page_result = SimpleNamespace(
|
||||
page_number=1,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
|
||||
next_page_detected=True,
|
||||
)
|
||||
|
||||
scraper._get_page_with_warmup = MagicMock(return_value=page)
|
||||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||||
scraper.listing_collector.apply_filters = MagicMock(return_value={
|
||||
"make": None,
|
||||
"model": None,
|
||||
"year_min": None,
|
||||
"year_max": None,
|
||||
})
|
||||
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
|
||||
scraper.listing_collector.go_to_next_page = MagicMock(return_value=False)
|
||||
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
|
||||
scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("resume failed"))
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
})
|
||||
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
)
|
||||
|
||||
self.assertTrue(result["listing"]["pagination_interrupted"])
|
||||
|
||||
def test_sync_listing_attempts_next_page_even_without_detected_next_control(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.celery.batch_size = 1000
|
||||
|
||||
page = MagicMock()
|
||||
first_page = SimpleNamespace(
|
||||
page_number=1,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/111~US", lot_number="111")],
|
||||
next_page_detected=False,
|
||||
)
|
||||
second_page = SimpleNamespace(
|
||||
page_number=2,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/222~US", lot_number="222")],
|
||||
next_page_detected=False,
|
||||
)
|
||||
|
||||
scraper._get_page_with_warmup = MagicMock(return_value=page)
|
||||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||||
scraper.listing_collector.apply_filters = MagicMock(return_value={
|
||||
"make": None,
|
||||
"model": None,
|
||||
"year_min": None,
|
||||
"year_max": None,
|
||||
})
|
||||
scraper.listing_collector.collect_current_page = MagicMock(side_effect=[first_page, second_page])
|
||||
scraper.listing_collector.go_to_next_page = MagicMock(side_effect=[True, False])
|
||||
scraper._extract_page_urls = MagicMock(side_effect=[
|
||||
["https://www.iaai.com/VehicleDetail/111~US"],
|
||||
["https://www.iaai.com/VehicleDetail/222~US"],
|
||||
])
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
})
|
||||
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
)
|
||||
|
||||
self.assertEqual(scraper.listing_collector.go_to_next_page.call_count, 2)
|
||||
scraper.listing_collector.go_to_next_page.assert_any_call(page, expected_page_number=2)
|
||||
self.assertEqual(result["listing"]["pages_collected"], 2)
|
||||
|
||||
def test_sync_listing_treats_pagination_interrupted_as_partial_scan(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=77)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls = MagicMock()
|
||||
scraper._sync_listing_streaming = MagicMock(return_value={
|
||||
"listing": {
|
||||
"vehicles_collected": 10,
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
"pagination_interrupted": True,
|
||||
},
|
||||
"total": 10,
|
||||
"skipped_existing": 0,
|
||||
"cars_upserted": 10,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
"all_listing_origin_urls": {"https://www.iaai.com/VehicleDetail/999~US"},
|
||||
})
|
||||
|
||||
result = scraper.sync_listing(listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA")
|
||||
|
||||
self.assertTrue(result["full_scan_completed"] is False)
|
||||
scraper.persistence.mark_sold_not_in_listing_by_urls.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
217
tests/test_sitemap_discovery.py
Normal file
217
tests/test_sitemap_discovery.py
Normal file
@@ -0,0 +1,217 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.discovery.sitemap import (
|
||||
SitemapBlockedError,
|
||||
SitemapDiscoveryError,
|
||||
SitemapFetchResult,
|
||||
_filter_vehicle_urls,
|
||||
discover_vehicle_urls_from_sitemap,
|
||||
discover_vehicle_urls_from_sitemap_with_stats,
|
||||
)
|
||||
|
||||
|
||||
class TestSitemapDiscovery(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _fetch_result(url: str, payload: bytes, *, source: str = "curl_cffi") -> SitemapFetchResult:
|
||||
return SitemapFetchResult(url=url, payload=payload, source=source)
|
||||
|
||||
def test_filter_vehicle_urls_dedupes_and_normalizes(self) -> None:
|
||||
urls = _filter_vehicle_urls([
|
||||
"https://www.iaai.com/VehicleDetail/111~US?foo=1",
|
||||
"https://www.iaai.com/VehicleDetail/111~US?bar=2",
|
||||
"https://www.iaai.com/VehicleDetail/222~US",
|
||||
"https://www.iaai.com/about",
|
||||
])
|
||||
|
||||
self.assertEqual(
|
||||
urls,
|
||||
[
|
||||
"https://www.iaai.com/VehicleDetail/111~US",
|
||||
"https://www.iaai.com/VehicleDetail/222~US",
|
||||
],
|
||||
)
|
||||
|
||||
def test_discover_vehicle_urls_from_sitemap(self) -> None:
|
||||
index_xml = b"""
|
||||
<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
|
||||
<sitemap><loc>https://www.iaai.com/sitemap-a.xml</loc></sitemap>
|
||||
<sitemap><loc>https://www.iaai.com/sitemap-b.xml</loc></sitemap>
|
||||
<sitemap><loc>https://www.iaai.com/sitemapauctions1.xml</loc></sitemap>
|
||||
</sitemapindex>
|
||||
"""
|
||||
sitemap_a = b"""
|
||||
<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
|
||||
<url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
|
||||
<url><loc>https://www.iaai.com/VehicleDetail/222~US?x=1</loc></url>
|
||||
</urlset>
|
||||
"""
|
||||
sitemap_b = b"""
|
||||
<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
|
||||
<url><loc>https://www.iaai.com/VehicleDetail/222~US?y=2</loc></url>
|
||||
<url><loc>https://www.iaai.com/VehicleDetail/333~US</loc></url>
|
||||
</urlset>
|
||||
"""
|
||||
|
||||
def _fake_fetch(url: str) -> SitemapFetchResult:
|
||||
if url.endswith("sitemap_index.xml"):
|
||||
return self._fetch_result(url, index_xml)
|
||||
if url.endswith("sitemap-a.xml"):
|
||||
return self._fetch_result(url, sitemap_a)
|
||||
if url.endswith("sitemap-b.xml"):
|
||||
return self._fetch_result(url, sitemap_b)
|
||||
raise AssertionError(f"unexpected url: {url}")
|
||||
|
||||
with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
|
||||
result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
|
||||
|
||||
self.assertEqual(
|
||||
result,
|
||||
[
|
||||
"https://www.iaai.com/VehicleDetail/111~US",
|
||||
"https://www.iaai.com/VehicleDetail/222~US",
|
||||
"https://www.iaai.com/VehicleDetail/333~US",
|
||||
],
|
||||
)
|
||||
|
||||
def test_discover_vehicle_urls_raises_on_empty_index(self) -> None:
|
||||
empty_index = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
|
||||
with patch(
|
||||
"iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
|
||||
return_value=self._fetch_result("https://www.iaai.com/sitemap_index.xml", empty_index),
|
||||
):
|
||||
with self.assertRaises(SitemapDiscoveryError):
|
||||
discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
|
||||
|
||||
def test_discover_vehicle_urls_skips_malformed_and_non_vehicle_sitemaps(self) -> None:
|
||||
sitemap_vehicle = b"""
|
||||
<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
|
||||
<url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
|
||||
</urlset>
|
||||
"""
|
||||
sitemap_non_vehicle = b"""
|
||||
<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
|
||||
<url><loc>https://www.iaai.com/SalesList/111~US/04222026</loc></url>
|
||||
</urlset>
|
||||
"""
|
||||
index_xml = b"""
|
||||
<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
|
||||
<sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>
|
||||
<sitemap><loc>https://www.iaai.com/sitemapbranches1.xml</loc></sitemap>
|
||||
<sitemap><loc>https://www.iaai.com/sitemap2.xml</loc></sitemap>
|
||||
<sitemap><loc>https://www.iaai.com/sitemap3.xml</loc></sitemap>
|
||||
</sitemapindex>
|
||||
"""
|
||||
|
||||
def _fake_fetch(url: str) -> SitemapFetchResult:
|
||||
if url.endswith("sitemap_index.xml"):
|
||||
return self._fetch_result(url, index_xml)
|
||||
if url.endswith("sitemap1.xml"):
|
||||
return self._fetch_result(url, sitemap_vehicle)
|
||||
if url.endswith("sitemap2.xml"):
|
||||
return self._fetch_result(url, sitemap_non_vehicle)
|
||||
if url.endswith("sitemap3.xml"):
|
||||
raise SitemapDiscoveryError("broken sitemap")
|
||||
raise AssertionError(f"unexpected url: {url}")
|
||||
|
||||
with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
|
||||
result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
|
||||
|
||||
self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
|
||||
|
||||
def test_discover_vehicle_urls_falls_back_to_regex_loc_extraction(self) -> None:
|
||||
malformed_index = (
|
||||
b"<sitemapindex>"
|
||||
b"<sitemap><loc>https://www.iaai.com/sitemap1.xml</loc></sitemap>"
|
||||
b"<broken"
|
||||
)
|
||||
sitemap_vehicle = b"""
|
||||
<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
|
||||
<url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
|
||||
</urlset>
|
||||
"""
|
||||
|
||||
def _fake_fetch(url: str) -> SitemapFetchResult:
|
||||
if url.endswith("sitemap_index.xml"):
|
||||
return self._fetch_result(url, malformed_index)
|
||||
if url.endswith("sitemap1.xml"):
|
||||
return self._fetch_result(url, sitemap_vehicle)
|
||||
raise AssertionError(f"unexpected url: {url}")
|
||||
|
||||
with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
|
||||
result = discover_vehicle_urls_from_sitemap("https://www.iaai.com/sitemap_index.xml")
|
||||
|
||||
self.assertEqual(result, ["https://www.iaai.com/VehicleDetail/111~US"])
|
||||
|
||||
def test_discovery_uses_direct_probe_when_index_has_no_urls(self) -> None:
|
||||
index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
|
||||
sitemap_one = b"""
|
||||
<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
|
||||
<url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
|
||||
</urlset>
|
||||
"""
|
||||
|
||||
settings = Settings()
|
||||
settings.discovery.sitemap_direct_probe_limit = 2
|
||||
settings.discovery.sitemap_direct_probe_stop_after_misses = 1
|
||||
|
||||
def _fake_fetch(url: str) -> SitemapFetchResult:
|
||||
if url.endswith("sitemap_index.xml"):
|
||||
return self._fetch_result(url, index_xml)
|
||||
if url.endswith("sitemap1.xml"):
|
||||
return self._fetch_result(url, sitemap_one)
|
||||
raise SitemapDiscoveryError("not found")
|
||||
|
||||
with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
|
||||
result = discover_vehicle_urls_from_sitemap_with_stats(
|
||||
"https://www.iaai.com/sitemap_index.xml",
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
|
||||
self.assertEqual(result.stats.direct_probe_hits, 1)
|
||||
|
||||
def test_discovery_stats_report_direct_probe_hits(self) -> None:
|
||||
index_xml = b"<sitemapindex xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\"></sitemapindex>"
|
||||
sitemap1 = b"""
|
||||
<urlset xmlns=\"http://www.sitemaps.org/schemas/sitemap/0.9\">
|
||||
<url><loc>https://www.iaai.com/VehicleDetail/111~US</loc></url>
|
||||
</urlset>
|
||||
"""
|
||||
|
||||
def _fake_fetch(url: str):
|
||||
if url.endswith("sitemap_index.xml"):
|
||||
return self._fetch_result(url, index_xml)
|
||||
if url.endswith("sitemap1.xml"):
|
||||
return self._fetch_result(url, sitemap1)
|
||||
raise SitemapDiscoveryError("not found")
|
||||
|
||||
with patch("iaai_scraper.discovery.sitemap._SitemapDownloader.fetch", side_effect=_fake_fetch):
|
||||
result = discover_vehicle_urls_from_sitemap_with_stats("https://www.iaai.com/sitemap_index.xml")
|
||||
|
||||
self.assertEqual(result.vehicle_urls, ["https://www.iaai.com/VehicleDetail/111~US"])
|
||||
self.assertEqual(result.stats.transport, "curl_cffi")
|
||||
self.assertEqual(result.stats.direct_probe_hits, 1)
|
||||
self.assertGreaterEqual(result.stats.direct_probe_misses, 1)
|
||||
|
||||
def test_block_page_raises_discovery_error(self) -> None:
|
||||
settings = Settings()
|
||||
settings.discovery.sitemap_direct_probe_limit = 1
|
||||
settings.discovery.sitemap_direct_probe_stop_after_misses = 1
|
||||
|
||||
with patch(
|
||||
"iaai_scraper.discovery.sitemap._SitemapDownloader.fetch",
|
||||
side_effect=SitemapBlockedError("Anti-bot page returned for https://www.iaai.com/sitemap_index.xml"),
|
||||
):
|
||||
with self.assertRaises(SitemapDiscoveryError):
|
||||
discover_vehicle_urls_from_sitemap(
|
||||
"https://www.iaai.com/sitemap_index.xml",
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -6,7 +6,210 @@ from unittest.mock import MagicMock, patch
|
||||
from iaai_scraper.worker import tasks
|
||||
|
||||
|
||||
def make_settings_stub(*, parallel_segments: bool = False) -> MagicMock:
|
||||
settings_stub = MagicMock()
|
||||
settings_stub.celery.parallel_segments = parallel_segments
|
||||
settings_stub.celery.task_soft_time_limit = 3300
|
||||
settings_stub.celery.task_time_limit = 3600
|
||||
settings_stub.celery.task_stall_timeout_seconds = 600
|
||||
settings_stub.listing.listing_segments_json = "ignored"
|
||||
settings_stub.discovery.mode = "listing"
|
||||
settings_stub.discovery.hourly_mode = "rolling_refresh"
|
||||
return settings_stub
|
||||
|
||||
|
||||
class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
def test_sync_listing_task_uses_hourly_sitemap_even_when_mode_is_not_sitemap(self) -> None:
|
||||
settings_stub = make_settings_stub(parallel_segments=False)
|
||||
settings_stub.discovery.mode = "listing"
|
||||
settings_stub.discovery.hourly_mode = "rolling_refresh"
|
||||
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "Settings", return_value=settings_stub), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
|
||||
"status": "success",
|
||||
"run_id": None,
|
||||
"cars_upserted": 25,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 50,
|
||||
"skipped_existing": 100,
|
||||
"elapsed_seconds": None,
|
||||
"failures": [],
|
||||
"hourly_mode": "sitemap_rolling_refresh",
|
||||
"discovered_urls": 105,
|
||||
"new_urls": 5,
|
||||
"refresh_urls": 20,
|
||||
"sold_marked": 2,
|
||||
}) as hourly_sync, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-hourly-force-sitemap")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
|
||||
hourly_sync.assert_called_once()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_uses_hourly_sitemap_rolling_refresh_after_bootstrap(self) -> None:
|
||||
settings_stub = make_settings_stub(parallel_segments=False)
|
||||
settings_stub.discovery.mode = "sitemap"
|
||||
settings_stub.discovery.hourly_mode = "rolling_refresh"
|
||||
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "Settings", return_value=settings_stub), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
|
||||
"status": "success",
|
||||
"run_id": None,
|
||||
"cars_upserted": 25,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 50,
|
||||
"skipped_existing": 100,
|
||||
"elapsed_seconds": None,
|
||||
"failures": [],
|
||||
"hourly_mode": "sitemap_rolling_refresh",
|
||||
"discovered_urls": 105,
|
||||
"new_urls": 5,
|
||||
"refresh_urls": 20,
|
||||
"sold_marked": 2,
|
||||
}) as hourly_sync, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-hourly")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
|
||||
hourly_sync.assert_called_once()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_can_use_hourly_sitemap_diff_when_configured(self) -> None:
|
||||
settings_stub = make_settings_stub(parallel_segments=False)
|
||||
settings_stub.discovery.mode = "sitemap"
|
||||
settings_stub.discovery.hourly_mode = "diff"
|
||||
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "Settings", return_value=settings_stub), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_hourly_sitemap_diff_sync", return_value={
|
||||
"status": "success",
|
||||
"run_id": None,
|
||||
"cars_upserted": 5,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 10,
|
||||
"skipped_existing": 100,
|
||||
"elapsed_seconds": None,
|
||||
"failures": [],
|
||||
"hourly_mode": "sitemap_diff",
|
||||
"discovered_urls": 105,
|
||||
"new_urls": 5,
|
||||
"sold_marked": 2,
|
||||
}) as hourly_sync, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-hourly-diff")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(result["hourly_mode"], "sitemap_diff")
|
||||
hourly_sync.assert_called_once()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_forces_sitemap_full_scan_in_bootstrap(self) -> None:
|
||||
settings_stub = make_settings_stub(parallel_segments=False)
|
||||
settings_stub.discovery.mode = "listing"
|
||||
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "Settings", return_value=settings_stub), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[{"make": "HONDA"}]):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_listing_mock = MagicMock(return_value={
|
||||
"run_id": 99,
|
||||
"status": "success",
|
||||
"full_scan_completed": True,
|
||||
"cars_upserted": 10,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 20,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [],
|
||||
})
|
||||
sync_listing_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
|
||||
scraper = MagicMock()
|
||||
scraper.sync_listing = sync_listing_mock
|
||||
scraper.sync_listing_segmented = sync_listing_segmented_mock
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value = scraper
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-bootstrap-sitemap")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
sync_listing_mock.assert_called_once_with(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
only_new=False,
|
||||
)
|
||||
sync_listing_segmented_mock.assert_not_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_lock_acquire_refresh_release(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
|
||||
@@ -175,15 +378,17 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
|
||||
redis_client.delete.assert_not_called()
|
||||
|
||||
def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
|
||||
def test_sync_listing_bootstrap_prefers_sitemap_over_segment_resume(self) -> None:
|
||||
segments = [
|
||||
{"make": "ACURA"},
|
||||
{"make": "AUDI"},
|
||||
{"make": "BMW"},
|
||||
{"make": "EAGLE"},
|
||||
]
|
||||
settings_stub = make_settings_stub()
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "Settings", return_value=settings_stub), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
@@ -199,13 +404,15 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
|
||||
sync_listing_mock = MagicMock(return_value={
|
||||
"run_id": 11, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
@@ -216,21 +423,42 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
# last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
|
||||
sync_listing_mock.assert_called_once_with(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
only_new=False,
|
||||
)
|
||||
sync_segmented_mock.assert_not_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
|
||||
def test_sync_listing_hourly_path_ignores_checkpoint_after_full_scan_completed(self) -> None:
|
||||
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
|
||||
settings_stub = make_settings_stub()
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "Settings", return_value=settings_stub), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
|
||||
"status": "success",
|
||||
"run_id": None,
|
||||
"cars_upserted": 1,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": None,
|
||||
"failures": [],
|
||||
"hourly_mode": "sitemap_rolling_refresh",
|
||||
"discovered_urls": 10,
|
||||
"new_urls": 1,
|
||||
"refresh_urls": 1,
|
||||
"sold_marked": 0,
|
||||
}) as hourly_sync, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
@@ -242,32 +470,24 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
"run_id": 14, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-792")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
tasks.sync_listing_task.push_request(id="task-792")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
|
||||
self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
|
||||
self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
|
||||
hourly_sync.assert_called_once()
|
||||
clear_checkpoint.assert_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
|
||||
def test_sync_listing_bootstrap_ignores_beyond_segment_checkpoint(self) -> None:
|
||||
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
|
||||
settings_stub = make_settings_stub()
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "Settings", return_value=settings_stub), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
@@ -283,13 +503,15 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
|
||||
sync_listing_mock = MagicMock(return_value={
|
||||
"run_id": 15, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
@@ -300,7 +522,14 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
|
||||
sync_listing_mock.assert_called_once_with(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
only_new=False,
|
||||
)
|
||||
sync_segmented_mock.assert_not_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
|
||||
@@ -377,28 +606,38 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
self.assertTrue(should_enqueue)
|
||||
|
||||
def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
|
||||
settings_stub = make_settings_stub(parallel_segments=False)
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
|
||||
patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
|
||||
patch.object(tasks, "_clear_followup_pending") as clear_pending, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 99,
|
||||
"status": "failed",
|
||||
"full_scan_completed": False,
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [{"vehicle_url": "listing", "error": "bad resume"}],
|
||||
"listing": {"vehicles_collected": 0},
|
||||
}):
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "Settings", return_value=settings_stub), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
|
||||
patch.object(
|
||||
tasks,
|
||||
"_bump_bootstrap_failure_streak",
|
||||
return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False),
|
||||
) as bump_streak, \
|
||||
patch.object(tasks, "_clear_followup_pending") as clear_pending, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(
|
||||
tasks,
|
||||
"_run_browser_job",
|
||||
return_value={
|
||||
"run_id": 99,
|
||||
"status": "failed",
|
||||
"full_scan_completed": False,
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [{"vehicle_url": "listing", "error": "bad resume"}],
|
||||
"listing": {"vehicles_collected": 0},
|
||||
},
|
||||
):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
|
||||
Reference in New Issue
Block a user