qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update scraper package

Browse Source
This commit is contained in:
qananasikq
2026-04-24 20:46:58 +03:00
parent 6aba4f0dbd
commit d484088e18
59 changed files with 2548 additions and 1272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

76
tests/test_scraper.py
View File

@@ -5,29 +5,29 @@ from concurrent.futures import TimeoutError as FuturesTimeoutError
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from iaai_scraper.core.config import Settings
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.storage.schemas import CarRecord
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from dubizzle_scraper.scraper import DUBIZZLEScraper
from dubizzle_scraper.storage.schemas import CarRecord
def make_db_record(origin_id: str) -> dict[str, object]:
return CarRecord(
parser_id=f"iaai:{origin_id}",
parser_id=f"dubizzle:{origin_id}",
brand="Toyota",
model="Camry",
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id,
slug=f"toyota-camry-{origin_id}",
).model_dump(mode="json")
class TestScraperSync(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper:
def _make_scraper(self) -> DUBIZZLEScraper:
s = Settings()
s.log_level = "CRITICAL"
s.database.url = "sqlite://"
return IAAIScraper(s)
return DUBIZZLEScraper(s)
def test_sync_vehicle_uses_db_record(self) -> None:
scraper = self._make_scraper()
@@ -37,7 +37,7 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
result = scraper.sync_vehicle("https://www.dubizzle.com/VehicleDetail/111~US")
self.assertEqual(result["status"], "success")
self.assertIn("trace_id", result)
@@ -50,13 +50,13 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.start_sync_run = MagicMock(return_value=2)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
{"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
{"https://www.dubizzle.com/VehicleDetail/111~US"}, {"dubizzle:222"},
))
scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [
"https://www.iaai.com/VehicleDetail/111~US",
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
"https://www.dubizzle.com/VehicleDetail/111~US",
"https://www.dubizzle.com/VehicleDetail/222~US",
"https://www.dubizzle.com/VehicleDetail/333~US",
]
})
scraper.sync_batch = MagicMock(return_value={
@@ -79,7 +79,7 @@ class TestScraperSync(unittest.TestCase):
})
result2 = scraper2.sync_listing(
only_new=True,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
year_min=2020, year_max=2027,
)
self.assertEqual(result2["cars_upserted"], 10)
@@ -155,35 +155,35 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["status"], "partial_success")
def test_build_segment_listing_url(self) -> None:
base = "https://www.iaai.com/Vehiclelisting/Cars"
base = "https://www.dubizzle.com/Vehiclelisting/Cars"
self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
DUBIZZLEScraper._build_segment_listing_url(base, "TOYOTA"),
"https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
)
self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
DUBIZZLEScraper._build_segment_listing_url(base, "LAND ROVER"),
"https://www.dubizzle.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
)
self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, None), base)
self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, ""), base)
def test_guard_and_protection_detection(self) -> None:
with self.assertRaises(AntiBotDetectedError):
IAAIScraper._raise_if_blocked_or_incomplete(
DUBIZZLEScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
"access_notes": {}, "vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US",
"https://www.dubizzle.com/VehicleDetail/999~US",
)
with self.assertRaises(SiteStructureChangedError):
IAAIScraper._raise_if_blocked_or_incomplete(
DUBIZZLEScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
"access_notes": {"possible_captcha": False, "possible_antibot": False},
"vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US",
"https://www.dubizzle.com/VehicleDetail/999~US",
)
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
self.assertFalse(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
def test_close_resets_browser_state(self) -> None:
scraper = self._make_scraper()
@@ -201,7 +201,7 @@ class TestScraperSync(unittest.TestCase):
scraper = self._make_scraper()
page = MagicMock()
page_result = SimpleNamespace(
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/123~US")],
)
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper.listing_collector.open_cars_listing = MagicMock()
@@ -228,7 +228,7 @@ class TestScraperSync(unittest.TestCase):
page = MagicMock()
page_result = SimpleNamespace(
page_number=1,
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/999~US", lot_number="999")],
next_page_detected=False,
)
@@ -245,7 +245,7 @@ class TestScraperSync(unittest.TestCase):
"year_max": None,
})
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
scraper._extract_page_urls = MagicMock(return_value=["https://www.dubizzle.com/VehicleDetail/999~US"])
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 1,
"cars_failed": 0,
@@ -256,11 +256,11 @@ class TestScraperSync(unittest.TestCase):
result = scraper._sync_listing_streaming(
make=None,
model=None,
lane="iaai_cars",
lane="dubizzle_cars",
limit=None,
effective_only_new=False,
started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=EAGLE",
)
scraper.listing_collector.open_cars_listing.assert_called_once()
@@ -278,9 +278,9 @@ class TestScraperSync(unittest.TestCase):
})
urls = [
"https://www.iaai.com/VehicleDetail/111~US",
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
"https://www.dubizzle.com/VehicleDetail/111~US",
"https://www.dubizzle.com/VehicleDetail/222~US",
"https://www.dubizzle.com/VehicleDetail/333~US",
]
first_record = CarRecord.model_validate(make_db_record("111"))
@@ -298,8 +298,8 @@ class TestScraperSync(unittest.TestCase):
yield 0, first_record
raise FuturesTimeoutError()
with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
with patch("dubizzle_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
patch("dubizzle_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
patch.object(scraper, "_browser_fallback_parallel", return_value={
"records": [],
"failures": [],