220 lines
9.0 KiB
Python
220 lines
9.0 KiB
Python
from __future__ import annotations
|
||
|
||
import unittest
|
||
from types import SimpleNamespace
|
||
from unittest.mock import MagicMock
|
||
|
||
from iaai_scraper.core.config import Settings
|
||
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
|
||
from iaai_scraper.scraper import IAAIScraper
|
||
from iaai_scraper.storage.schemas import CarRecord
|
||
|
||
|
||
def make_db_record(origin_id: str) -> dict[str, object]:
|
||
return CarRecord(
|
||
parser_id=f"iaai:{origin_id}",
|
||
brand="Toyota",
|
||
model="Camry",
|
||
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
|
||
origin_id=origin_id,
|
||
slug=f"toyota-camry-{origin_id}",
|
||
).model_dump(mode="json")
|
||
|
||
|
||
class TestScraperSync(unittest.TestCase):
|
||
def _make_scraper(self) -> IAAIScraper:
|
||
s = Settings()
|
||
s.log_level = "CRITICAL"
|
||
s.database.url = "sqlite://"
|
||
return IAAIScraper(s)
|
||
|
||
def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
|
||
scraper = self._make_scraper()
|
||
|
||
scraper.persistence.create_tables = MagicMock()
|
||
scraper.persistence.start_sync_run = MagicMock(return_value=1)
|
||
scraper.persistence.finish_sync_run = MagicMock()
|
||
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
|
||
|
||
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
|
||
scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
|
||
|
||
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
|
||
|
||
self.assertEqual(result["status"], "success")
|
||
self.assertIn("trace_id", result)
|
||
self.assertIn("elapsed_seconds", result)
|
||
self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
|
||
|
||
def test_sync_listing_uses_db_record_without_remapping(self) -> None:
|
||
scraper = self._make_scraper()
|
||
|
||
scraper.persistence.create_tables = MagicMock()
|
||
scraper.persistence.start_sync_run = MagicMock(return_value=2)
|
||
scraper.persistence.finish_sync_run = MagicMock()
|
||
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set()))
|
||
|
||
scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
|
||
scraper.sync_batch = MagicMock(return_value={
|
||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
|
||
})
|
||
|
||
result = scraper.sync_listing()
|
||
|
||
self.assertEqual(result["cars_upserted"], 1)
|
||
self.assertEqual(result["cars_failed"], 0)
|
||
self.assertIn("trace_id", result)
|
||
self.assertIn("elapsed_seconds", result)
|
||
scraper.sync_batch.assert_called_once()
|
||
|
||
def test_sync_listing_respects_limit(self) -> None:
|
||
scraper = self._make_scraper()
|
||
|
||
scraper.persistence.create_tables = MagicMock()
|
||
scraper.persistence.start_sync_run = MagicMock(return_value=3)
|
||
scraper.persistence.finish_sync_run = MagicMock()
|
||
|
||
# Проверка пути only_new с limit.
|
||
scraper._collect_listing_iterative = MagicMock(return_value=(
|
||
["https://www.iaai.com/VehicleDetail/222~US"],
|
||
["https://www.iaai.com/VehicleDetail/222~US",
|
||
"https://www.iaai.com/VehicleDetail/333~US"],
|
||
{"vehicle_urls": [], "pages_collected": 1, "early_stopped": False},
|
||
0,
|
||
))
|
||
scraper.sync_batch = MagicMock(return_value={
|
||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
|
||
})
|
||
|
||
scraper.sync_listing(limit=1)
|
||
|
||
# Должен уйти только один URL.
|
||
scraper.sync_batch.assert_called_once()
|
||
batch_urls = scraper.sync_batch.call_args[0][0]
|
||
self.assertEqual(len(batch_urls), 1)
|
||
|
||
def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
|
||
scraper = self._make_scraper()
|
||
|
||
scraper.persistence.create_tables = MagicMock()
|
||
scraper.persistence.start_sync_run = MagicMock(return_value=5)
|
||
scraper.persistence.finish_sync_run = MagicMock()
|
||
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
|
||
{"https://www.iaai.com/VehicleDetail/111~US"},
|
||
{"iaai:222"},
|
||
))
|
||
|
||
scraper.collect_listing = MagicMock(return_value={
|
||
"vehicle_urls": [
|
||
"https://www.iaai.com/VehicleDetail/111~US", # exists by URL
|
||
"https://www.iaai.com/VehicleDetail/222~US", # exists by ID
|
||
"https://www.iaai.com/VehicleDetail/333~US", # new
|
||
]
|
||
})
|
||
# Возвращаем результат для одного нового авто.
|
||
scraper.sync_batch = MagicMock(return_value={
|
||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
|
||
})
|
||
|
||
result = scraper.sync_listing(only_new=True)
|
||
|
||
self.assertEqual(result["skipped_existing"], 2)
|
||
self.assertEqual(result["cars_upserted"], 1)
|
||
# В batch должен попасть только новый URL.
|
||
scraper.sync_batch.assert_called_once()
|
||
batch_urls = scraper.sync_batch.call_args[0][0]
|
||
self.assertEqual(len(batch_urls), 1)
|
||
self.assertIn("333", batch_urls[0])
|
||
scraper.persistence.get_existing_urls_and_ids.assert_called_once()
|
||
|
||
def test_close_resets_browser_state(self) -> None:
|
||
scraper = self._make_scraper()
|
||
http_pool = MagicMock()
|
||
scraper._http_pool = http_pool
|
||
scraper.context = MagicMock()
|
||
scraper.browser = MagicMock()
|
||
scraper.playwright = MagicMock()
|
||
|
||
scraper.close()
|
||
|
||
http_pool.clear.assert_called_once()
|
||
self.assertIsNone(scraper._http_pool)
|
||
self.assertIsNone(scraper.context)
|
||
self.assertIsNone(scraper.browser)
|
||
self.assertIsNone(scraper.playwright)
|
||
|
||
def test_guard_raises_on_antibot_signals(self) -> None:
|
||
with self.assertRaises(AntiBotDetectedError):
|
||
IAAIScraper._raise_if_blocked_or_incomplete(
|
||
{
|
||
"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
|
||
"access_notes": {},
|
||
"vehicle_summary": {},
|
||
},
|
||
"https://www.iaai.com/VehicleDetail/999~US",
|
||
)
|
||
|
||
def test_guard_raises_on_empty_vehicle_page(self) -> None:
|
||
with self.assertRaises(SiteStructureChangedError):
|
||
IAAIScraper._raise_if_blocked_or_incomplete(
|
||
{
|
||
"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
|
||
"access_notes": {"possible_captcha": False, "possible_antibot": False},
|
||
"vehicle_summary": {},
|
||
},
|
||
"https://www.iaai.com/VehicleDetail/999~US",
|
||
)
|
||
|
||
def test_is_protection_or_network_error_detects_known_signals(self) -> None:
|
||
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
|
||
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
|
||
self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
|
||
|
||
def test_recover_empty_listing_page_reload_recovers_links(self) -> None:
|
||
scraper = self._make_scraper()
|
||
page = MagicMock()
|
||
page_result = SimpleNamespace(
|
||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
|
||
)
|
||
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
|
||
|
||
all_raw_urls: list[str] = []
|
||
seen_urls: set[str] = set()
|
||
|
||
recovered_result, recovered_urls = scraper._recover_empty_listing_page(
|
||
page,
|
||
page_number=5,
|
||
all_raw_urls=all_raw_urls,
|
||
seen_urls=seen_urls,
|
||
)
|
||
|
||
page.reload.assert_called_once()
|
||
self.assertIs(recovered_result, page_result)
|
||
self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/123~US"])
|
||
self.assertEqual(all_raw_urls, ["https://www.iaai.com/VehicleDetail/123~US"])
|
||
|
||
def test_recover_empty_listing_page_reopens_listing_for_first_page(self) -> None:
|
||
scraper = self._make_scraper()
|
||
page = MagicMock()
|
||
page_result = SimpleNamespace(
|
||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/456~US")],
|
||
)
|
||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
|
||
|
||
recovered_result, recovered_urls = scraper._recover_empty_listing_page(
|
||
page,
|
||
page_number=1,
|
||
all_raw_urls=[],
|
||
seen_urls=set(),
|
||
)
|
||
|
||
scraper.listing_collector.open_cars_listing.assert_called_once_with(page)
|
||
page.reload.assert_not_called()
|
||
self.assertIs(recovered_result, page_result)
|
||
self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/456~US"])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|