remove iaai scraper and old tests
This commit is contained in:
qananasikq
@@ -1,36 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from iaai_scraper.browser.pace import HumanPacer
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.browser.listing import ListingCollector
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self, counts: dict[str, int]) -> None:
|
||||
self._counts = counts
|
||||
|
||||
class _Locator:
|
||||
def __init__(self, count_value: int) -> None:
|
||||
self._count_value = count_value
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count_value
|
||||
|
||||
def locator(self, selector: str) -> "_FakePage._Locator":
|
||||
return _FakePage._Locator(self._counts.get(selector, 0))
|
||||
|
||||
|
||||
class TestListingUnit(unittest.TestCase):
|
||||
def test_has_next_page_true_for_known_selector(self) -> None:
|
||||
page = _FakePage({"a[aria-label*='Next']": 1})
|
||||
self.assertTrue(ListingCollector._has_next_page(page))
|
||||
|
||||
def test_has_next_page_false_when_no_selectors(self) -> None:
|
||||
page = _FakePage({})
|
||||
self.assertFalse(ListingCollector._has_next_page(page))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,100 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from iaai_scraper.parsing.mapper import CarMapper
|
||||
|
||||
|
||||
class TestCarMapper(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.mapper = CarMapper()
|
||||
|
||||
def test_deduplicates_images_by_image_key(self) -> None:
|
||||
urls = [
|
||||
"https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
|
||||
"https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
|
||||
"https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
|
||||
]
|
||||
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
|
||||
vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
|
||||
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
|
||||
)
|
||||
|
||||
self.assertEqual(len(record.images), 2)
|
||||
self.assertIn("width=845", record.images[0].fullres_image)
|
||||
self.assertIn("height=633", record.images[0].fullres_image)
|
||||
|
||||
def test_no_damage_marker_is_not_damaged(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
|
||||
vehicle_summary={"make": "Ford", "model": "Focus"},
|
||||
payload_insights={
|
||||
"vehicle_core": {},
|
||||
"pricing": {},
|
||||
"damage": {"primary": "normal wear"},
|
||||
"auction": {},
|
||||
"images": {},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(record.is_damaged)
|
||||
|
||||
def test_unknown_and_empty_values_fallbacks(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
|
||||
vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
|
||||
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
|
||||
)
|
||||
|
||||
self.assertEqual(record.brand, "UNKNOWN")
|
||||
self.assertEqual(record.model, "UNKNOWN")
|
||||
self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None)
|
||||
self.assertEqual(record.drive, "NA")
|
||||
self.assertEqual(record.gearbox, "NA")
|
||||
|
||||
def test_mapper_handles_case_and_spaces(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
|
||||
vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
|
||||
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
|
||||
)
|
||||
|
||||
self.assertEqual(record.drive, "FWD")
|
||||
self.assertEqual(record.gearbox, "AT")
|
||||
|
||||
def test_price_parsing_dirty_formats(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
|
||||
vehicle_summary={"make": "Toyota", "model": "Corolla"},
|
||||
payload_insights={
|
||||
"vehicle_core": {},
|
||||
"pricing": {"buy_now": "USD 4,500 - 5,200"},
|
||||
"damage": {},
|
||||
"auction": {},
|
||||
"images": {},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(record.price, 5200)
|
||||
|
||||
def test_currency_detection_from_symbol(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
|
||||
vehicle_summary={"make": "Toyota", "model": "Corolla"},
|
||||
payload_insights={
|
||||
"vehicle_core": {},
|
||||
"pricing": {"buy_now": "€4.500,00"},
|
||||
"damage": {},
|
||||
"auction": {},
|
||||
"images": {},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(record.currency, "EUR")
|
||||
self.assertEqual(record.price, 4500)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,68 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from iaai_scraper.parsing.parser import VehicleParser
|
||||
|
||||
|
||||
class TestVehicleParserUnit(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.parser = VehicleParser()
|
||||
|
||||
def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None:
|
||||
dom_text = """
|
||||
Stock #:
|
||||
45089484
|
||||
Primary Damage:
|
||||
Front End
|
||||
Odometer:
|
||||
50,123 mi (Actual)
|
||||
"""
|
||||
result = self.parser._parse_dom_key_value_pairs(dom_text)
|
||||
self.assertEqual(result.get("lot_number"), "45089484")
|
||||
self.assertEqual(result.get("primary_damage"), "Front End")
|
||||
self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
|
||||
|
||||
def test_parse_title_for_year_make_model(self) -> None:
|
||||
parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
|
||||
self.assertEqual(parsed["year"], "2014")
|
||||
self.assertEqual(parsed["make"], "TOYOTA")
|
||||
self.assertEqual(parsed["model"], "CAMRY")
|
||||
|
||||
def test_extract_image_urls_filters_other_vehicle(self) -> None:
|
||||
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
|
||||
payloads = [
|
||||
{
|
||||
"imageUrls": [
|
||||
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
||||
"https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
|
||||
]
|
||||
}
|
||||
]
|
||||
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
|
||||
self.assertEqual(len(urls), 1)
|
||||
self.assertIn("45089484", urls[0])
|
||||
|
||||
def test_extract_image_urls_deduplicates_same_url(self) -> None:
|
||||
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
|
||||
payloads = [{"imageUrls": [
|
||||
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
||||
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
||||
]}]
|
||||
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
|
||||
self.assertEqual(len(urls), 1)
|
||||
|
||||
def test_dom_hints_detect_captcha_and_antibot(self) -> None:
|
||||
hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
|
||||
self.assertTrue(hints["has_captcha_text"])
|
||||
self.assertTrue(hints["has_antibot_text"])
|
||||
|
||||
def test_access_notes_reflect_antibot_signals(self) -> None:
|
||||
summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
|
||||
notes = self.parser._build_access_notes(summary, [])
|
||||
self.assertTrue(notes["possible_captcha"])
|
||||
self.assertTrue(notes["possible_antibot"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,174 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
|
||||
from iaai_scraper.scraper import IAAIScraper
|
||||
from iaai_scraper.storage.schemas import CarRecord
|
||||
|
||||
|
||||
def make_db_record(origin_id: str) -> dict[str, object]:
|
||||
return CarRecord(
|
||||
parser_id=f"iaai:{origin_id}",
|
||||
brand="Toyota",
|
||||
model="Camry",
|
||||
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
|
||||
origin_id=origin_id,
|
||||
slug=f"toyota-camry-{origin_id}",
|
||||
).model_dump(mode="json")
|
||||
|
||||
|
||||
class TestScraperSync(unittest.TestCase):
|
||||
def _make_scraper(self) -> IAAIScraper:
|
||||
s = Settings()
|
||||
s.log_level = "CRITICAL"
|
||||
s.database.url = "sqlite://"
|
||||
return IAAIScraper(s)
|
||||
|
||||
def test_sync_vehicle_uses_db_record_without_remapping(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=1)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
|
||||
|
||||
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
|
||||
scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
|
||||
|
||||
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertIn("trace_id", result)
|
||||
self.assertIn("elapsed_seconds", result)
|
||||
self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
|
||||
|
||||
def test_sync_listing_uses_db_record_without_remapping(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=2)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set()))
|
||||
|
||||
scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
|
||||
})
|
||||
|
||||
result = scraper.sync_listing()
|
||||
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
self.assertEqual(result["cars_failed"], 0)
|
||||
self.assertIn("trace_id", result)
|
||||
self.assertIn("elapsed_seconds", result)
|
||||
scraper.sync_batch.assert_called_once()
|
||||
|
||||
def test_sync_listing_respects_limit(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=3)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
|
||||
# Проверка пути only_new с limit.
|
||||
scraper._collect_listing_iterative = MagicMock(return_value=(
|
||||
["https://www.iaai.com/VehicleDetail/222~US"],
|
||||
["https://www.iaai.com/VehicleDetail/222~US",
|
||||
"https://www.iaai.com/VehicleDetail/333~US"],
|
||||
{"vehicle_urls": [], "pages_collected": 1, "early_stopped": False},
|
||||
0,
|
||||
))
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
|
||||
})
|
||||
|
||||
scraper.sync_listing(limit=1)
|
||||
|
||||
# Должен уйти только один URL.
|
||||
scraper.sync_batch.assert_called_once()
|
||||
batch_urls = scraper.sync_batch.call_args[0][0]
|
||||
self.assertEqual(len(batch_urls), 1)
|
||||
|
||||
def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=5)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
|
||||
{"https://www.iaai.com/VehicleDetail/111~US"},
|
||||
{"iaai:222"},
|
||||
))
|
||||
|
||||
scraper.collect_listing = MagicMock(return_value={
|
||||
"vehicle_urls": [
|
||||
"https://www.iaai.com/VehicleDetail/111~US", # exists by URL
|
||||
"https://www.iaai.com/VehicleDetail/222~US", # exists by ID
|
||||
"https://www.iaai.com/VehicleDetail/333~US", # new
|
||||
]
|
||||
})
|
||||
# Возвращаем результат для одного нового авто.
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
|
||||
})
|
||||
|
||||
result = scraper.sync_listing(only_new=True)
|
||||
|
||||
self.assertEqual(result["skipped_existing"], 2)
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
# В batch должен попасть только новый URL.
|
||||
scraper.sync_batch.assert_called_once()
|
||||
batch_urls = scraper.sync_batch.call_args[0][0]
|
||||
self.assertEqual(len(batch_urls), 1)
|
||||
self.assertIn("333", batch_urls[0])
|
||||
scraper.persistence.get_existing_urls_and_ids.assert_called_once()
|
||||
|
||||
def test_close_resets_browser_state(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
http_pool = MagicMock()
|
||||
scraper._http_pool = http_pool
|
||||
scraper.context = MagicMock()
|
||||
scraper.browser = MagicMock()
|
||||
scraper.playwright = MagicMock()
|
||||
|
||||
scraper.close()
|
||||
|
||||
http_pool.clear.assert_called_once()
|
||||
self.assertIsNone(scraper._http_pool)
|
||||
self.assertIsNone(scraper.context)
|
||||
self.assertIsNone(scraper.browser)
|
||||
self.assertIsNone(scraper.playwright)
|
||||
|
||||
def test_guard_raises_on_antibot_signals(self) -> None:
|
||||
with self.assertRaises(AntiBotDetectedError):
|
||||
IAAIScraper._raise_if_blocked_or_incomplete(
|
||||
{
|
||||
"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
|
||||
"access_notes": {},
|
||||
"vehicle_summary": {},
|
||||
},
|
||||
"https://www.iaai.com/VehicleDetail/999~US",
|
||||
)
|
||||
|
||||
def test_guard_raises_on_empty_vehicle_page(self) -> None:
|
||||
with self.assertRaises(SiteStructureChangedError):
|
||||
IAAIScraper._raise_if_blocked_or_incomplete(
|
||||
{
|
||||
"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
|
||||
"access_notes": {"possible_captcha": False, "possible_antibot": False},
|
||||
"vehicle_summary": {},
|
||||
},
|
||||
"https://www.iaai.com/VehicleDetail/999~US",
|
||||
)
|
||||
|
||||
def test_is_protection_or_network_error_detects_known_signals(self) -> None:
|
||||
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
|
||||
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
|
||||
self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user