add mobilede tests
This commit is contained in:
@@ -1,4 +1,4 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
@@ -6,10 +6,10 @@ from pathlib import Path
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.storage.db import PersistenceService
|
||||
from iaai_scraper.storage.models import Car, Image, SyncRun
|
||||
from iaai_scraper.storage.schemas import CarRecord, ImageRecord
|
||||
from mobilede_scraper.core.config import Settings
|
||||
from mobilede_scraper.storage.db import PersistenceService
|
||||
from mobilede_scraper.storage.models import Car, Image, SyncRun
|
||||
from mobilede_scraper.storage.schemas import CarRecord, ImageRecord
|
||||
|
||||
|
||||
class TestPersistenceServiceIntegration(unittest.TestCase):
|
||||
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
|
||||
return CarRecord(
|
||||
parser_id=f"iaai:{origin_id}",
|
||||
parser_id=f"mobilede:{origin_id}",
|
||||
brand="Toyota",
|
||||
model="Camry",
|
||||
year=2014,
|
||||
price=price,
|
||||
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
|
||||
origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
|
||||
origin_id=origin_id,
|
||||
slug=f"toyota-camry-{origin_id}",
|
||||
images=[
|
||||
ImageRecord(
|
||||
fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
|
||||
preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
|
||||
fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
|
||||
preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=400&height=300",
|
||||
order_index=0,
|
||||
)
|
||||
],
|
||||
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
|
||||
second = self._record("888")
|
||||
second.images = [
|
||||
ImageRecord(
|
||||
fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
|
||||
preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
|
||||
fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=845&height=633",
|
||||
preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300",
|
||||
order_index=0,
|
||||
)
|
||||
]
|
||||
@@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
|
||||
|
||||
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
|
||||
first = self._record("OLD-ID")
|
||||
first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
|
||||
first.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
|
||||
self.persistence.upsert_car(first)
|
||||
|
||||
second = self._record("NEW-ID")
|
||||
second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
|
||||
second.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
|
||||
result = self.persistence.upsert_car(second)
|
||||
|
||||
self.assertEqual(result["action"], "updated")
|
||||
@@ -125,3 +125,4 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from iaai_scraper.browser.fast_client import (
|
||||
from mobilede_scraper.browser.fast_client import (
|
||||
DETAIL_MARKER,
|
||||
LISTING_MARKER,
|
||||
build_resizer_images_from_keys,
|
||||
@@ -10,7 +10,7 @@ from iaai_scraper.browser.fast_client import (
|
||||
parse_listing_page,
|
||||
parse_product_details_vm,
|
||||
)
|
||||
from iaai_scraper.parsing.fast_mapper import FastCarMapper
|
||||
from mobilede_scraper.parsing.fast_mapper import FastCarMapper
|
||||
|
||||
|
||||
def test_parse_listing_page_extracts_hidden_payloads() -> None:
|
||||
@@ -78,10 +78,10 @@ def test_parse_product_details_vm_and_map_record() -> None:
|
||||
parsed = parse_product_details_vm(html)
|
||||
record = FastCarMapper().map_payload_to_record(
|
||||
detail_payload=parsed,
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/45078011~US",
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/45078011~US",
|
||||
)
|
||||
|
||||
assert record.origin_id == "iaai:45078011~US"
|
||||
assert record.origin_id == "mobilede:45078011"
|
||||
assert record.brand == "ACURA"
|
||||
assert record.model == "RSX BASE"
|
||||
assert record.year == 2002
|
||||
@@ -94,6 +94,51 @@ def test_parse_product_details_vm_and_map_record() -> None:
|
||||
assert len(record.images) == 1
|
||||
|
||||
|
||||
def test_fast_mapper_uses_fallback_keys_and_normalization() -> None:
|
||||
payload = {
|
||||
"inventoryView": {
|
||||
"attributes": {
|
||||
"Id": "454502861",
|
||||
"FirstRegistration": "05/2012",
|
||||
"Make": "BMW",
|
||||
"Model": "X1",
|
||||
"Variant": "xDrive 20d",
|
||||
"Currency": "EUR",
|
||||
"Mileage": "50.100 km",
|
||||
"ColorDesc": "silber metallic",
|
||||
"Category": "Kombi",
|
||||
"TransmissionType": "Schaltgetriebe",
|
||||
"EngineInformation": "1995 ccm, Diesel",
|
||||
},
|
||||
"imageDimensions": {
|
||||
"keys": {
|
||||
"$values": [
|
||||
{"k": "454502861~I1", "w": 1600, "h": 1200, "i": 0},
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
"auctionInformation": {
|
||||
"biddingInformation": {"buyNowPrice": "9.900 €"},
|
||||
},
|
||||
}
|
||||
|
||||
record = FastCarMapper().map_payload_to_record(
|
||||
detail_payload=payload,
|
||||
vehicle_url="https://www.mobile.de/ru/транспортные-средства/подробности.html?id=454502861&vc=Car&s=Car",
|
||||
)
|
||||
|
||||
assert record.model == "X1 xDrive 20d"
|
||||
assert record.year == 2012
|
||||
assert record.price == 9900
|
||||
assert record.mileage == 50100
|
||||
assert record.color == "silver"
|
||||
assert record.drive == "4WD"
|
||||
assert record.gearbox == "MT"
|
||||
assert record.body_type == "STATION_WAGON"
|
||||
assert record.engine_volume == 1995
|
||||
|
||||
|
||||
def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None:
|
||||
keys = [
|
||||
{"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
|
||||
|
||||
@@ -1,12 +1,13 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
from iaai_scraper.browser.fast_client import FastListingVehicle
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig
|
||||
from iaai_scraper.fast_sync import FastSyncEngine
|
||||
from iaai_scraper.parsing.fast_mapper import FastCarMapper
|
||||
from mobilede_scraper.browser.fast_client import FastListingVehicle
|
||||
from mobilede_scraper.core.config import Settings
|
||||
from mobilede_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig
|
||||
from mobilede_scraper.fast_sync import FastSyncEngine
|
||||
from mobilede_scraper.parsing.fast_mapper import FastCarMapper
|
||||
|
||||
|
||||
def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle:
|
||||
@@ -54,12 +55,46 @@ def _detail_payload(inventory_id: str, *, make: str = "ACURA", model: str = "RSX
|
||||
}
|
||||
|
||||
|
||||
def _detail_payload_with_fallbacks(inventory_id: str) -> dict:
|
||||
return {
|
||||
"inventoryView": {
|
||||
"attributes": {
|
||||
"Id": inventory_id,
|
||||
"FirstRegistration": "2011-03",
|
||||
"Make": "BMW",
|
||||
"Model": "120",
|
||||
"Variant": "Cabrio",
|
||||
"Currency": "EUR",
|
||||
"Kilometerstand": "95 000",
|
||||
"Color": "weiss",
|
||||
"DriveType": "Front wheel drive",
|
||||
"Gearbox": "Automatik",
|
||||
"VehicleClass": "Cabrio",
|
||||
"EngineInformation": "2,0 l",
|
||||
"VehicleGrade": "A",
|
||||
},
|
||||
"imageDimensions": {
|
||||
"keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]}
|
||||
},
|
||||
},
|
||||
"auctionInformation": {
|
||||
"biddingInformation": {"buyNowPrice": "8.899 €"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class FakeFastClient:
|
||||
def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None:
|
||||
self.listing = listing
|
||||
self.details = details
|
||||
self.detail_calls = 0
|
||||
self.persist_calls = 0
|
||||
self._settings = SimpleNamespace(
|
||||
scraping_profile=SimpleNamespace(
|
||||
verbose_progress_logs=False,
|
||||
request_jitter_max_s=0.0,
|
||||
)
|
||||
)
|
||||
|
||||
def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None): # noqa: ANN001, ANN202
|
||||
del listing_start_url, make, max_pages
|
||||
@@ -87,7 +122,7 @@ class FakePersistence:
|
||||
self.images += sum(len(record.images) for record in records)
|
||||
return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)}
|
||||
|
||||
def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="iaai"): # noqa: ANN001, ANN202
|
||||
def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="MOBILEDE"): # noqa: ANN001, ANN202
|
||||
del active_origin_urls, lane
|
||||
return 0
|
||||
|
||||
@@ -139,3 +174,24 @@ def test_fast_sync_engine_applies_runtime_filters_before_db() -> None:
|
||||
assert result["cars_upserted"] == 1
|
||||
assert result["cars_filtered"] == 1
|
||||
assert persistence.inserted == 1
|
||||
|
||||
|
||||
def test_fast_sync_engine_maps_fallback_mobilede_fields() -> None:
|
||||
listing = [_listing_vehicle("449252166")]
|
||||
details = {
|
||||
"449252166": _detail_payload_with_fallbacks("449252166"),
|
||||
}
|
||||
persistence = FakePersistence()
|
||||
engine = FastSyncEngine(
|
||||
client=FakeFastClient(listing, details), # type: ignore[arg-type]
|
||||
mapper=FastCarMapper(),
|
||||
persistence=persistence, # type: ignore[arg-type]
|
||||
batch_size=10,
|
||||
fetch_concurrency=1,
|
||||
)
|
||||
|
||||
result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False)
|
||||
|
||||
assert result["cars_upserted"] == 1
|
||||
assert persistence.inserted == 1
|
||||
assert persistence.images == 1
|
||||
|
||||
@@ -2,9 +2,9 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from iaai_scraper.browser.pace import HumanPacer
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.browser.listing import ListingCollector
|
||||
from mobilede_scraper.browser.pace import HumanPacer
|
||||
from mobilede_scraper.core.config import Settings
|
||||
from mobilede_scraper.browser.listing import ListingCollector
|
||||
|
||||
|
||||
class _FakePage:
|
||||
@@ -58,8 +58,8 @@ class TestListingUnit(unittest.TestCase):
|
||||
self.assertEqual(
|
||||
links,
|
||||
[
|
||||
("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
|
||||
("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
|
||||
("https://www.MOBILEDE.com/VehicleDetail/45184893~US", "45184893"),
|
||||
("https://www.MOBILEDE.com/VehicleDetail/45171480~US", "45171480"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from iaai_scraper.parsing.mapper import CarMapper
|
||||
from mobilede_scraper.parsing.mapper import CarMapper
|
||||
|
||||
|
||||
class TestCarMapper(unittest.TestCase):
|
||||
@@ -11,13 +11,13 @@ class TestCarMapper(unittest.TestCase):
|
||||
|
||||
def test_deduplicates_images_by_image_key(self) -> None:
|
||||
urls = [
|
||||
"https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
|
||||
"https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
|
||||
"https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=1&width=200&height=150",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300",
|
||||
]
|
||||
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US",
|
||||
vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
|
||||
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
|
||||
)
|
||||
@@ -28,7 +28,7 @@ class TestCarMapper(unittest.TestCase):
|
||||
|
||||
def test_no_damage_marker_is_not_damaged(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US",
|
||||
vehicle_summary={"make": "Ford", "model": "Focus"},
|
||||
payload_insights={
|
||||
"vehicle_core": {},
|
||||
@@ -43,7 +43,7 @@ class TestCarMapper(unittest.TestCase):
|
||||
|
||||
def test_unknown_empty_values_and_normalization(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/999~US",
|
||||
vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
|
||||
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
|
||||
)
|
||||
@@ -54,7 +54,7 @@ class TestCarMapper(unittest.TestCase):
|
||||
|
||||
# Нормализация регистра и пробелов.
|
||||
record2 = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/888~US",
|
||||
vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
|
||||
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
|
||||
)
|
||||
@@ -63,7 +63,7 @@ class TestCarMapper(unittest.TestCase):
|
||||
|
||||
def test_price_and_currency_parsing(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/777~US",
|
||||
vehicle_summary={"make": "Toyota", "model": "Corolla"},
|
||||
payload_insights={
|
||||
"vehicle_core": {},
|
||||
@@ -76,7 +76,7 @@ class TestCarMapper(unittest.TestCase):
|
||||
self.assertEqual(record.price, 5200)
|
||||
|
||||
record2 = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/778~US",
|
||||
vehicle_summary={"make": "Toyota", "model": "Corolla"},
|
||||
payload_insights={
|
||||
"vehicle_core": {},
|
||||
|
||||
@@ -2,7 +2,7 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from iaai_scraper.parsing.parser import VehicleParser
|
||||
from mobilede_scraper.parsing.parser import VehicleParser
|
||||
|
||||
|
||||
class TestVehicleParserUnit(unittest.TestCase):
|
||||
@@ -29,11 +29,11 @@ class TestVehicleParserUnit(unittest.TestCase):
|
||||
self.assertEqual(parsed["model"], "CAMRY")
|
||||
|
||||
def test_extract_image_urls_filters_and_deduplicates(self) -> None:
|
||||
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
|
||||
vehicle_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
|
||||
payloads = [{"imageUrls": [
|
||||
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
||||
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
||||
"https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
|
||||
]}]
|
||||
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
|
||||
self.assertEqual(len(urls), 1)
|
||||
|
||||
@@ -4,17 +4,17 @@ import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from iaai_scraper.scraper import IAAIScraper
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.worker import tasks
|
||||
from mobilede_scraper.scraper import MobiledeScraper
|
||||
from mobilede_scraper.core.config import Settings
|
||||
from mobilede_scraper.worker import tasks
|
||||
|
||||
|
||||
class TestResilience(unittest.TestCase):
|
||||
def _make_scraper(self) -> IAAIScraper:
|
||||
def _make_scraper(self) -> MobiledeScraper:
|
||||
s = Settings()
|
||||
s.log_level = "CRITICAL"
|
||||
s.database.url = "sqlite://"
|
||||
return IAAIScraper(s)
|
||||
return MobiledeScraper(s)
|
||||
|
||||
def test_update_task_progress_updates_global_marker(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
@@ -60,11 +60,11 @@ class TestResilience(unittest.TestCase):
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
lane="MOBILEDE_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
|
||||
listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TEST",
|
||||
)
|
||||
|
||||
self.assertEqual(result["total"], 0)
|
||||
|
||||
@@ -5,29 +5,29 @@ from concurrent.futures import TimeoutError as FuturesTimeoutError
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from iaai_scraper.core.config import Settings
|
||||
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
|
||||
from iaai_scraper.scraper import IAAIScraper
|
||||
from iaai_scraper.storage.schemas import CarRecord
|
||||
from mobilede_scraper.core.config import Settings
|
||||
from mobilede_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
|
||||
from mobilede_scraper.scraper import MobiledeScraper
|
||||
from mobilede_scraper.storage.schemas import CarRecord
|
||||
|
||||
|
||||
def make_db_record(origin_id: str) -> dict[str, object]:
|
||||
return CarRecord(
|
||||
parser_id=f"iaai:{origin_id}",
|
||||
parser_id=f"mobilede:{origin_id}",
|
||||
brand="Toyota",
|
||||
model="Camry",
|
||||
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
|
||||
origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
|
||||
origin_id=origin_id,
|
||||
slug=f"toyota-camry-{origin_id}",
|
||||
).model_dump(mode="json")
|
||||
|
||||
|
||||
class TestScraperSync(unittest.TestCase):
|
||||
def _make_scraper(self) -> IAAIScraper:
|
||||
def _make_scraper(self) -> MobiledeScraper:
|
||||
s = Settings()
|
||||
s.log_level = "CRITICAL"
|
||||
s.database.url = "sqlite://"
|
||||
return IAAIScraper(s)
|
||||
return MobiledeScraper(s)
|
||||
|
||||
def test_sync_vehicle_uses_db_record(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
@@ -37,7 +37,7 @@ class TestScraperSync(unittest.TestCase):
|
||||
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
|
||||
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
|
||||
|
||||
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
|
||||
result = scraper.sync_vehicle("https://www.MOBILEDE.com/VehicleDetail/111~US")
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertIn("trace_id", result)
|
||||
@@ -50,13 +50,13 @@ class TestScraperSync(unittest.TestCase):
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=2)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
|
||||
{"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
|
||||
{"https://www.MOBILEDE.com/VehicleDetail/111~US"}, {"mobilede:222"},
|
||||
))
|
||||
scraper.collect_listing = MagicMock(return_value={
|
||||
"vehicle_urls": [
|
||||
"https://www.iaai.com/VehicleDetail/111~US",
|
||||
"https://www.iaai.com/VehicleDetail/222~US",
|
||||
"https://www.iaai.com/VehicleDetail/333~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/111~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/222~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/333~US",
|
||||
]
|
||||
})
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
@@ -79,7 +79,7 @@ class TestScraperSync(unittest.TestCase):
|
||||
})
|
||||
result2 = scraper2.sync_listing(
|
||||
only_new=True,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
year_min=2020, year_max=2027,
|
||||
)
|
||||
self.assertEqual(result2["cars_upserted"], 10)
|
||||
@@ -155,35 +155,35 @@ class TestScraperSync(unittest.TestCase):
|
||||
self.assertEqual(result["status"], "partial_success")
|
||||
|
||||
def test_build_segment_listing_url(self) -> None:
|
||||
base = "https://www.iaai.com/Vehiclelisting/Cars"
|
||||
base = "https://www.MOBILEDE.com/Vehiclelisting/Cars"
|
||||
self.assertEqual(
|
||||
IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
|
||||
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
MobiledeScraper._build_segment_listing_url(base, "TOYOTA"),
|
||||
"https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
)
|
||||
self.assertEqual(
|
||||
IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
|
||||
"https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
|
||||
MobiledeScraper._build_segment_listing_url(base, "LAND ROVER"),
|
||||
"https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
|
||||
)
|
||||
self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
|
||||
self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
|
||||
self.assertEqual(MobiledeScraper._build_segment_listing_url(base, None), base)
|
||||
self.assertEqual(MobiledeScraper._build_segment_listing_url(base, ""), base)
|
||||
|
||||
def test_guard_and_protection_detection(self) -> None:
|
||||
with self.assertRaises(AntiBotDetectedError):
|
||||
IAAIScraper._raise_if_blocked_or_incomplete(
|
||||
MobiledeScraper._raise_if_blocked_or_incomplete(
|
||||
{"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
|
||||
"access_notes": {}, "vehicle_summary": {}},
|
||||
"https://www.iaai.com/VehicleDetail/999~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/999~US",
|
||||
)
|
||||
with self.assertRaises(SiteStructureChangedError):
|
||||
IAAIScraper._raise_if_blocked_or_incomplete(
|
||||
MobiledeScraper._raise_if_blocked_or_incomplete(
|
||||
{"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
|
||||
"access_notes": {"possible_captcha": False, "possible_antibot": False},
|
||||
"vehicle_summary": {}},
|
||||
"https://www.iaai.com/VehicleDetail/999~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/999~US",
|
||||
)
|
||||
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
|
||||
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
|
||||
self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
|
||||
self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
|
||||
self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
|
||||
self.assertFalse(MobiledeScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
|
||||
|
||||
def test_close_resets_browser_state(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
@@ -201,7 +201,7 @@ class TestScraperSync(unittest.TestCase):
|
||||
scraper = self._make_scraper()
|
||||
page = MagicMock()
|
||||
page_result = SimpleNamespace(
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
|
||||
vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/123~US")],
|
||||
)
|
||||
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
|
||||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||||
@@ -228,7 +228,7 @@ class TestScraperSync(unittest.TestCase):
|
||||
page = MagicMock()
|
||||
page_result = SimpleNamespace(
|
||||
page_number=1,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
|
||||
vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/999~US", lot_number="999")],
|
||||
next_page_detected=False,
|
||||
)
|
||||
|
||||
@@ -245,7 +245,7 @@ class TestScraperSync(unittest.TestCase):
|
||||
"year_max": None,
|
||||
})
|
||||
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
|
||||
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
|
||||
scraper._extract_page_urls = MagicMock(return_value=["https://www.MOBILEDE.com/VehicleDetail/999~US"])
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 1,
|
||||
"cars_failed": 0,
|
||||
@@ -256,11 +256,11 @@ class TestScraperSync(unittest.TestCase):
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
lane="MOBILEDE_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
|
||||
listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=EAGLE",
|
||||
)
|
||||
|
||||
scraper.listing_collector.open_cars_listing.assert_called_once()
|
||||
@@ -278,9 +278,9 @@ class TestScraperSync(unittest.TestCase):
|
||||
})
|
||||
|
||||
urls = [
|
||||
"https://www.iaai.com/VehicleDetail/111~US",
|
||||
"https://www.iaai.com/VehicleDetail/222~US",
|
||||
"https://www.iaai.com/VehicleDetail/333~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/111~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/222~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/333~US",
|
||||
]
|
||||
first_record = CarRecord.model_validate(make_db_record("111"))
|
||||
|
||||
@@ -298,8 +298,8 @@ class TestScraperSync(unittest.TestCase):
|
||||
yield 0, first_record
|
||||
raise FuturesTimeoutError()
|
||||
|
||||
with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
|
||||
patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
|
||||
with patch("mobilede_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
|
||||
patch("mobilede_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
|
||||
patch.object(scraper, "_browser_fallback_parallel", return_value={
|
||||
"records": [],
|
||||
"failures": [],
|
||||
|
||||
@@ -3,7 +3,7 @@ from __future__ import annotations
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from iaai_scraper.worker import self_heal
|
||||
from mobilede_scraper.worker import self_heal
|
||||
|
||||
|
||||
class TestSelfHeal(unittest.TestCase):
|
||||
@@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase):
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: {
|
||||
self_heal.GLOBAL_PROGRESS_TS_KEY: None,
|
||||
"iaai:state:task_progress:a": '{"ts": 100}',
|
||||
"iaai:state:task_progress:b": '{"ts": 250}',
|
||||
"iaai:state:task_progress:c": '{"ts": 150}',
|
||||
"mobilede:state:task_progress:a": '{"ts": 100}',
|
||||
"mobilede:state:task_progress:b": '{"ts": 250}',
|
||||
"mobilede:state:task_progress:c": '{"ts": 150}',
|
||||
}.get(key)
|
||||
redis_client.scan_iter.return_value = [
|
||||
"iaai:state:task_progress:a",
|
||||
"iaai:state:task_progress:b",
|
||||
"iaai:state:task_progress:c",
|
||||
"mobilede:state:task_progress:a",
|
||||
"mobilede:state:task_progress:b",
|
||||
"mobilede:state:task_progress:c",
|
||||
]
|
||||
|
||||
ts = self_heal._read_last_progress_ts(redis_client)
|
||||
@@ -38,20 +38,20 @@ class TestSelfHeal(unittest.TestCase):
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: {
|
||||
self_heal.GLOBAL_PROGRESS_TS_KEY: None,
|
||||
"iaai:state:task_progress:a": "{bad-json}",
|
||||
"iaai:state:task_progress:b": '{"foo": "bar"}',
|
||||
"mobilede:state:task_progress:a": "{bad-json}",
|
||||
"mobilede:state:task_progress:b": '{"foo": "bar"}',
|
||||
}.get(key)
|
||||
redis_client.scan_iter.return_value = [
|
||||
"iaai:state:task_progress:a",
|
||||
"iaai:state:task_progress:b",
|
||||
"mobilede:state:task_progress:a",
|
||||
"mobilede:state:task_progress:b",
|
||||
]
|
||||
|
||||
ts = self_heal._read_last_progress_ts(redis_client)
|
||||
|
||||
self.assertIsNone(ts)
|
||||
|
||||
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
|
||||
@patch("iaai_scraper.worker.self_heal.os.kill")
|
||||
@patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
|
||||
@patch("mobilede_scraper.worker.self_heal.os.kill")
|
||||
@patch("builtins.open")
|
||||
def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
|
||||
open_mock.return_value.__enter__.return_value.read.return_value = "123"
|
||||
@@ -64,8 +64,8 @@ class TestSelfHeal(unittest.TestCase):
|
||||
self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
|
||||
self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
|
||||
|
||||
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
|
||||
@patch("iaai_scraper.worker.self_heal.os.kill")
|
||||
@patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
|
||||
@patch("mobilede_scraper.worker.self_heal.os.kill")
|
||||
@patch("builtins.open")
|
||||
def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
|
||||
open_mock.return_value.__enter__.return_value.read.return_value = "123"
|
||||
|
||||
@@ -2,9 +2,9 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from iaai_scraper.core.utils import deep_find_key
|
||||
from iaai_scraper.core.config import (
|
||||
IAAI_DEFAULT_MAKES,
|
||||
from mobilede_scraper.core.utils import deep_find_key
|
||||
from mobilede_scraper.core.config import (
|
||||
MOBILEDE_DEFAULT_MAKES,
|
||||
_LARGE_MAKES,
|
||||
_YEAR_SPLITS,
|
||||
ProxyConfig,
|
||||
@@ -39,12 +39,12 @@ class TestParseListingSegments(unittest.TestCase):
|
||||
segs = parse_listing_segments("auto")
|
||||
|
||||
# Точное число сегментов.
|
||||
expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
|
||||
expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(MOBILEDE_DEFAULT_MAKES) - len(_LARGE_MAKES))
|
||||
self.assertEqual(len(segs), expected)
|
||||
|
||||
# Все бренды из списка присутствуют.
|
||||
makes_in_segments = {s["make"] for s in segs}
|
||||
for make in IAAI_DEFAULT_MAKES:
|
||||
for make in MOBILEDE_DEFAULT_MAKES:
|
||||
self.assertIn(make, makes_in_segments)
|
||||
|
||||
# Крупные бренды разбиты на 3 сегмента с годами.
|
||||
|
||||
@@ -7,8 +7,8 @@ import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from iaai_scraper.worker import tasks
|
||||
from iaai_scraper.core.config import Settings, settings as base_settings
|
||||
from mobilede_scraper.worker import tasks
|
||||
from mobilede_scraper.core.config import Settings, settings as base_settings
|
||||
|
||||
|
||||
class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
@@ -212,7 +212,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
@@ -230,7 +230,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-resume-seg")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
@@ -254,7 +254,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
# Оставшийся чекпоинт не должен использоваться.
|
||||
@@ -273,7 +273,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-792")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
@@ -296,7 +296,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
@@ -314,7 +314,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-beyond")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
@@ -526,7 +526,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job") as run_job, \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=[]):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
@@ -573,7 +573,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
@@ -598,7 +598,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
|
||||
with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-always-full-resume")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
|
||||
Reference in New Issue
Block a user