qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add mobilede tests

Browse Source
This commit is contained in:
qananasikq
2026-04-29 19:13:18 +03:00
parent f00819a21c
commit 8f94828b29
11 changed files with 221 additions and 119 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
tests/test_db.py
View File

@@ -1,4 +1,4 @@
from __future__ import annotations from __future__ import annotations
import tempfile import tempfile
import unittest import unittest
@@ -6,10 +6,10 @@ from pathlib import Path
from sqlalchemy import select from sqlalchemy import select
from iaai_scraper.core.config import Settings from mobilede_scraper.core.config import Settings
from iaai_scraper.storage.db import PersistenceService from mobilede_scraper.storage.db import PersistenceService
from iaai_scraper.storage.models import Car, Image, SyncRun from mobilede_scraper.storage.models import Car, Image, SyncRun
from iaai_scraper.storage.schemas import CarRecord, ImageRecord from mobilede_scraper.storage.schemas import CarRecord, ImageRecord
class TestPersistenceServiceIntegration(unittest.TestCase): class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
@staticmethod @staticmethod
def _record(origin_id: str, *, price: int = 1000) -> CarRecord: def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
return CarRecord( return CarRecord(
parser_id=f"iaai:{origin_id}", parser_id=f"mobilede:{origin_id}",
brand="Toyota", brand="Toyota",
model="Camry", model="Camry",
year=2014, year=2014,
price=price, price=price,
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id, origin_id=origin_id,
slug=f"toyota-camry-{origin_id}", slug=f"toyota-camry-{origin_id}",
images=[ images=[
ImageRecord( ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300", preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=400&height=300",
order_index=0, order_index=0,
) )
], ],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
second = self._record("888") second = self._record("888")
second.images = [ second.images = [
ImageRecord( ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633", fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300",
order_index=0, order_index=0,
) )
] ]
@@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
first = self._record("OLD-ID") first = self._record("OLD-ID")
first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" first.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
self.persistence.upsert_car(first) self.persistence.upsert_car(first)
second = self._record("NEW-ID") second = self._record("NEW-ID")
second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" second.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
result = self.persistence.upsert_car(second) result = self.persistence.upsert_car(second)
self.assertEqual(result["action"], "updated") self.assertEqual(result["action"], "updated")
@@ -125,3 +125,4 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
if __name__ == "__main__": if __name__ == "__main__":
unittest.main() unittest.main()

53
tests/test_fast_client.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
import json import json
from iaai_scraper.browser.fast_client import ( from mobilede_scraper.browser.fast_client import (
DETAIL_MARKER, DETAIL_MARKER,
LISTING_MARKER, LISTING_MARKER,
build_resizer_images_from_keys, build_resizer_images_from_keys,
@@ -10,7 +10,7 @@ from iaai_scraper.browser.fast_client import (
parse_listing_page, parse_listing_page,
parse_product_details_vm, parse_product_details_vm,
) )
from iaai_scraper.parsing.fast_mapper import FastCarMapper from mobilede_scraper.parsing.fast_mapper import FastCarMapper
def test_parse_listing_page_extracts_hidden_payloads() -> None: def test_parse_listing_page_extracts_hidden_payloads() -> None:
@@ -78,10 +78,10 @@ def test_parse_product_details_vm_and_map_record() -> None:
parsed = parse_product_details_vm(html) parsed = parse_product_details_vm(html)
record = FastCarMapper().map_payload_to_record( record = FastCarMapper().map_payload_to_record(
detail_payload=parsed, detail_payload=parsed,
vehicle_url="https://www.iaai.com/VehicleDetail/45078011~US", vehicle_url="https://www.MOBILEDE.com/VehicleDetail/45078011~US",
) )
assert record.origin_id == "iaai:45078011~US" assert record.origin_id == "mobilede:45078011"
assert record.brand == "ACURA" assert record.brand == "ACURA"
assert record.model == "RSX BASE" assert record.model == "RSX BASE"
assert record.year == 2002 assert record.year == 2002
@@ -94,6 +94,51 @@ def test_parse_product_details_vm_and_map_record() -> None:
assert len(record.images) == 1 assert len(record.images) == 1
def test_fast_mapper_uses_fallback_keys_and_normalization() -> None:
payload = {
"inventoryView": {
"attributes": {
"Id": "454502861",
"FirstRegistration": "05/2012",
"Make": "BMW",
"Model": "X1",
"Variant": "xDrive 20d",
"Currency": "EUR",
"Mileage": "50.100 km",
"ColorDesc": "silber metallic",
"Category": "Kombi",
"TransmissionType": "Schaltgetriebe",
"EngineInformation": "1995 ccm, Diesel",
},
"imageDimensions": {
"keys": {
"$values": [
{"k": "454502861~I1", "w": 1600, "h": 1200, "i": 0},
]
}
},
},
"auctionInformation": {
"biddingInformation": {"buyNowPrice": "9.900 €"},
},
}
record = FastCarMapper().map_payload_to_record(
detail_payload=payload,
vehicle_url="https://www.mobile.de/ru/транспортные-средства/подробности.html?id=454502861&vc=Car&s=Car",
)
assert record.model == "X1 xDrive 20d"
assert record.year == 2012
assert record.price == 9900
assert record.mileage == 50100
assert record.color == "silver"
assert record.drive == "4WD"
assert record.gearbox == "MT"
assert record.body_type == "STATION_WAGON"
assert record.engine_volume == 1995
def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None: def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None:
keys = [ keys = [
{"k": "abc~1", "w": 2000, "h": 1500, "i": 2}, {"k": "abc~1", "w": 2000, "h": 1500, "i": 2},

68
tests/test_fast_sync.py
View File

@@ -1,12 +1,13 @@
from __future__ import annotations from __future__ import annotations
from dataclasses import dataclass from dataclasses import dataclass
from types import SimpleNamespace
from iaai_scraper.browser.fast_client import FastListingVehicle from mobilede_scraper.browser.fast_client import FastListingVehicle
from iaai_scraper.core.config import Settings from mobilede_scraper.core.config import Settings
from iaai_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig from mobilede_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig
from iaai_scraper.fast_sync import FastSyncEngine from mobilede_scraper.fast_sync import FastSyncEngine
from iaai_scraper.parsing.fast_mapper import FastCarMapper from mobilede_scraper.parsing.fast_mapper import FastCarMapper
def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle: def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle:
@@ -54,12 +55,46 @@ def _detail_payload(inventory_id: str, *, make: str = "ACURA", model: str = "RSX
} }
def _detail_payload_with_fallbacks(inventory_id: str) -> dict:
return {
"inventoryView": {
"attributes": {
"Id": inventory_id,
"FirstRegistration": "2011-03",
"Make": "BMW",
"Model": "120",
"Variant": "Cabrio",
"Currency": "EUR",
"Kilometerstand": "95 000",
"Color": "weiss",
"DriveType": "Front wheel drive",
"Gearbox": "Automatik",
"VehicleClass": "Cabrio",
"EngineInformation": "2,0 l",
"VehicleGrade": "A",
},
"imageDimensions": {
"keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]}
},
},
"auctionInformation": {
"biddingInformation": {"buyNowPrice": "8.899 €"},
},
}
class FakeFastClient: class FakeFastClient:
def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None: def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None:
self.listing = listing self.listing = listing
self.details = details self.details = details
self.detail_calls = 0 self.detail_calls = 0
self.persist_calls = 0 self.persist_calls = 0
self._settings = SimpleNamespace(
scraping_profile=SimpleNamespace(
verbose_progress_logs=False,
request_jitter_max_s=0.0,
)
)
def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None): # noqa: ANN001, ANN202 def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None): # noqa: ANN001, ANN202
del listing_start_url, make, max_pages del listing_start_url, make, max_pages
@@ -87,7 +122,7 @@ class FakePersistence:
self.images += sum(len(record.images) for record in records) self.images += sum(len(record.images) for record in records)
return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)} return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)}
def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="iaai"): # noqa: ANN001, ANN202 def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="MOBILEDE"): # noqa: ANN001, ANN202
del active_origin_urls, lane del active_origin_urls, lane
return 0 return 0
@@ -139,3 +174,24 @@ def test_fast_sync_engine_applies_runtime_filters_before_db() -> None:
assert result["cars_upserted"] == 1 assert result["cars_upserted"] == 1
assert result["cars_filtered"] == 1 assert result["cars_filtered"] == 1
assert persistence.inserted == 1 assert persistence.inserted == 1
def test_fast_sync_engine_maps_fallback_mobilede_fields() -> None:
listing = [_listing_vehicle("449252166")]
details = {
"449252166": _detail_payload_with_fallbacks("449252166"),
}
persistence = FakePersistence()
engine = FastSyncEngine(
client=FakeFastClient(listing, details), # type: ignore[arg-type]
mapper=FastCarMapper(),
persistence=persistence, # type: ignore[arg-type]
batch_size=10,
fetch_concurrency=1,
)
result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False)
assert result["cars_upserted"] == 1
assert persistence.inserted == 1
assert persistence.images == 1

10
tests/test_listing.py
View File

@@ -2,9 +2,9 @@ from __future__ import annotations
import unittest import unittest
from iaai_scraper.browser.pace import HumanPacer from mobilede_scraper.browser.pace import HumanPacer
from iaai_scraper.core.config import Settings from mobilede_scraper.core.config import Settings
from iaai_scraper.browser.listing import ListingCollector from mobilede_scraper.browser.listing import ListingCollector
class _FakePage: class _FakePage:
@@ -58,8 +58,8 @@ class TestListingUnit(unittest.TestCase):
self.assertEqual( self.assertEqual(
links, links,
[ [
("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"), ("https://www.MOBILEDE.com/VehicleDetail/45184893~US", "45184893"),
("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"), ("https://www.MOBILEDE.com/VehicleDetail/45171480~US", "45171480"),
], ],
) )

20
tests/test_mappers.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
import unittest import unittest
from iaai_scraper.parsing.mapper import CarMapper from mobilede_scraper.parsing.mapper import CarMapper
class TestCarMapper(unittest.TestCase): class TestCarMapper(unittest.TestCase):
@@ -11,13 +11,13 @@ class TestCarMapper(unittest.TestCase):
def test_deduplicates_images_by_image_key(self) -> None: def test_deduplicates_images_by_image_key(self) -> None:
urls = [ urls = [
"https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", "https://vis.MOBILEDE.com/resizer?imageKeys=1&width=200&height=150",
"https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", "https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", "https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300",
] ]
record = self.mapper.map_to_car_record( record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US", vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US",
vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
) )
@@ -28,7 +28,7 @@ class TestCarMapper(unittest.TestCase):
def test_no_damage_marker_is_not_damaged(self) -> None: def test_no_damage_marker_is_not_damaged(self) -> None:
record = self.mapper.map_to_car_record( record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US", vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US",
vehicle_summary={"make": "Ford", "model": "Focus"}, vehicle_summary={"make": "Ford", "model": "Focus"},
payload_insights={ payload_insights={
"vehicle_core": {}, "vehicle_core": {},
@@ -43,7 +43,7 @@ class TestCarMapper(unittest.TestCase):
def test_unknown_empty_values_and_normalization(self) -> None: def test_unknown_empty_values_and_normalization(self) -> None:
record = self.mapper.map_to_car_record( record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/999~US", vehicle_url="https://www.MOBILEDE.com/VehicleDetail/999~US",
vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
) )
@@ -54,7 +54,7 @@ class TestCarMapper(unittest.TestCase):
# Нормализация регистра и пробелов. # Нормализация регистра и пробелов.
record2 = self.mapper.map_to_car_record( record2 = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/888~US", vehicle_url="https://www.MOBILEDE.com/VehicleDetail/888~US",
vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
) )
@@ -63,7 +63,7 @@ class TestCarMapper(unittest.TestCase):
def test_price_and_currency_parsing(self) -> None: def test_price_and_currency_parsing(self) -> None:
record = self.mapper.map_to_car_record( record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/777~US", vehicle_url="https://www.MOBILEDE.com/VehicleDetail/777~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"}, vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={ payload_insights={
"vehicle_core": {}, "vehicle_core": {},
@@ -76,7 +76,7 @@ class TestCarMapper(unittest.TestCase):
self.assertEqual(record.price, 5200) self.assertEqual(record.price, 5200)
record2 = self.mapper.map_to_car_record( record2 = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/778~US", vehicle_url="https://www.MOBILEDE.com/VehicleDetail/778~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"}, vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={ payload_insights={
"vehicle_core": {}, "vehicle_core": {},

10
tests/test_parser.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
import unittest import unittest
from iaai_scraper.parsing.parser import VehicleParser from mobilede_scraper.parsing.parser import VehicleParser
class TestVehicleParserUnit(unittest.TestCase): class TestVehicleParserUnit(unittest.TestCase):
@@ -29,11 +29,11 @@ class TestVehicleParserUnit(unittest.TestCase):
self.assertEqual(parsed["model"], "CAMRY") self.assertEqual(parsed["model"], "CAMRY")
def test_extract_image_urls_filters_and_deduplicates(self) -> None: def test_extract_image_urls_filters_and_deduplicates(self) -> None:
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" vehicle_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
payloads = [{"imageUrls": [ payloads = [{"imageUrls": [
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", "https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", "https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", "https://vis.MOBILEDE.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
]}] ]}]
urls = self.parser._extract_image_urls(payloads, "", vehicle_url) urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
self.assertEqual(len(urls), 1) self.assertEqual(len(urls), 1)

14
tests/test_resilience.py
View File

@@ -4,17 +4,17 @@ import unittest
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from iaai_scraper.scraper import IAAIScraper from mobilede_scraper.scraper import MobiledeScraper
from iaai_scraper.core.config import Settings from mobilede_scraper.core.config import Settings
from iaai_scraper.worker import tasks from mobilede_scraper.worker import tasks
class TestResilience(unittest.TestCase): class TestResilience(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper: def _make_scraper(self) -> MobiledeScraper:
s = Settings() s = Settings()
s.log_level = "CRITICAL" s.log_level = "CRITICAL"
s.database.url = "sqlite://" s.database.url = "sqlite://"
return IAAIScraper(s) return MobiledeScraper(s)
def test_update_task_progress_updates_global_marker(self) -> None: def test_update_task_progress_updates_global_marker(self) -> None:
redis_client = MagicMock() redis_client = MagicMock()
@@ -60,11 +60,11 @@ class TestResilience(unittest.TestCase):
result = scraper._sync_listing_streaming( result = scraper._sync_listing_streaming(
make=None, make=None,
model=None, model=None,
lane="iaai_cars", lane="MOBILEDE_cars",
limit=None, limit=None,
effective_only_new=False, effective_only_new=False,
started_at=0.0, started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST", listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TEST",
) )
self.assertEqual(result["total"], 0) self.assertEqual(result["total"], 0)

76
tests/test_scraper.py
View File

@@ -5,29 +5,29 @@ from concurrent.futures import TimeoutError as FuturesTimeoutError
from types import SimpleNamespace from types import SimpleNamespace
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from iaai_scraper.core.config import Settings from mobilede_scraper.core.config import Settings
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError from mobilede_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from iaai_scraper.scraper import IAAIScraper from mobilede_scraper.scraper import MobiledeScraper
from iaai_scraper.storage.schemas import CarRecord from mobilede_scraper.storage.schemas import CarRecord
def make_db_record(origin_id: str) -> dict[str, object]: def make_db_record(origin_id: str) -> dict[str, object]:
return CarRecord( return CarRecord(
parser_id=f"iaai:{origin_id}", parser_id=f"mobilede:{origin_id}",
brand="Toyota", brand="Toyota",
model="Camry", model="Camry",
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id, origin_id=origin_id,
slug=f"toyota-camry-{origin_id}", slug=f"toyota-camry-{origin_id}",
).model_dump(mode="json") ).model_dump(mode="json")
class TestScraperSync(unittest.TestCase): class TestScraperSync(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper: def _make_scraper(self) -> MobiledeScraper:
s = Settings() s = Settings()
s.log_level = "CRITICAL" s.log_level = "CRITICAL"
s.database.url = "sqlite://" s.database.url = "sqlite://"
return IAAIScraper(s) return MobiledeScraper(s)
def test_sync_vehicle_uses_db_record(self) -> None: def test_sync_vehicle_uses_db_record(self) -> None:
scraper = self._make_scraper() scraper = self._make_scraper()
@@ -37,7 +37,7 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") result = scraper.sync_vehicle("https://www.MOBILEDE.com/VehicleDetail/111~US")
self.assertEqual(result["status"], "success") self.assertEqual(result["status"], "success")
self.assertIn("trace_id", result) self.assertIn("trace_id", result)
@@ -50,13 +50,13 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.start_sync_run = MagicMock(return_value=2) scraper.persistence.start_sync_run = MagicMock(return_value=2)
scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
{"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"}, {"https://www.MOBILEDE.com/VehicleDetail/111~US"}, {"mobilede:222"},
)) ))
scraper.collect_listing = MagicMock(return_value={ scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [ "vehicle_urls": [
"https://www.iaai.com/VehicleDetail/111~US", "https://www.MOBILEDE.com/VehicleDetail/111~US",
"https://www.iaai.com/VehicleDetail/222~US", "https://www.MOBILEDE.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US", "https://www.MOBILEDE.com/VehicleDetail/333~US",
] ]
}) })
scraper.sync_batch = MagicMock(return_value={ scraper.sync_batch = MagicMock(return_value={
@@ -79,7 +79,7 @@ class TestScraperSync(unittest.TestCase):
}) })
result2 = scraper2.sync_listing( result2 = scraper2.sync_listing(
only_new=True, only_new=True,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA",
year_min=2020, year_max=2027, year_min=2020, year_max=2027,
) )
self.assertEqual(result2["cars_upserted"], 10) self.assertEqual(result2["cars_upserted"], 10)
@@ -155,35 +155,35 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["status"], "partial_success") self.assertEqual(result["status"], "partial_success")
def test_build_segment_listing_url(self) -> None: def test_build_segment_listing_url(self) -> None:
base = "https://www.iaai.com/Vehiclelisting/Cars" base = "https://www.MOBILEDE.com/Vehiclelisting/Cars"
self.assertEqual( self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "TOYOTA"), MobiledeScraper._build_segment_listing_url(base, "TOYOTA"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", "https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA",
) )
self.assertEqual( self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "LAND ROVER"), MobiledeScraper._build_segment_listing_url(base, "LAND ROVER"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER", "https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
) )
self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base) self.assertEqual(MobiledeScraper._build_segment_listing_url(base, None), base)
self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base) self.assertEqual(MobiledeScraper._build_segment_listing_url(base, ""), base)
def test_guard_and_protection_detection(self) -> None: def test_guard_and_protection_detection(self) -> None:
with self.assertRaises(AntiBotDetectedError): with self.assertRaises(AntiBotDetectedError):
IAAIScraper._raise_if_blocked_or_incomplete( MobiledeScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
"access_notes": {}, "vehicle_summary": {}}, "access_notes": {}, "vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US", "https://www.MOBILEDE.com/VehicleDetail/999~US",
) )
with self.assertRaises(SiteStructureChangedError): with self.assertRaises(SiteStructureChangedError):
IAAIScraper._raise_if_blocked_or_incomplete( MobiledeScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
"access_notes": {"possible_captcha": False, "possible_antibot": False}, "access_notes": {"possible_captcha": False, "possible_antibot": False},
"vehicle_summary": {}}, "vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US", "https://www.MOBILEDE.com/VehicleDetail/999~US",
) )
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) self.assertFalse(MobiledeScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
def test_close_resets_browser_state(self) -> None: def test_close_resets_browser_state(self) -> None:
scraper = self._make_scraper() scraper = self._make_scraper()
@@ -201,7 +201,7 @@ class TestScraperSync(unittest.TestCase):
scraper = self._make_scraper() scraper = self._make_scraper()
page = MagicMock() page = MagicMock()
page_result = SimpleNamespace( page_result = SimpleNamespace(
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")], vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/123~US")],
) )
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper.listing_collector.open_cars_listing = MagicMock() scraper.listing_collector.open_cars_listing = MagicMock()
@@ -228,7 +228,7 @@ class TestScraperSync(unittest.TestCase):
page = MagicMock() page = MagicMock()
page_result = SimpleNamespace( page_result = SimpleNamespace(
page_number=1, page_number=1,
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")], vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/999~US", lot_number="999")],
next_page_detected=False, next_page_detected=False,
) )
@@ -245,7 +245,7 @@ class TestScraperSync(unittest.TestCase):
"year_max": None, "year_max": None,
}) })
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"]) scraper._extract_page_urls = MagicMock(return_value=["https://www.MOBILEDE.com/VehicleDetail/999~US"])
scraper.sync_batch = MagicMock(return_value={ scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 1, "cars_upserted": 1,
"cars_failed": 0, "cars_failed": 0,
@@ -256,11 +256,11 @@ class TestScraperSync(unittest.TestCase):
result = scraper._sync_listing_streaming( result = scraper._sync_listing_streaming(
make=None, make=None,
model=None, model=None,
lane="iaai_cars", lane="MOBILEDE_cars",
limit=None, limit=None,
effective_only_new=False, effective_only_new=False,
started_at=0.0, started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE", listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=EAGLE",
) )
scraper.listing_collector.open_cars_listing.assert_called_once() scraper.listing_collector.open_cars_listing.assert_called_once()
@@ -278,9 +278,9 @@ class TestScraperSync(unittest.TestCase):
}) })
urls = [ urls = [
"https://www.iaai.com/VehicleDetail/111~US", "https://www.MOBILEDE.com/VehicleDetail/111~US",
"https://www.iaai.com/VehicleDetail/222~US", "https://www.MOBILEDE.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US", "https://www.MOBILEDE.com/VehicleDetail/333~US",
] ]
first_record = CarRecord.model_validate(make_db_record("111")) first_record = CarRecord.model_validate(make_db_record("111"))
@@ -298,8 +298,8 @@ class TestScraperSync(unittest.TestCase):
yield 0, first_record yield 0, first_record
raise FuturesTimeoutError() raise FuturesTimeoutError()
with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ with patch("mobilede_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ patch("mobilede_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
patch.object(scraper, "_browser_fallback_parallel", return_value={ patch.object(scraper, "_browser_fallback_parallel", return_value={
"records": [], "records": [],
"failures": [], "failures": [],

30
tests/test_self_heal.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
import unittest import unittest
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from iaai_scraper.worker import self_heal from mobilede_scraper.worker import self_heal
class TestSelfHeal(unittest.TestCase): class TestSelfHeal(unittest.TestCase):
@@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase):
redis_client = MagicMock() redis_client = MagicMock()
redis_client.get.side_effect = lambda key: { redis_client.get.side_effect = lambda key: {
self_heal.GLOBAL_PROGRESS_TS_KEY: None, self_heal.GLOBAL_PROGRESS_TS_KEY: None,
"iaai:state:task_progress:a": '{"ts": 100}', "mobilede:state:task_progress:a": '{"ts": 100}',
"iaai:state:task_progress:b": '{"ts": 250}', "mobilede:state:task_progress:b": '{"ts": 250}',
"iaai:state:task_progress:c": '{"ts": 150}', "mobilede:state:task_progress:c": '{"ts": 150}',
}.get(key) }.get(key)
redis_client.scan_iter.return_value = [ redis_client.scan_iter.return_value = [
"iaai:state:task_progress:a", "mobilede:state:task_progress:a",
"iaai:state:task_progress:b", "mobilede:state:task_progress:b",
"iaai:state:task_progress:c", "mobilede:state:task_progress:c",
] ]
ts = self_heal._read_last_progress_ts(redis_client) ts = self_heal._read_last_progress_ts(redis_client)
@@ -38,20 +38,20 @@ class TestSelfHeal(unittest.TestCase):
redis_client = MagicMock() redis_client = MagicMock()
redis_client.get.side_effect = lambda key: { redis_client.get.side_effect = lambda key: {
self_heal.GLOBAL_PROGRESS_TS_KEY: None, self_heal.GLOBAL_PROGRESS_TS_KEY: None,
"iaai:state:task_progress:a": "{bad-json}", "mobilede:state:task_progress:a": "{bad-json}",
"iaai:state:task_progress:b": '{"foo": "bar"}', "mobilede:state:task_progress:b": '{"foo": "bar"}',
}.get(key) }.get(key)
redis_client.scan_iter.return_value = [ redis_client.scan_iter.return_value = [
"iaai:state:task_progress:a", "mobilede:state:task_progress:a",
"iaai:state:task_progress:b", "mobilede:state:task_progress:b",
] ]
ts = self_heal._read_last_progress_ts(redis_client) ts = self_heal._read_last_progress_ts(redis_client)
self.assertIsNone(ts) self.assertIsNone(ts)
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) @patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("iaai_scraper.worker.self_heal.os.kill") @patch("mobilede_scraper.worker.self_heal.os.kill")
@patch("builtins.open") @patch("builtins.open")
def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None: def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
open_mock.return_value.__enter__.return_value.read.return_value = "123" open_mock.return_value.__enter__.return_value.read.return_value = "123"
@@ -64,8 +64,8 @@ class TestSelfHeal(unittest.TestCase):
self.assertEqual(kill_mock.call_args_list[1].args, (123, 0)) self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
self.assertEqual(kill_mock.call_args_list[2].args[0], 123) self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) @patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("iaai_scraper.worker.self_heal.os.kill") @patch("mobilede_scraper.worker.self_heal.os.kill")
@patch("builtins.open") @patch("builtins.open")
def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None: def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
open_mock.return_value.__enter__.return_value.read.return_value = "123" open_mock.return_value.__enter__.return_value.read.return_value = "123"

10
tests/test_utils.py
View File

@@ -2,9 +2,9 @@ from __future__ import annotations
import unittest import unittest
from iaai_scraper.core.utils import deep_find_key from mobilede_scraper.core.utils import deep_find_key
from iaai_scraper.core.config import ( from mobilede_scraper.core.config import (
IAAI_DEFAULT_MAKES, MOBILEDE_DEFAULT_MAKES,
_LARGE_MAKES, _LARGE_MAKES,
_YEAR_SPLITS, _YEAR_SPLITS,
ProxyConfig, ProxyConfig,
@@ -39,12 +39,12 @@ class TestParseListingSegments(unittest.TestCase):
segs = parse_listing_segments("auto") segs = parse_listing_segments("auto")
# Точное число сегментов. # Точное число сегментов.
expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES)) expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(MOBILEDE_DEFAULT_MAKES) - len(_LARGE_MAKES))
self.assertEqual(len(segs), expected) self.assertEqual(len(segs), expected)
# Все бренды из списка присутствуют. # Все бренды из списка присутствуют.
makes_in_segments = {s["make"] for s in segs} makes_in_segments = {s["make"] for s in segs}
for make in IAAI_DEFAULT_MAKES: for make in MOBILEDE_DEFAULT_MAKES:
self.assertIn(make, makes_in_segments) self.assertIn(make, makes_in_segments)
# Крупные бренды разбиты на 3 сегмента с годами. # Крупные бренды разбиты на 3 сегмента с годами.

22
tests/test_worker_tasks.py
View File

@@ -7,8 +7,8 @@ import tempfile
import unittest import unittest
from unittest.mock import MagicMock, patch from unittest.mock import MagicMock, patch
from iaai_scraper.worker import tasks from mobilede_scraper.worker import tasks
from iaai_scraper.core.config import Settings, settings as base_settings from mobilede_scraper.core.config import Settings, settings as base_settings
class TestWorkerTaskLockHelpers(unittest.TestCase): class TestWorkerTaskLockHelpers(unittest.TestCase):
@@ -212,7 +212,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock() get_persistence.return_value = MagicMock()
redis_client = MagicMock() redis_client = MagicMock()
redis_client.get.side_effect = lambda key: ( redis_client.get.side_effect = lambda key: (
@@ -230,7 +230,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-resume-seg") tasks.sync_listing_task.push_request(id="task-resume-seg")
try: try:
result = tasks.sync_listing_task.run() result = tasks.sync_listing_task.run()
@@ -254,7 +254,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock() get_persistence.return_value = MagicMock()
redis_client = MagicMock() redis_client = MagicMock()
# Оставшийся чекпоинт не должен использоваться. # Оставшийся чекпоинт не должен использоваться.
@@ -273,7 +273,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-792") tasks.sync_listing_task.push_request(id="task-792")
try: try:
result = tasks.sync_listing_task.run() result = tasks.sync_listing_task.run()
@@ -296,7 +296,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock() get_persistence.return_value = MagicMock()
redis_client = MagicMock() redis_client = MagicMock()
redis_client.get.side_effect = lambda key: ( redis_client.get.side_effect = lambda key: (
@@ -314,7 +314,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-beyond") tasks.sync_listing_task.push_request(id="task-beyond")
try: try:
result = tasks.sync_listing_task.run() result = tasks.sync_listing_task.run()
@@ -526,7 +526,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks.sync_listing_task, "update_state"), \
patch.object(tasks, "_run_browser_job") as run_job, \ patch.object(tasks, "_run_browser_job") as run_job, \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]): patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=[]):
get_persistence.return_value = MagicMock() get_persistence.return_value = MagicMock()
redis_client = MagicMock() redis_client = MagicMock()
redis_client.get.return_value = None redis_client.get.return_value = None
@@ -573,7 +573,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock() get_persistence.return_value = MagicMock()
redis_client = MagicMock() redis_client = MagicMock()
redis_client.get.side_effect = lambda key: ( redis_client.get.side_effect = lambda key: (
@@ -598,7 +598,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing = MagicMock() scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
scraper_ctx.__exit__.return_value = None scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-always-full-resume") tasks.sync_listing_task.push_request(id="task-always-full-resume")
try: try:
result = tasks.sync_listing_task.run() result = tasks.sync_listing_task.run()