diff --git a/tests/test_db.py b/tests/test_db.py index 7720590..b38b6ac 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,4 +1,4 @@ -from __future__ import annotations +from __future__ import annotations import tempfile import unittest @@ -6,10 +6,10 @@ from pathlib import Path from sqlalchemy import select -from iaai_scraper.core.config import Settings -from iaai_scraper.storage.db import PersistenceService -from iaai_scraper.storage.models import Car, Image, SyncRun -from iaai_scraper.storage.schemas import CarRecord, ImageRecord +from mobilede_scraper.core.config import Settings +from mobilede_scraper.storage.db import PersistenceService +from mobilede_scraper.storage.models import Car, Image, SyncRun +from mobilede_scraper.storage.schemas import CarRecord, ImageRecord class TestPersistenceServiceIntegration(unittest.TestCase): @@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase): @staticmethod def _record(origin_id: str, *, price: int = 1000) -> CarRecord: return CarRecord( - parser_id=f"iaai:{origin_id}", + parser_id=f"mobilede:{origin_id}", brand="Toyota", model="Camry", year=2014, price=price, - origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US", origin_id=origin_id, slug=f"toyota-camry-{origin_id}", images=[ ImageRecord( - fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", - preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300", + fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633", + preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=1&width=400&height=300", order_index=0, ) ], @@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase): second = self._record("888") second.images = [ ImageRecord( - fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633", - preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + fullres_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=845&height=633", + preview_image="https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300", order_index=0, ) ] @@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase): def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: first = self._record("OLD-ID") - first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + first.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US" self.persistence.upsert_car(first) second = self._record("NEW-ID") - second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + second.origin_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US" result = self.persistence.upsert_car(second) self.assertEqual(result["action"], "updated") @@ -125,3 +125,4 @@ class TestPersistenceServiceIntegration(unittest.TestCase): if __name__ == "__main__": unittest.main() + diff --git a/tests/test_fast_client.py b/tests/test_fast_client.py index 2f38d28..14feca7 100644 --- a/tests/test_fast_client.py +++ b/tests/test_fast_client.py @@ -2,7 +2,7 @@ from __future__ import annotations import json -from iaai_scraper.browser.fast_client import ( +from mobilede_scraper.browser.fast_client import ( DETAIL_MARKER, LISTING_MARKER, build_resizer_images_from_keys, @@ -10,7 +10,7 @@ from iaai_scraper.browser.fast_client import ( parse_listing_page, parse_product_details_vm, ) -from iaai_scraper.parsing.fast_mapper import FastCarMapper +from mobilede_scraper.parsing.fast_mapper import FastCarMapper def test_parse_listing_page_extracts_hidden_payloads() -> None: @@ -78,10 +78,10 @@ def test_parse_product_details_vm_and_map_record() -> None: parsed = parse_product_details_vm(html) record = FastCarMapper().map_payload_to_record( detail_payload=parsed, - vehicle_url="https://www.iaai.com/VehicleDetail/45078011~US", + vehicle_url="https://www.MOBILEDE.com/VehicleDetail/45078011~US", ) - assert record.origin_id == "iaai:45078011~US" + assert record.origin_id == "mobilede:45078011" assert record.brand == "ACURA" assert record.model == "RSX BASE" assert record.year == 2002 @@ -94,6 +94,51 @@ def test_parse_product_details_vm_and_map_record() -> None: assert len(record.images) == 1 +def test_fast_mapper_uses_fallback_keys_and_normalization() -> None: + payload = { + "inventoryView": { + "attributes": { + "Id": "454502861", + "FirstRegistration": "05/2012", + "Make": "BMW", + "Model": "X1", + "Variant": "xDrive 20d", + "Currency": "EUR", + "Mileage": "50.100 km", + "ColorDesc": "silber metallic", + "Category": "Kombi", + "TransmissionType": "Schaltgetriebe", + "EngineInformation": "1995 ccm, Diesel", + }, + "imageDimensions": { + "keys": { + "$values": [ + {"k": "454502861~I1", "w": 1600, "h": 1200, "i": 0}, + ] + } + }, + }, + "auctionInformation": { + "biddingInformation": {"buyNowPrice": "9.900 €"}, + }, + } + + record = FastCarMapper().map_payload_to_record( + detail_payload=payload, + vehicle_url="https://www.mobile.de/ru/транспортные-средства/подробности.html?id=454502861&vc=Car&s=Car", + ) + + assert record.model == "X1 xDrive 20d" + assert record.year == 2012 + assert record.price == 9900 + assert record.mileage == 50100 + assert record.color == "silver" + assert record.drive == "4WD" + assert record.gearbox == "MT" + assert record.body_type == "STATION_WAGON" + assert record.engine_volume == 1995 + + def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None: keys = [ {"k": "abc~1", "w": 2000, "h": 1500, "i": 2}, diff --git a/tests/test_fast_sync.py b/tests/test_fast_sync.py index c3e92ef..6718015 100644 --- a/tests/test_fast_sync.py +++ b/tests/test_fast_sync.py @@ -1,12 +1,13 @@ from __future__ import annotations from dataclasses import dataclass +from types import SimpleNamespace -from iaai_scraper.browser.fast_client import FastListingVehicle -from iaai_scraper.core.config import Settings -from iaai_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig -from iaai_scraper.fast_sync import FastSyncEngine -from iaai_scraper.parsing.fast_mapper import FastCarMapper +from mobilede_scraper.browser.fast_client import FastListingVehicle +from mobilede_scraper.core.config import Settings +from mobilede_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig +from mobilede_scraper.fast_sync import FastSyncEngine +from mobilede_scraper.parsing.fast_mapper import FastCarMapper def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle: @@ -54,12 +55,46 @@ def _detail_payload(inventory_id: str, *, make: str = "ACURA", model: str = "RSX } +def _detail_payload_with_fallbacks(inventory_id: str) -> dict: + return { + "inventoryView": { + "attributes": { + "Id": inventory_id, + "FirstRegistration": "2011-03", + "Make": "BMW", + "Model": "120", + "Variant": "Cabrio", + "Currency": "EUR", + "Kilometerstand": "95 000", + "Color": "weiss", + "DriveType": "Front wheel drive", + "Gearbox": "Automatik", + "VehicleClass": "Cabrio", + "EngineInformation": "2,0 l", + "VehicleGrade": "A", + }, + "imageDimensions": { + "keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]} + }, + }, + "auctionInformation": { + "biddingInformation": {"buyNowPrice": "8.899 €"}, + }, + } + + class FakeFastClient: def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None: self.listing = listing self.details = details self.detail_calls = 0 self.persist_calls = 0 + self._settings = SimpleNamespace( + scraping_profile=SimpleNamespace( + verbose_progress_logs=False, + request_jitter_max_s=0.0, + ) + ) def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None): # noqa: ANN001, ANN202 del listing_start_url, make, max_pages @@ -87,7 +122,7 @@ class FakePersistence: self.images += sum(len(record.images) for record in records) return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)} - def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="iaai"): # noqa: ANN001, ANN202 + def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="MOBILEDE"): # noqa: ANN001, ANN202 del active_origin_urls, lane return 0 @@ -139,3 +174,24 @@ def test_fast_sync_engine_applies_runtime_filters_before_db() -> None: assert result["cars_upserted"] == 1 assert result["cars_filtered"] == 1 assert persistence.inserted == 1 + + +def test_fast_sync_engine_maps_fallback_mobilede_fields() -> None: + listing = [_listing_vehicle("449252166")] + details = { + "449252166": _detail_payload_with_fallbacks("449252166"), + } + persistence = FakePersistence() + engine = FastSyncEngine( + client=FakeFastClient(listing, details), # type: ignore[arg-type] + mapper=FastCarMapper(), + persistence=persistence, # type: ignore[arg-type] + batch_size=10, + fetch_concurrency=1, + ) + + result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False) + + assert result["cars_upserted"] == 1 + assert persistence.inserted == 1 + assert persistence.images == 1 diff --git a/tests/test_listing.py b/tests/test_listing.py index 7d8e991..8331127 100644 --- a/tests/test_listing.py +++ b/tests/test_listing.py @@ -2,9 +2,9 @@ from __future__ import annotations import unittest -from iaai_scraper.browser.pace import HumanPacer -from iaai_scraper.core.config import Settings -from iaai_scraper.browser.listing import ListingCollector +from mobilede_scraper.browser.pace import HumanPacer +from mobilede_scraper.core.config import Settings +from mobilede_scraper.browser.listing import ListingCollector class _FakePage: @@ -58,8 +58,8 @@ class TestListingUnit(unittest.TestCase): self.assertEqual( links, [ - ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"), - ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"), + ("https://www.MOBILEDE.com/VehicleDetail/45184893~US", "45184893"), + ("https://www.MOBILEDE.com/VehicleDetail/45171480~US", "45171480"), ], ) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 621a5fc..50011b9 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -2,7 +2,7 @@ from __future__ import annotations import unittest -from iaai_scraper.parsing.mapper import CarMapper +from mobilede_scraper.parsing.mapper import CarMapper class TestCarMapper(unittest.TestCase): @@ -11,13 +11,13 @@ class TestCarMapper(unittest.TestCase): def test_deduplicates_images_by_image_key(self) -> None: urls = [ - "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", - "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + "https://vis.MOBILEDE.com/resizer?imageKeys=1&width=200&height=150", + "https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633", + "https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300", ] record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US", vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, ) @@ -28,7 +28,7 @@ class TestCarMapper(unittest.TestCase): def test_no_damage_marker_is_not_damaged(self) -> None: record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US", vehicle_summary={"make": "Ford", "model": "Focus"}, payload_insights={ "vehicle_core": {}, @@ -43,7 +43,7 @@ class TestCarMapper(unittest.TestCase): def test_unknown_empty_values_and_normalization(self) -> None: record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/999~US", + vehicle_url="https://www.MOBILEDE.com/VehicleDetail/999~US", vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, ) @@ -54,7 +54,7 @@ class TestCarMapper(unittest.TestCase): # Нормализация регистра и пробелов. record2 = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/888~US", + vehicle_url="https://www.MOBILEDE.com/VehicleDetail/888~US", vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, ) @@ -63,7 +63,7 @@ class TestCarMapper(unittest.TestCase): def test_price_and_currency_parsing(self) -> None: record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/777~US", + vehicle_url="https://www.MOBILEDE.com/VehicleDetail/777~US", vehicle_summary={"make": "Toyota", "model": "Corolla"}, payload_insights={ "vehicle_core": {}, @@ -76,7 +76,7 @@ class TestCarMapper(unittest.TestCase): self.assertEqual(record.price, 5200) record2 = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/778~US", + vehicle_url="https://www.MOBILEDE.com/VehicleDetail/778~US", vehicle_summary={"make": "Toyota", "model": "Corolla"}, payload_insights={ "vehicle_core": {}, diff --git a/tests/test_parser.py b/tests/test_parser.py index 72a2b6c..cc666c4 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -2,7 +2,7 @@ from __future__ import annotations import unittest -from iaai_scraper.parsing.parser import VehicleParser +from mobilede_scraper.parsing.parser import VehicleParser class TestVehicleParserUnit(unittest.TestCase): @@ -29,11 +29,11 @@ class TestVehicleParserUnit(unittest.TestCase): self.assertEqual(parsed["model"], "CAMRY") def test_extract_image_urls_filters_and_deduplicates(self) -> None: - vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" + vehicle_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US" payloads = [{"imageUrls": [ - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", + "https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.MOBILEDE.com/resizer?imageKeys=99999999~SID2&width=845&height=633", ]}] urls = self.parser._extract_image_urls(payloads, "", vehicle_url) self.assertEqual(len(urls), 1) diff --git a/tests/test_resilience.py b/tests/test_resilience.py index 8b2bf81..f7da815 100644 --- a/tests/test_resilience.py +++ b/tests/test_resilience.py @@ -4,17 +4,17 @@ import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch -from iaai_scraper.scraper import IAAIScraper -from iaai_scraper.core.config import Settings -from iaai_scraper.worker import tasks +from mobilede_scraper.scraper import MobiledeScraper +from mobilede_scraper.core.config import Settings +from mobilede_scraper.worker import tasks class TestResilience(unittest.TestCase): - def _make_scraper(self) -> IAAIScraper: + def _make_scraper(self) -> MobiledeScraper: s = Settings() s.log_level = "CRITICAL" s.database.url = "sqlite://" - return IAAIScraper(s) + return MobiledeScraper(s) def test_update_task_progress_updates_global_marker(self) -> None: redis_client = MagicMock() @@ -60,11 +60,11 @@ class TestResilience(unittest.TestCase): result = scraper._sync_listing_streaming( make=None, model=None, - lane="iaai_cars", + lane="MOBILEDE_cars", limit=None, effective_only_new=False, started_at=0.0, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST", + listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TEST", ) self.assertEqual(result["total"], 0) diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 7d6192a..3755f82 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -5,29 +5,29 @@ from concurrent.futures import TimeoutError as FuturesTimeoutError from types import SimpleNamespace from unittest.mock import MagicMock, patch -from iaai_scraper.core.config import Settings -from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError -from iaai_scraper.scraper import IAAIScraper -from iaai_scraper.storage.schemas import CarRecord +from mobilede_scraper.core.config import Settings +from mobilede_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError +from mobilede_scraper.scraper import MobiledeScraper +from mobilede_scraper.storage.schemas import CarRecord def make_db_record(origin_id: str) -> dict[str, object]: return CarRecord( - parser_id=f"iaai:{origin_id}", + parser_id=f"mobilede:{origin_id}", brand="Toyota", model="Camry", - origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US", origin_id=origin_id, slug=f"toyota-camry-{origin_id}", ).model_dump(mode="json") class TestScraperSync(unittest.TestCase): - def _make_scraper(self) -> IAAIScraper: + def _make_scraper(self) -> MobiledeScraper: s = Settings() s.log_level = "CRITICAL" s.database.url = "sqlite://" - return IAAIScraper(s) + return MobiledeScraper(s) def test_sync_vehicle_uses_db_record(self) -> None: scraper = self._make_scraper() @@ -37,7 +37,7 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) - result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") + result = scraper.sync_vehicle("https://www.MOBILEDE.com/VehicleDetail/111~US") self.assertEqual(result["status"], "success") self.assertIn("trace_id", result) @@ -50,13 +50,13 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.start_sync_run = MagicMock(return_value=2) scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( - {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"}, + {"https://www.MOBILEDE.com/VehicleDetail/111~US"}, {"mobilede:222"}, )) scraper.collect_listing = MagicMock(return_value={ "vehicle_urls": [ - "https://www.iaai.com/VehicleDetail/111~US", - "https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/VehicleDetail/333~US", + "https://www.MOBILEDE.com/VehicleDetail/111~US", + "https://www.MOBILEDE.com/VehicleDetail/222~US", + "https://www.MOBILEDE.com/VehicleDetail/333~US", ] }) scraper.sync_batch = MagicMock(return_value={ @@ -79,7 +79,7 @@ class TestScraperSync(unittest.TestCase): }) result2 = scraper2.sync_listing( only_new=True, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA", year_min=2020, year_max=2027, ) self.assertEqual(result2["cars_upserted"], 10) @@ -155,35 +155,35 @@ class TestScraperSync(unittest.TestCase): self.assertEqual(result["status"], "partial_success") def test_build_segment_listing_url(self) -> None: - base = "https://www.iaai.com/Vehiclelisting/Cars" + base = "https://www.MOBILEDE.com/Vehiclelisting/Cars" self.assertEqual( - IAAIScraper._build_segment_listing_url(base, "TOYOTA"), - "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + MobiledeScraper._build_segment_listing_url(base, "TOYOTA"), + "https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA", ) self.assertEqual( - IAAIScraper._build_segment_listing_url(base, "LAND ROVER"), - "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER", + MobiledeScraper._build_segment_listing_url(base, "LAND ROVER"), + "https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=LAND%20ROVER", ) - self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base) - self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base) + self.assertEqual(MobiledeScraper._build_segment_listing_url(base, None), base) + self.assertEqual(MobiledeScraper._build_segment_listing_url(base, ""), base) def test_guard_and_protection_detection(self) -> None: with self.assertRaises(AntiBotDetectedError): - IAAIScraper._raise_if_blocked_or_incomplete( + MobiledeScraper._raise_if_blocked_or_incomplete( {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, "access_notes": {}, "vehicle_summary": {}}, - "https://www.iaai.com/VehicleDetail/999~US", + "https://www.MOBILEDE.com/VehicleDetail/999~US", ) with self.assertRaises(SiteStructureChangedError): - IAAIScraper._raise_if_blocked_or_incomplete( + MobiledeScraper._raise_if_blocked_or_incomplete( {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, "access_notes": {"possible_captcha": False, "possible_antibot": False}, "vehicle_summary": {}}, - "https://www.iaai.com/VehicleDetail/999~US", + "https://www.MOBILEDE.com/VehicleDetail/999~US", ) - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) - self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) + self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) + self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) + self.assertFalse(MobiledeScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) def test_close_resets_browser_state(self) -> None: scraper = self._make_scraper() @@ -201,7 +201,7 @@ class TestScraperSync(unittest.TestCase): scraper = self._make_scraper() page = MagicMock() page_result = SimpleNamespace( - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")], + vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/123~US")], ) scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) scraper.listing_collector.open_cars_listing = MagicMock() @@ -228,7 +228,7 @@ class TestScraperSync(unittest.TestCase): page = MagicMock() page_result = SimpleNamespace( page_number=1, - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")], + vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/999~US", lot_number="999")], next_page_detected=False, ) @@ -245,7 +245,7 @@ class TestScraperSync(unittest.TestCase): "year_max": None, }) scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) - scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"]) + scraper._extract_page_urls = MagicMock(return_value=["https://www.MOBILEDE.com/VehicleDetail/999~US"]) scraper.sync_batch = MagicMock(return_value={ "cars_upserted": 1, "cars_failed": 0, @@ -256,11 +256,11 @@ class TestScraperSync(unittest.TestCase): result = scraper._sync_listing_streaming( make=None, model=None, - lane="iaai_cars", + lane="MOBILEDE_cars", limit=None, effective_only_new=False, started_at=0.0, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE", + listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=EAGLE", ) scraper.listing_collector.open_cars_listing.assert_called_once() @@ -278,9 +278,9 @@ class TestScraperSync(unittest.TestCase): }) urls = [ - "https://www.iaai.com/VehicleDetail/111~US", - "https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/VehicleDetail/333~US", + "https://www.MOBILEDE.com/VehicleDetail/111~US", + "https://www.MOBILEDE.com/VehicleDetail/222~US", + "https://www.MOBILEDE.com/VehicleDetail/333~US", ] first_record = CarRecord.model_validate(make_db_record("111")) @@ -298,8 +298,8 @@ class TestScraperSync(unittest.TestCase): yield 0, first_record raise FuturesTimeoutError() - with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ - patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ + with patch("mobilede_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ + patch("mobilede_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ patch.object(scraper, "_browser_fallback_parallel", return_value={ "records": [], "failures": [], diff --git a/tests/test_self_heal.py b/tests/test_self_heal.py index f5c74e1..fd09e7a 100644 --- a/tests/test_self_heal.py +++ b/tests/test_self_heal.py @@ -3,7 +3,7 @@ from __future__ import annotations import unittest from unittest.mock import MagicMock, patch -from iaai_scraper.worker import self_heal +from mobilede_scraper.worker import self_heal class TestSelfHeal(unittest.TestCase): @@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase): redis_client = MagicMock() redis_client.get.side_effect = lambda key: { self_heal.GLOBAL_PROGRESS_TS_KEY: None, - "iaai:state:task_progress:a": '{"ts": 100}', - "iaai:state:task_progress:b": '{"ts": 250}', - "iaai:state:task_progress:c": '{"ts": 150}', + "mobilede:state:task_progress:a": '{"ts": 100}', + "mobilede:state:task_progress:b": '{"ts": 250}', + "mobilede:state:task_progress:c": '{"ts": 150}', }.get(key) redis_client.scan_iter.return_value = [ - "iaai:state:task_progress:a", - "iaai:state:task_progress:b", - "iaai:state:task_progress:c", + "mobilede:state:task_progress:a", + "mobilede:state:task_progress:b", + "mobilede:state:task_progress:c", ] ts = self_heal._read_last_progress_ts(redis_client) @@ -38,20 +38,20 @@ class TestSelfHeal(unittest.TestCase): redis_client = MagicMock() redis_client.get.side_effect = lambda key: { self_heal.GLOBAL_PROGRESS_TS_KEY: None, - "iaai:state:task_progress:a": "{bad-json}", - "iaai:state:task_progress:b": '{"foo": "bar"}', + "mobilede:state:task_progress:a": "{bad-json}", + "mobilede:state:task_progress:b": '{"foo": "bar"}', }.get(key) redis_client.scan_iter.return_value = [ - "iaai:state:task_progress:a", - "iaai:state:task_progress:b", + "mobilede:state:task_progress:a", + "mobilede:state:task_progress:b", ] ts = self_heal._read_last_progress_ts(redis_client) self.assertIsNone(ts) - @patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) - @patch("iaai_scraper.worker.self_heal.os.kill") + @patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("mobilede_scraper.worker.self_heal.os.kill") @patch("builtins.open") def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None: open_mock.return_value.__enter__.return_value.read.return_value = "123" @@ -64,8 +64,8 @@ class TestSelfHeal(unittest.TestCase): self.assertEqual(kill_mock.call_args_list[1].args, (123, 0)) self.assertEqual(kill_mock.call_args_list[2].args[0], 123) - @patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) - @patch("iaai_scraper.worker.self_heal.os.kill") + @patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("mobilede_scraper.worker.self_heal.os.kill") @patch("builtins.open") def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None: open_mock.return_value.__enter__.return_value.read.return_value = "123" diff --git a/tests/test_utils.py b/tests/test_utils.py index 97f4ff6..1cfdcbe 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,9 +2,9 @@ from __future__ import annotations import unittest -from iaai_scraper.core.utils import deep_find_key -from iaai_scraper.core.config import ( - IAAI_DEFAULT_MAKES, +from mobilede_scraper.core.utils import deep_find_key +from mobilede_scraper.core.config import ( + MOBILEDE_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS, ProxyConfig, @@ -39,12 +39,12 @@ class TestParseListingSegments(unittest.TestCase): segs = parse_listing_segments("auto") # Точное число сегментов. - expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES)) + expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(MOBILEDE_DEFAULT_MAKES) - len(_LARGE_MAKES)) self.assertEqual(len(segs), expected) # Все бренды из списка присутствуют. makes_in_segments = {s["make"] for s in segs} - for make in IAAI_DEFAULT_MAKES: + for make in MOBILEDE_DEFAULT_MAKES: self.assertIn(make, makes_in_segments) # Крупные бренды разбиты на 3 сегмента с годами. diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py index b7e625b..664f29c 100644 --- a/tests/test_worker_tasks.py +++ b/tests/test_worker_tasks.py @@ -7,8 +7,8 @@ import tempfile import unittest from unittest.mock import MagicMock, patch -from iaai_scraper.worker import tasks -from iaai_scraper.core.config import Settings, settings as base_settings +from mobilede_scraper.worker import tasks +from mobilede_scraper.core.config import Settings, settings as base_settings class TestWorkerTaskLockHelpers(unittest.TestCase): @@ -212,7 +212,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.side_effect = lambda key: ( @@ -230,7 +230,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__exit__.return_value = None - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-resume-seg") try: result = tasks.sync_listing_task.run() @@ -254,7 +254,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() # Оставшийся чекпоинт не должен использоваться. @@ -273,7 +273,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__exit__.return_value = None - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-792") try: result = tasks.sync_listing_task.run() @@ -296,7 +296,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.side_effect = lambda key: ( @@ -314,7 +314,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__exit__.return_value = None - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-beyond") try: result = tasks.sync_listing_task.run() @@ -526,7 +526,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks, "_run_browser_job") as run_job, \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]): + patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=[]): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None @@ -573,7 +573,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.side_effect = lambda key: ( @@ -598,7 +598,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): scraper_ctx.__enter__.return_value.sync_listing = MagicMock() scraper_ctx.__exit__.return_value = None - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-always-full-resume") try: result = tasks.sync_listing_task.run()