refactor mobile.de parser, fix country mapping, update README
This commit is contained in:
qananasikq
@@ -1,162 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
from mobilede_scraper.browser.fast_client import (
|
||||
DETAIL_MARKER,
|
||||
LISTING_MARKER,
|
||||
build_resizer_images_from_keys,
|
||||
is_challenge_response,
|
||||
parse_listing_page,
|
||||
parse_product_details_vm,
|
||||
)
|
||||
from mobilede_scraper.parsing.fast_mapper import FastCarMapper
|
||||
|
||||
|
||||
def test_parse_listing_page_extracts_hidden_payloads() -> None:
|
||||
gbp = {"CurrentPage": 1, "PageSize": 100}
|
||||
vehicles = [
|
||||
{
|
||||
"Id": "45078011~US",
|
||||
"Tenant": "US",
|
||||
"InventoryStatus": "RS",
|
||||
"Currency": "USD",
|
||||
"PreBidIndicator": True,
|
||||
}
|
||||
]
|
||||
html = (
|
||||
f'<input id="GBPSearchQuery" value="{json.dumps(gbp).replace(chr(34), """)}" />'
|
||||
f'<input id="VehicleDetails" value="{json.dumps(vehicles).replace(chr(34), """)}" />'
|
||||
'<input id="ResultCount" value="1" />'
|
||||
'<input id="PageSize" value="100" />'
|
||||
'<input id="CurrentPage" value="1" />'
|
||||
)
|
||||
|
||||
parsed = parse_listing_page(html)
|
||||
|
||||
assert parsed.result_count == 1
|
||||
assert parsed.page_size == 100
|
||||
assert parsed.current_page == 1
|
||||
assert parsed.vehicles[0].inventory_id == "45078011~US"
|
||||
assert parsed.vehicles[0].inventory_status == "RS"
|
||||
|
||||
|
||||
def test_parse_product_details_vm_and_map_record() -> None:
|
||||
payload = {
|
||||
"inventoryView": {
|
||||
"attributes": {
|
||||
"Id": "45078011~US",
|
||||
"Year": "2002",
|
||||
"Make": "ACURA",
|
||||
"Model": "RSX",
|
||||
"Series": "BASE",
|
||||
"Currency": "USD",
|
||||
"ODOValue": "219187",
|
||||
"ExteriorColor": "GRAY",
|
||||
"DriveLineTypeDesc": "FWD",
|
||||
"Transmission": "Automatic",
|
||||
"BodyStyleName": "COUPE",
|
||||
"EngineSize": "2.0L I-4",
|
||||
"VehicleGrade": "50",
|
||||
"PrimaryDamageDesc": "NORMAL WEAR & TEAR",
|
||||
},
|
||||
"imageDimensions": {
|
||||
"keys": {
|
||||
"$values": [
|
||||
{"k": "45078011~US~I1", "w": 1600, "h": 1200, "i": 0},
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
"auctionInformation": {
|
||||
"prebidInformation": {"decimalHighBidAmount": "600", "highBidAmount": "$600"},
|
||||
"biddingInformation": {"buyNowPrice": "$0"},
|
||||
},
|
||||
}
|
||||
html = f'<script id="ProductDetailsVM" type="application/json">{json.dumps(payload)}</script>'
|
||||
|
||||
parsed = parse_product_details_vm(html)
|
||||
record = FastCarMapper().map_payload_to_record(
|
||||
detail_payload=parsed,
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/45078011~US",
|
||||
)
|
||||
|
||||
assert record.origin_id == "mobilede:45078011"
|
||||
assert record.brand == "ACURA"
|
||||
assert record.model == "RSX BASE"
|
||||
assert record.year == 2002
|
||||
assert record.price == 600
|
||||
assert record.drive == "FWD"
|
||||
assert record.gearbox == "AT"
|
||||
assert record.body_type == "COUPE"
|
||||
assert record.engine_volume == 2000
|
||||
assert record.is_damaged is False
|
||||
assert len(record.images) == 1
|
||||
|
||||
|
||||
def test_fast_mapper_uses_fallback_keys_and_normalization() -> None:
|
||||
payload = {
|
||||
"inventoryView": {
|
||||
"attributes": {
|
||||
"Id": "454502861",
|
||||
"FirstRegistration": "05/2012",
|
||||
"Make": "BMW",
|
||||
"Model": "X1",
|
||||
"Variant": "xDrive 20d",
|
||||
"Currency": "EUR",
|
||||
"Mileage": "50.100 km",
|
||||
"ColorDesc": "silber metallic",
|
||||
"Category": "Kombi",
|
||||
"TransmissionType": "Schaltgetriebe",
|
||||
"EngineInformation": "1995 ccm, Diesel",
|
||||
},
|
||||
"imageDimensions": {
|
||||
"keys": {
|
||||
"$values": [
|
||||
{"k": "454502861~I1", "w": 1600, "h": 1200, "i": 0},
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
"auctionInformation": {
|
||||
"biddingInformation": {"buyNowPrice": "9.900 €"},
|
||||
},
|
||||
}
|
||||
|
||||
record = FastCarMapper().map_payload_to_record(
|
||||
detail_payload=payload,
|
||||
vehicle_url="https://www.mobile.de/ru/транспортные-средства/подробности.html?id=454502861&vc=Car&s=Car",
|
||||
)
|
||||
|
||||
assert record.model == "X1 xDrive 20d"
|
||||
assert record.year == 2012
|
||||
assert record.price == 9900
|
||||
assert record.mileage == 50100
|
||||
assert record.color == "silver"
|
||||
assert record.drive == "4WD"
|
||||
assert record.gearbox == "MT"
|
||||
assert record.body_type == "STATION_WAGON"
|
||||
assert record.engine_volume == 1995
|
||||
|
||||
|
||||
def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None:
|
||||
keys = [
|
||||
{"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
|
||||
{"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
|
||||
{"k": "abc~2", "w": 1800, "h": 1200, "i": 1},
|
||||
]
|
||||
images = build_resizer_images_from_keys(keys)
|
||||
|
||||
assert len(images) == 2
|
||||
assert images[0]["order_index"] == 1
|
||||
assert "imageKeys=abc~2" in str(images[0]["fullres_image"])
|
||||
|
||||
|
||||
def test_is_challenge_response_marker_rules() -> None:
|
||||
assert is_challenge_response(status_code=403, body_text="", expected_marker=None) is True
|
||||
assert is_challenge_response(status_code=200, body_text="<html>blocked</html>", expected_marker=LISTING_MARKER) is True
|
||||
assert is_challenge_response(
|
||||
status_code=200,
|
||||
body_text=f'<html>{DETAIL_MARKER}<script src="/_Incapsula_Resource"></script></html>',
|
||||
expected_marker=DETAIL_MARKER,
|
||||
) is False
|
||||
@@ -1,197 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from types import SimpleNamespace
|
||||
|
||||
from mobilede_scraper.browser.fast_client import FastListingVehicle
|
||||
from mobilede_scraper.core.config import Settings
|
||||
from mobilede_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig
|
||||
from mobilede_scraper.fast_sync import FastSyncEngine
|
||||
from mobilede_scraper.parsing.fast_mapper import FastCarMapper
|
||||
|
||||
|
||||
def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle:
|
||||
return FastListingVehicle(
|
||||
inventory_id=inventory_id,
|
||||
tenant="US",
|
||||
auction_id="1_1",
|
||||
auction_date="2026-04-08T08:30:00+00:00",
|
||||
inventory_status=status,
|
||||
currency="USD",
|
||||
timed_auction_closed=False,
|
||||
timed_auction_indicator=False,
|
||||
prebid_indicator=True,
|
||||
buynow_indicator=False,
|
||||
)
|
||||
|
||||
|
||||
def _detail_payload(inventory_id: str, *, make: str = "ACURA", model: str = "RSX", bid: int = 500) -> dict:
|
||||
return {
|
||||
"inventoryView": {
|
||||
"attributes": {
|
||||
"Id": inventory_id,
|
||||
"Year": "2002",
|
||||
"Make": make,
|
||||
"Model": model,
|
||||
"Series": "BASE",
|
||||
"Currency": "USD",
|
||||
"ODOValue": "219187",
|
||||
"ExteriorColor": "GRAY",
|
||||
"DriveLineTypeDesc": "FWD",
|
||||
"Transmission": "Automatic",
|
||||
"BodyStyleName": "COUPE",
|
||||
"EngineSize": "2.0L I-4",
|
||||
"VehicleGrade": "50",
|
||||
"PrimaryDamageDesc": "NORMAL WEAR & TEAR",
|
||||
},
|
||||
"imageDimensions": {
|
||||
"keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]}
|
||||
},
|
||||
},
|
||||
"auctionInformation": {
|
||||
"prebidInformation": {"decimalHighBidAmount": str(bid), "highBidAmount": f"${bid}"},
|
||||
"biddingInformation": {"buyNowPrice": "$0"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _detail_payload_with_fallbacks(inventory_id: str) -> dict:
|
||||
return {
|
||||
"inventoryView": {
|
||||
"attributes": {
|
||||
"Id": inventory_id,
|
||||
"FirstRegistration": "2011-03",
|
||||
"Make": "BMW",
|
||||
"Model": "120",
|
||||
"Variant": "Cabrio",
|
||||
"Currency": "EUR",
|
||||
"Kilometerstand": "95 000",
|
||||
"Color": "weiss",
|
||||
"DriveType": "Front wheel drive",
|
||||
"Gearbox": "Automatik",
|
||||
"VehicleClass": "Cabrio",
|
||||
"EngineInformation": "2,0 l",
|
||||
"VehicleGrade": "A",
|
||||
},
|
||||
"imageDimensions": {
|
||||
"keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]}
|
||||
},
|
||||
},
|
||||
"auctionInformation": {
|
||||
"biddingInformation": {"buyNowPrice": "8.899 €"},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
class FakeFastClient:
|
||||
def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None:
|
||||
self.listing = listing
|
||||
self.details = details
|
||||
self.detail_calls = 0
|
||||
self.persist_calls = 0
|
||||
self._settings = SimpleNamespace(
|
||||
scraping_profile=SimpleNamespace(
|
||||
verbose_progress_logs=False,
|
||||
request_jitter_max_s=0.0,
|
||||
)
|
||||
)
|
||||
|
||||
def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None): # noqa: ANN001, ANN202
|
||||
del listing_start_url, make, max_pages
|
||||
return iter(self.listing)
|
||||
|
||||
def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict:
|
||||
self.detail_calls += 1
|
||||
return self.details[inventory_id]
|
||||
|
||||
def persist_session_state(self) -> None:
|
||||
self.persist_calls += 1
|
||||
|
||||
|
||||
@dataclass
|
||||
class FakePersistence:
|
||||
inserted: int = 0
|
||||
images: int = 0
|
||||
|
||||
def get_existing_urls_and_ids(self, origin_urls, origin_ids): # noqa: ANN001, ANN202
|
||||
del origin_urls, origin_ids
|
||||
return set(), set()
|
||||
|
||||
def upsert_cars_batch(self, records): # noqa: ANN001, ANN202
|
||||
self.inserted += len(records)
|
||||
self.images += sum(len(record.images) for record in records)
|
||||
return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)}
|
||||
|
||||
def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="MOBILEDE"): # noqa: ANN001, ANN202
|
||||
del active_origin_urls, lane
|
||||
return 0
|
||||
|
||||
|
||||
def test_fast_sync_engine_collects_details_and_bulk_upserts() -> None:
|
||||
listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
|
||||
details = {
|
||||
"45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
|
||||
"45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
|
||||
}
|
||||
client = FakeFastClient(listing, details)
|
||||
persistence = FakePersistence()
|
||||
engine = FastSyncEngine(
|
||||
client=client, # type: ignore[arg-type]
|
||||
mapper=FastCarMapper(),
|
||||
persistence=persistence, # type: ignore[arg-type]
|
||||
batch_size=10,
|
||||
fetch_concurrency=2,
|
||||
)
|
||||
|
||||
result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False)
|
||||
|
||||
assert result["status"] == "success"
|
||||
assert result["cars_upserted"] == 2
|
||||
assert result["images_upserted"] == 2
|
||||
assert result["listing"]["mode"] == "fast_hidden_payload"
|
||||
assert client.detail_calls == 2
|
||||
assert client.persist_calls == 1
|
||||
|
||||
|
||||
def test_fast_sync_engine_applies_runtime_filters_before_db() -> None:
|
||||
listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
|
||||
details = {
|
||||
"45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
|
||||
"45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
|
||||
}
|
||||
runtime = RuntimeConfig(filters=RuntimeFiltersConfig.from_dict({"brands": ["BMW"]}))
|
||||
persistence = FakePersistence()
|
||||
engine = FastSyncEngine(
|
||||
client=FakeFastClient(listing, details), # type: ignore[arg-type]
|
||||
mapper=FastCarMapper(),
|
||||
persistence=persistence, # type: ignore[arg-type]
|
||||
batch_size=10,
|
||||
fetch_concurrency=2,
|
||||
)
|
||||
|
||||
result = engine.sync_listing(runtime_config=runtime, only_new=False)
|
||||
|
||||
assert result["cars_upserted"] == 1
|
||||
assert result["cars_filtered"] == 1
|
||||
assert persistence.inserted == 1
|
||||
|
||||
|
||||
def test_fast_sync_engine_maps_fallback_mobilede_fields() -> None:
|
||||
listing = [_listing_vehicle("449252166")]
|
||||
details = {
|
||||
"449252166": _detail_payload_with_fallbacks("449252166"),
|
||||
}
|
||||
persistence = FakePersistence()
|
||||
engine = FastSyncEngine(
|
||||
client=FakeFastClient(listing, details), # type: ignore[arg-type]
|
||||
mapper=FastCarMapper(),
|
||||
persistence=persistence, # type: ignore[arg-type]
|
||||
batch_size=10,
|
||||
fetch_concurrency=1,
|
||||
)
|
||||
|
||||
result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False)
|
||||
|
||||
assert result["cars_upserted"] == 1
|
||||
assert persistence.inserted == 1
|
||||
assert persistence.images == 1
|
||||
@@ -1,68 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from mobilede_scraper.browser.pace import HumanPacer
|
||||
from mobilede_scraper.core.config import Settings
|
||||
from mobilede_scraper.browser.listing import ListingCollector
|
||||
|
||||
|
||||
class _FakePage:
|
||||
def __init__(self, counts: dict[str, int]) -> None:
|
||||
self._counts = counts
|
||||
self._evaluate_result = False
|
||||
self._evaluate_values: list[object] = []
|
||||
|
||||
class _Locator:
|
||||
def __init__(self, count_value: int) -> None:
|
||||
self._count_value = count_value
|
||||
|
||||
def count(self) -> int:
|
||||
return self._count_value
|
||||
|
||||
def locator(self, selector: str) -> "_FakePage._Locator":
|
||||
return _FakePage._Locator(self._counts.get(selector, 0))
|
||||
|
||||
def evaluate(self, _script: str):
|
||||
if self._evaluate_values:
|
||||
return self._evaluate_values.pop(0)
|
||||
return self._evaluate_result
|
||||
|
||||
|
||||
class TestListingUnit(unittest.TestCase):
|
||||
def test_pagination_detection_and_page_number(self) -> None:
|
||||
# Есть кнопка Next → True.
|
||||
self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1})))
|
||||
# Нет селекторов → False.
|
||||
self.assertFalse(ListingCollector._has_next_page(_FakePage({})))
|
||||
# Числовая пагинация через JS → True только если evaluate явно нашёл next.
|
||||
page = _FakePage({})
|
||||
page._evaluate_result = True
|
||||
self.assertTrue(ListingCollector._has_next_page(page))
|
||||
# Номер текущей страницы.
|
||||
page2 = _FakePage({})
|
||||
page2._evaluate_values = [5]
|
||||
self.assertEqual(ListingCollector._get_current_page_number(page2), 5)
|
||||
|
||||
def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
|
||||
collector = ListingCollector(Settings(), HumanPacer(Settings()))
|
||||
html = """
|
||||
<div>
|
||||
<h4><a href=\"/VehicleDetail/45184893~US\">Car 1</a></h4>
|
||||
<script>window.__data = {\"href\":\"\\/VehicleDetail\\/45171480~US\"}</script>
|
||||
</div>
|
||||
"""
|
||||
|
||||
links = collector._extract_vehicle_links_from_html(html)
|
||||
|
||||
self.assertEqual(
|
||||
links,
|
||||
[
|
||||
("https://www.MOBILEDE.com/VehicleDetail/45184893~US", "45184893"),
|
||||
("https://www.MOBILEDE.com/VehicleDetail/45171480~US", "45171480"),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -2,92 +2,87 @@ from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from mobilede_scraper.parsing.mapper import CarMapper
|
||||
from mobilede_scraper.mobile_de.mapper import MobileDeMapper
|
||||
from mobilede_scraper.mobile_de.models import MobileDeListing
|
||||
|
||||
|
||||
class TestCarMapper(unittest.TestCase):
|
||||
class TestMobileDeMapper(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.mapper = CarMapper()
|
||||
self.mapper = MobileDeMapper()
|
||||
|
||||
def test_deduplicates_images_by_image_key(self) -> None:
|
||||
def test_deduplicates_images(self) -> None:
|
||||
urls = [
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=1&width=200&height=150",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=1&width=845&height=633",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=2&width=400&height=300",
|
||||
"https://img.classistatic.de/api/v1/mo-prod/images/1",
|
||||
"https://img.classistatic.de/api/v1/mo-prod/images/1",
|
||||
"//img.classistatic.de/api/v1/mo-prod/images/2",
|
||||
]
|
||||
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US",
|
||||
vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
|
||||
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
|
||||
record = self.mapper.listing_to_car_record(
|
||||
MobileDeListing(
|
||||
id="123",
|
||||
url="https://suchen.mobile.de/fahrzeuge/details.html?id=123",
|
||||
title="Honda Civic",
|
||||
raw={"images": urls},
|
||||
)
|
||||
)
|
||||
|
||||
self.assertEqual(len(record.images), 2)
|
||||
self.assertIn("width=845", record.images[0].fullres_image)
|
||||
self.assertIn("height=633", record.images[0].fullres_image)
|
||||
self.assertEqual(record.images[0].fullres_image, urls[0])
|
||||
self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2")
|
||||
|
||||
def test_no_damage_marker_is_not_damaged(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/123~US",
|
||||
vehicle_summary={"make": "Ford", "model": "Focus"},
|
||||
payload_insights={
|
||||
"vehicle_core": {},
|
||||
"pricing": {},
|
||||
"damage": {"primary": "normal wear"},
|
||||
"auction": {},
|
||||
"images": {},
|
||||
},
|
||||
def test_origin_id_uses_canonical_prefix(self) -> None:
|
||||
record = self.mapper.listing_to_car_record(
|
||||
MobileDeListing(
|
||||
id="456",
|
||||
url="https://suchen.mobile.de/fahrzeuge/details.html?id=456",
|
||||
title="Ford Focus",
|
||||
)
|
||||
)
|
||||
|
||||
self.assertFalse(record.is_damaged)
|
||||
self.assertEqual(record.origin_id, "mobile.de:456")
|
||||
self.assertEqual(record.origin, "MOBILE_DE")
|
||||
|
||||
def test_unknown_empty_values_and_normalization(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/999~US",
|
||||
vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
|
||||
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
|
||||
record = self.mapper.listing_to_car_record(
|
||||
MobileDeListing(
|
||||
id="999",
|
||||
url="https://suchen.mobile.de/fahrzeuge/details.html?id=999",
|
||||
title="",
|
||||
subtitle="",
|
||||
raw={"make": {"localized": " "}, "model": {"localized": ""}, "attr": {"tr": "unknown"}},
|
||||
)
|
||||
)
|
||||
self.assertEqual(record.brand, "UNKNOWN")
|
||||
self.assertEqual(record.model, "UNKNOWN")
|
||||
self.assertEqual(record.drive, "NA")
|
||||
self.assertEqual(record.gearbox, "NA")
|
||||
self.assertIsNone(record.drive)
|
||||
self.assertIsNone(record.gearbox)
|
||||
|
||||
# Нормализация регистра и пробелов.
|
||||
record2 = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/888~US",
|
||||
vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
|
||||
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
|
||||
record2 = self.mapper.listing_to_car_record(
|
||||
MobileDeListing(
|
||||
id="888",
|
||||
url="https://suchen.mobile.de/fahrzeuge/details.html?id=888",
|
||||
title="Honda Civic",
|
||||
transmission=" Automatik ",
|
||||
)
|
||||
)
|
||||
self.assertEqual(record2.drive, "FWD")
|
||||
self.assertEqual(record2.gearbox, "AT")
|
||||
|
||||
def test_price_and_currency_parsing(self) -> None:
|
||||
record = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/777~US",
|
||||
vehicle_summary={"make": "Toyota", "model": "Corolla"},
|
||||
payload_insights={
|
||||
"vehicle_core": {},
|
||||
"pricing": {"buy_now": "USD 4,500 - 5,200"},
|
||||
"damage": {},
|
||||
"auction": {},
|
||||
"images": {},
|
||||
},
|
||||
record = self.mapper.listing_to_car_record(
|
||||
MobileDeListing(
|
||||
id="777",
|
||||
url="https://suchen.mobile.de/fahrzeuge/details.html?id=777",
|
||||
title="Toyota Corolla",
|
||||
price="€ 5.200",
|
||||
first_registration="05/2019",
|
||||
mileage="50.100 km",
|
||||
)
|
||||
)
|
||||
self.assertEqual(record.price, 5200)
|
||||
|
||||
record2 = self.mapper.map_to_car_record(
|
||||
vehicle_url="https://www.MOBILEDE.com/VehicleDetail/778~US",
|
||||
vehicle_summary={"make": "Toyota", "model": "Corolla"},
|
||||
payload_insights={
|
||||
"vehicle_core": {},
|
||||
"pricing": {"buy_now": "€4.500,00"},
|
||||
"damage": {},
|
||||
"auction": {},
|
||||
"images": {},
|
||||
},
|
||||
)
|
||||
self.assertEqual(record2.currency, "EUR")
|
||||
self.assertEqual(record2.price, 4500)
|
||||
self.assertEqual(record.currency, "EUR")
|
||||
self.assertEqual(record.year, 2019)
|
||||
self.assertEqual(record.mileage, 50100)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
87
tests/test_mobilede_scraper.py
Normal file
87
tests/test_mobilede_scraper.py
Normal file
@@ -0,0 +1,87 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
|
||||
from mobilede_scraper.mobile_de.scraper import MobileDeScraper
|
||||
|
||||
|
||||
class _FakePersistence:
|
||||
def __init__(self) -> None:
|
||||
self.upsert_calls: list[list[str]] = []
|
||||
self.finish_payload: dict[str, object] | None = None
|
||||
|
||||
def create_tables(self) -> None:
|
||||
return None
|
||||
|
||||
def start_sync_run(self, lane: str) -> int:
|
||||
return 1
|
||||
|
||||
def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
|
||||
return set()
|
||||
|
||||
def upsert_cars_batch(self, records):
|
||||
self.upsert_calls.append([record.origin_id for record in records])
|
||||
return {
|
||||
"inserted": len(records),
|
||||
"updated": 0,
|
||||
"images_upserted": len(records),
|
||||
}
|
||||
|
||||
def finish_sync_run(self, run_id: int, **kwargs) -> None:
|
||||
self.finish_payload = {"run_id": run_id, **kwargs}
|
||||
|
||||
|
||||
class _FakeClient:
|
||||
def __init__(self, pages: list[MobileDeSearchPage]) -> None:
|
||||
self.pages = pages
|
||||
|
||||
def build_make_model_param(self, make_id: str, model_id: str | None) -> str:
|
||||
return make_id if not model_id else f"{make_id};{model_id}"
|
||||
|
||||
def iter_search_pages(self, **kwargs):
|
||||
del kwargs
|
||||
yield from self.pages
|
||||
|
||||
|
||||
class TestMobileDeScraperStreamingSync(unittest.TestCase):
|
||||
def test_sync_search_upserts_each_page_during_run(self) -> None:
|
||||
pages = [
|
||||
MobileDeSearchPage(
|
||||
url="https://example.test/page-1",
|
||||
page_number=1,
|
||||
total_results=3,
|
||||
listings=[
|
||||
MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
|
||||
MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
|
||||
],
|
||||
),
|
||||
MobileDeSearchPage(
|
||||
url="https://example.test/page-2",
|
||||
page_number=2,
|
||||
total_results=3,
|
||||
listings=[
|
||||
MobileDeListing(id="3", url="https://example.test/3", title="Car 3"),
|
||||
],
|
||||
),
|
||||
]
|
||||
persistence = _FakePersistence()
|
||||
scraper = MobileDeScraper(
|
||||
client=_FakeClient(pages),
|
||||
persistence=persistence,
|
||||
)
|
||||
|
||||
result = scraper.sync_search(max_pages=2)
|
||||
|
||||
self.assertEqual(len(persistence.upsert_calls), 2)
|
||||
self.assertEqual(persistence.upsert_calls[0], ["mobile.de:1", "mobile.de:2"])
|
||||
self.assertEqual(persistence.upsert_calls[1], ["mobile.de:3"])
|
||||
self.assertEqual(int(result["upsert"]["inserted"]), 3)
|
||||
self.assertEqual(int(result["listing_count"]), 3)
|
||||
self.assertEqual(int(result["unique_listing_count"]), 3)
|
||||
self.assertIsNotNone(persistence.finish_payload)
|
||||
self.assertEqual(int(persistence.finish_payload["cars_upserted"]), 3)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,54 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
|
||||
from mobilede_scraper.parsing.parser import VehicleParser
|
||||
|
||||
|
||||
class TestVehicleParserUnit(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
self.parser = VehicleParser()
|
||||
|
||||
def test_parse_dom_pairs_and_title(self) -> None:
|
||||
dom_text = """
|
||||
Stock #:
|
||||
45089484
|
||||
Primary Damage:
|
||||
Front End
|
||||
Odometer:
|
||||
50,123 mi (Actual)
|
||||
"""
|
||||
result = self.parser._parse_dom_key_value_pairs(dom_text)
|
||||
self.assertEqual(result.get("lot_number"), "45089484")
|
||||
self.assertEqual(result.get("primary_damage"), "Front End")
|
||||
self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
|
||||
|
||||
parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")
|
||||
self.assertEqual(parsed["year"], "2014")
|
||||
self.assertEqual(parsed["make"], "TOYOTA")
|
||||
self.assertEqual(parsed["model"], "CAMRY")
|
||||
|
||||
def test_extract_image_urls_filters_and_deduplicates(self) -> None:
|
||||
vehicle_url = "https://www.MOBILEDE.com/VehicleDetail/45089484~US"
|
||||
payloads = [{"imageUrls": [
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
|
||||
"https://vis.MOBILEDE.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
|
||||
]}]
|
||||
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
|
||||
self.assertEqual(len(urls), 1)
|
||||
self.assertIn("45089484", urls[0])
|
||||
|
||||
def test_dom_hints_and_access_notes_detect_antibot(self) -> None:
|
||||
hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.")
|
||||
self.assertTrue(hints["has_captcha_text"])
|
||||
self.assertTrue(hints["has_antibot_text"])
|
||||
|
||||
summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."}
|
||||
notes = self.parser._build_access_notes(summary, [])
|
||||
self.assertTrue(notes["possible_captcha"])
|
||||
self.assertTrue(notes["possible_antibot"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,79 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from mobilede_scraper.scraper import MobiledeScraper
|
||||
from mobilede_scraper.core.config import Settings
|
||||
from mobilede_scraper.worker import tasks
|
||||
|
||||
|
||||
class TestResilience(unittest.TestCase):
|
||||
def _make_scraper(self) -> MobiledeScraper:
|
||||
s = Settings()
|
||||
s.log_level = "CRITICAL"
|
||||
s.database.url = "sqlite://"
|
||||
return MobiledeScraper(s)
|
||||
|
||||
def test_update_task_progress_updates_global_marker(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
pipe = MagicMock()
|
||||
redis_client.pipeline.return_value = pipe
|
||||
|
||||
tasks._update_task_progress(
|
||||
redis_client,
|
||||
task_id="task-abc",
|
||||
stage="batch_upserted",
|
||||
ttl_seconds=180,
|
||||
cars_upserted=10,
|
||||
)
|
||||
|
||||
redis_client.pipeline.assert_called_once()
|
||||
self.assertEqual(pipe.set.call_count, 2)
|
||||
first_call = pipe.set.call_args_list[0]
|
||||
second_call = pipe.set.call_args_list[1]
|
||||
|
||||
self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc"))
|
||||
self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY)
|
||||
pipe.execute.assert_called_once()
|
||||
|
||||
def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.celery.batch_size = 50
|
||||
|
||||
page = MagicMock()
|
||||
empty_page_result = SimpleNamespace(
|
||||
page_number=1,
|
||||
vehicle_links=[],
|
||||
next_page_detected=True,
|
||||
)
|
||||
|
||||
scraper._open_listing_for_stream = MagicMock(return_value=(
|
||||
page,
|
||||
{"make": None, "model": None, "year_min": None, "year_max": None},
|
||||
))
|
||||
scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result)
|
||||
scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, []))
|
||||
scraper.sync_batch = MagicMock()
|
||||
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="MOBILEDE_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TEST",
|
||||
)
|
||||
|
||||
self.assertEqual(result["total"], 0)
|
||||
self.assertEqual(result["cars_upserted"], 0)
|
||||
self.assertEqual(result["cars_failed"], 0)
|
||||
self.assertEqual(result["listing"]["pages_collected"], 1)
|
||||
scraper._recover_empty_listing_page.assert_called_once()
|
||||
scraper.sync_batch.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,319 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from concurrent.futures import TimeoutError as FuturesTimeoutError
|
||||
from types import SimpleNamespace
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from mobilede_scraper.core.config import Settings
|
||||
from mobilede_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
|
||||
from mobilede_scraper.scraper import MobiledeScraper
|
||||
from mobilede_scraper.storage.schemas import CarRecord
|
||||
|
||||
|
||||
def make_db_record(origin_id: str) -> dict[str, object]:
|
||||
return CarRecord(
|
||||
parser_id=f"mobilede:{origin_id}",
|
||||
brand="Toyota",
|
||||
model="Camry",
|
||||
origin_url=f"https://www.MOBILEDE.com/VehicleDetail/{origin_id}~US",
|
||||
origin_id=origin_id,
|
||||
slug=f"toyota-camry-{origin_id}",
|
||||
).model_dump(mode="json")
|
||||
|
||||
|
||||
class TestScraperSync(unittest.TestCase):
|
||||
def _make_scraper(self) -> MobiledeScraper:
|
||||
s = Settings()
|
||||
s.log_level = "CRITICAL"
|
||||
s.database.url = "sqlite://"
|
||||
return MobiledeScraper(s)
|
||||
|
||||
def test_sync_vehicle_uses_db_record(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=1)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
|
||||
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
|
||||
|
||||
result = scraper.sync_vehicle("https://www.MOBILEDE.com/VehicleDetail/111~US")
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertIn("trace_id", result)
|
||||
scraper.persistence.upsert_car.assert_called_once()
|
||||
|
||||
def test_only_new_routing_legacy_and_streaming(self) -> None:
|
||||
# Legacy path: only_new без listing_url.
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=2)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
|
||||
{"https://www.MOBILEDE.com/VehicleDetail/111~US"}, {"mobilede:222"},
|
||||
))
|
||||
scraper.collect_listing = MagicMock(return_value={
|
||||
"vehicle_urls": [
|
||||
"https://www.MOBILEDE.com/VehicleDetail/111~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/222~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/333~US",
|
||||
]
|
||||
})
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
|
||||
})
|
||||
result = scraper.sync_listing(only_new=True)
|
||||
self.assertEqual(result["skipped_existing"], 2)
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
|
||||
# Streaming path: only_new + listing_url.
|
||||
scraper2 = self._make_scraper()
|
||||
scraper2.persistence.create_tables = MagicMock()
|
||||
scraper2.persistence.start_sync_run = MagicMock(return_value=6)
|
||||
scraper2.persistence.finish_sync_run = MagicMock()
|
||||
scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used"))
|
||||
scraper2._sync_listing_streaming = MagicMock(return_value={
|
||||
"listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False},
|
||||
"total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0,
|
||||
"images_upserted": 20, "failures": [], "all_listing_origin_urls": set(),
|
||||
})
|
||||
result2 = scraper2.sync_listing(
|
||||
only_new=True,
|
||||
listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
year_min=2020, year_max=2027,
|
||||
)
|
||||
self.assertEqual(result2["cars_upserted"], 10)
|
||||
scraper2._sync_listing_streaming.assert_called_once()
|
||||
scraper2.collect_listing.assert_not_called()
|
||||
|
||||
def test_segmented_sync_calls_per_segment_and_resumes(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=3)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
|
||||
call_args_log: list[dict] = []
|
||||
def _fake_sync_listing(**kwargs):
|
||||
call_args_log.append(kwargs)
|
||||
return {
|
||||
"status": "success", "cars_upserted": 5, "cars_failed": 0,
|
||||
"images_upserted": 10, "skipped_existing": 0,
|
||||
"listing": {"vehicles_collected": 50}, "failures": [],
|
||||
}
|
||||
|
||||
scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing)
|
||||
segments = [
|
||||
{"make": "TOYOTA", "year_min": 2020, "year_max": 2027},
|
||||
{"make": "FORD", "year_min": None, "year_max": None},
|
||||
{"make": "HONDA", "year_min": None, "year_max": None},
|
||||
]
|
||||
|
||||
# Resume: пропускаем TOYOTA, начинаем сразу с FORD.
|
||||
result = scraper.sync_listing_segmented(
|
||||
segments=segments, start_segment=1,
|
||||
)
|
||||
|
||||
self.assertEqual(result["segments_completed"], 2) # FORD + HONDA
|
||||
self.assertEqual(result["cars_upserted"], 10)
|
||||
# URL содержит бренд.
|
||||
self.assertIn("FORD", call_args_log[0]["listing_url"])
|
||||
self.assertIn("HONDA", call_args_log[1]["listing_url"])
|
||||
|
||||
def test_segmented_sync_does_not_mark_full_scan_completed_when_segment_failed(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.sync_listing = MagicMock(side_effect=[
|
||||
{
|
||||
"status": "failed",
|
||||
"full_scan_completed": False,
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"listing": {"vehicles_collected": 0},
|
||||
"failures": [{"vehicle_url": "segment", "error": "resume failed"}],
|
||||
},
|
||||
{
|
||||
"status": "success",
|
||||
"full_scan_completed": True,
|
||||
"cars_upserted": 1,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"listing": {"vehicles_collected": 10},
|
||||
"failures": [],
|
||||
},
|
||||
])
|
||||
|
||||
result = scraper.sync_listing_segmented(
|
||||
segments=[
|
||||
{"make": "EAGLE", "year_min": None, "year_max": None},
|
||||
{"make": "FORD", "year_min": None, "year_max": None},
|
||||
]
|
||||
)
|
||||
|
||||
self.assertFalse(result["full_scan_completed"])
|
||||
self.assertEqual(result["status"], "partial_success")
|
||||
|
||||
def test_build_segment_listing_url(self) -> None:
|
||||
base = "https://www.MOBILEDE.com/Vehiclelisting/Cars"
|
||||
self.assertEqual(
|
||||
MobiledeScraper._build_segment_listing_url(base, "TOYOTA"),
|
||||
"https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=TOYOTA",
|
||||
)
|
||||
self.assertEqual(
|
||||
MobiledeScraper._build_segment_listing_url(base, "LAND ROVER"),
|
||||
"https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
|
||||
)
|
||||
self.assertEqual(MobiledeScraper._build_segment_listing_url(base, None), base)
|
||||
self.assertEqual(MobiledeScraper._build_segment_listing_url(base, ""), base)
|
||||
|
||||
def test_guard_and_protection_detection(self) -> None:
|
||||
with self.assertRaises(AntiBotDetectedError):
|
||||
MobiledeScraper._raise_if_blocked_or_incomplete(
|
||||
{"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
|
||||
"access_notes": {}, "vehicle_summary": {}},
|
||||
"https://www.MOBILEDE.com/VehicleDetail/999~US",
|
||||
)
|
||||
with self.assertRaises(SiteStructureChangedError):
|
||||
MobiledeScraper._raise_if_blocked_or_incomplete(
|
||||
{"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
|
||||
"access_notes": {"possible_captcha": False, "possible_antibot": False},
|
||||
"vehicle_summary": {}},
|
||||
"https://www.MOBILEDE.com/VehicleDetail/999~US",
|
||||
)
|
||||
self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
|
||||
self.assertTrue(MobiledeScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
|
||||
self.assertFalse(MobiledeScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
|
||||
|
||||
def test_close_resets_browser_state(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
http_pool = MagicMock()
|
||||
scraper._http_pool = http_pool
|
||||
scraper.context = MagicMock()
|
||||
scraper.browser = MagicMock()
|
||||
scraper.playwright = MagicMock()
|
||||
scraper.close()
|
||||
http_pool.clear.assert_called_once()
|
||||
self.assertIsNone(scraper.context)
|
||||
self.assertIsNone(scraper.browser)
|
||||
|
||||
def test_recover_empty_listing_page(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
page = MagicMock()
|
||||
page_result = SimpleNamespace(
|
||||
vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/123~US")],
|
||||
)
|
||||
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
|
||||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||||
|
||||
# Страница > 1: reload.
|
||||
_, urls = scraper._recover_empty_listing_page(
|
||||
page, page_number=5, all_raw_urls=[], seen_urls=set(),
|
||||
)
|
||||
page.reload.assert_called_once()
|
||||
self.assertEqual(len(urls), 1)
|
||||
|
||||
# Страница 1: переоткрытие листинга.
|
||||
page.reset_mock()
|
||||
_, urls = scraper._recover_empty_listing_page(
|
||||
page, page_number=1, all_raw_urls=[], seen_urls=set(),
|
||||
)
|
||||
scraper.listing_collector.open_cars_listing.assert_called_once()
|
||||
page.reload.assert_not_called()
|
||||
|
||||
def test_sync_listing_always_starts_from_page_one(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.settings.celery.batch_size = 1
|
||||
|
||||
page = MagicMock()
|
||||
page_result = SimpleNamespace(
|
||||
page_number=1,
|
||||
vehicle_links=[SimpleNamespace(href="https://www.MOBILEDE.com/VehicleDetail/999~US", lot_number="999")],
|
||||
next_page_detected=False,
|
||||
)
|
||||
|
||||
scraper._get_page_with_warmup = MagicMock(return_value=page)
|
||||
# Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
|
||||
scraper._reopen_listing_and_resume = MagicMock(
|
||||
side_effect=AssertionError("pagination resume must not be used")
|
||||
)
|
||||
scraper.listing_collector.open_cars_listing = MagicMock()
|
||||
scraper.listing_collector.apply_filters = MagicMock(return_value={
|
||||
"make": None,
|
||||
"model": None,
|
||||
"year_min": None,
|
||||
"year_max": None,
|
||||
})
|
||||
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
|
||||
scraper._extract_page_urls = MagicMock(return_value=["https://www.MOBILEDE.com/VehicleDetail/999~US"])
|
||||
scraper.sync_batch = MagicMock(return_value={
|
||||
"cars_upserted": 1,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"failures": [],
|
||||
})
|
||||
|
||||
result = scraper._sync_listing_streaming(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="MOBILEDE_cars",
|
||||
limit=None,
|
||||
effective_only_new=False,
|
||||
started_at=0.0,
|
||||
listing_url="https://www.MOBILEDE.com/Vehiclelisting/Cars?Make=EAGLE",
|
||||
)
|
||||
|
||||
scraper.listing_collector.open_cars_listing.assert_called_once()
|
||||
scraper._reopen_listing_and_resume.assert_not_called()
|
||||
scraper.sync_batch.assert_called_once()
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
self.assertEqual(result["total"], 1)
|
||||
|
||||
def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.upsert_cars_batch = MagicMock(return_value={
|
||||
"inserted": 1,
|
||||
"updated": 0,
|
||||
"images_upserted": 0,
|
||||
})
|
||||
|
||||
urls = [
|
||||
"https://www.MOBILEDE.com/VehicleDetail/111~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/222~US",
|
||||
"https://www.MOBILEDE.com/VehicleDetail/333~US",
|
||||
]
|
||||
first_record = CarRecord.model_validate(make_db_record("111"))
|
||||
|
||||
class _FakeExecutor:
|
||||
def __init__(self, *args, **kwargs):
|
||||
pass
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def map(self, fn, iterable, timeout=None): # noqa: ARG002
|
||||
yield 0, first_record
|
||||
raise FuturesTimeoutError()
|
||||
|
||||
with patch("mobilede_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
|
||||
patch("mobilede_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
|
||||
patch.object(scraper, "_browser_fallback_parallel", return_value={
|
||||
"records": [],
|
||||
"failures": [],
|
||||
"cars_failed": 0,
|
||||
"protection_events": 0,
|
||||
}) as fallback_mock:
|
||||
result = scraper.sync_batch(urls)
|
||||
|
||||
self.assertEqual(result["cars_upserted"], 1)
|
||||
fallback_urls = fallback_mock.call_args.args[0]
|
||||
self.assertEqual(len(fallback_urls), 2)
|
||||
self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
|
||||
self.assertNotIn(urls[0], {u for u, _ in fallback_urls})
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -50,6 +50,16 @@ class TestSelfHeal(unittest.TestCase):
|
||||
|
||||
self.assertIsNone(ts)
|
||||
|
||||
def test_has_inflight_work_uses_canonical_segment_locks(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.llen.return_value = 0
|
||||
redis_client.scan_iter.side_effect = [iter(["mobilede:locks:segment:abc"]), iter([])]
|
||||
|
||||
has_inflight, flags = self_heal._has_inflight_work(redis_client, "mobilede_sync")
|
||||
|
||||
self.assertTrue(has_inflight)
|
||||
self.assertEqual(flags["has_segment_lock"], 1)
|
||||
|
||||
@patch("mobilede_scraper.worker.self_heal.time.sleep", return_value=None)
|
||||
@patch("mobilede_scraper.worker.self_heal.os.kill")
|
||||
@patch("builtins.open")
|
||||
|
||||
@@ -4,12 +4,7 @@ import unittest
|
||||
|
||||
from mobilede_scraper.core.utils import deep_find_key
|
||||
from mobilede_scraper.core.config import (
|
||||
MOBILEDE_DEFAULT_MAKES,
|
||||
_LARGE_MAKES,
|
||||
_YEAR_SPLITS,
|
||||
ProxyConfig,
|
||||
build_listing_segments_for_makes,
|
||||
parse_listing_segments,
|
||||
)
|
||||
|
||||
|
||||
@@ -28,66 +23,6 @@ class TestDeepFindKey(unittest.TestCase):
|
||||
self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"])
|
||||
|
||||
|
||||
class TestParseListingSegments(unittest.TestCase):
|
||||
def test_empty_and_invalid_return_empty(self) -> None:
|
||||
self.assertEqual(parse_listing_segments(""), [])
|
||||
self.assertEqual(parse_listing_segments(" "), [])
|
||||
self.assertEqual(parse_listing_segments("invalid"), [])
|
||||
|
||||
def test_auto_segments_complete_coverage(self) -> None:
|
||||
"""auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
|
||||
segs = parse_listing_segments("auto")
|
||||
|
||||
# Точное число сегментов.
|
||||
expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(MOBILEDE_DEFAULT_MAKES) - len(_LARGE_MAKES))
|
||||
self.assertEqual(len(segs), expected)
|
||||
|
||||
# Все бренды из списка присутствуют.
|
||||
makes_in_segments = {s["make"] for s in segs}
|
||||
for make in MOBILEDE_DEFAULT_MAKES:
|
||||
self.assertIn(make, makes_in_segments)
|
||||
|
||||
# Крупные бренды разбиты на 3 сегмента с годами.
|
||||
toyota = [s for s in segs if s["make"] == "TOYOTA"]
|
||||
self.assertEqual(len(toyota), 3)
|
||||
for s in toyota:
|
||||
self.assertIsNotNone(s["year_min"])
|
||||
|
||||
# Мелкие бренды без годов.
|
||||
lexus = [s for s in segs if s["make"] == "LEXUS"]
|
||||
self.assertEqual(len(lexus), 1)
|
||||
self.assertIsNone(lexus[0]["year_min"])
|
||||
|
||||
# Годовые диапазоны покрывают 1950-2027.
|
||||
years = set()
|
||||
for yr_min, yr_max in _YEAR_SPLITS:
|
||||
years.update(range(yr_min, yr_max + 1))
|
||||
for year in range(1950, 2027):
|
||||
self.assertIn(year, years)
|
||||
|
||||
def test_json_input_formats(self) -> None:
|
||||
# Массив строк.
|
||||
segs = parse_listing_segments('["toyota", "ford"]')
|
||||
self.assertEqual(len(segs), 2)
|
||||
self.assertEqual(segs[0]["make"], "TOYOTA")
|
||||
|
||||
# Массив объектов с годами.
|
||||
segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]')
|
||||
self.assertEqual(segs[0]["make"], "BMW")
|
||||
self.assertEqual(segs[0]["year_min"], 2020)
|
||||
self.assertEqual(segs[0]["year_max"], 2025)
|
||||
|
||||
def test_build_listing_segments_for_makes(self) -> None:
|
||||
segs = build_listing_segments_for_makes(["toyota", "Lexus", "TOYOTA", " "])
|
||||
|
||||
toyota = [s for s in segs if s["make"] == "TOYOTA"]
|
||||
lexus = [s for s in segs if s["make"] == "LEXUS"]
|
||||
|
||||
self.assertEqual(len(toyota), len(_YEAR_SPLITS))
|
||||
self.assertEqual(len(lexus), 1)
|
||||
self.assertIsNone(lexus[0]["year_min"])
|
||||
|
||||
|
||||
class TestProxyConfig(unittest.TestCase):
|
||||
def test_requests_proxy_url_includes_encoded_credentials(self) -> None:
|
||||
cfg = ProxyConfig(
|
||||
|
||||
471
tests/test_worker_runtime_tasks.py
Normal file
471
tests/test_worker_runtime_tasks.py
Normal file
@@ -0,0 +1,471 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
from mobilede_scraper.worker import tasks
|
||||
|
||||
|
||||
class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
|
||||
class _FakeRedis:
|
||||
def __init__(self) -> None:
|
||||
self.store: dict[str, str] = {}
|
||||
self.sets: dict[str, set[str]] = {}
|
||||
|
||||
def get(self, key: str):
|
||||
return self.store.get(key)
|
||||
|
||||
def set(self, key: str, value: str, nx: bool = False, ex: int | None = None): # noqa: ARG002
|
||||
if nx and key in self.store:
|
||||
return False
|
||||
self.store[key] = str(value)
|
||||
return True
|
||||
|
||||
def incr(self, key: str):
|
||||
current = int(self.store.get(key, "0"))
|
||||
current += 1
|
||||
self.store[key] = str(current)
|
||||
return current
|
||||
|
||||
def sadd(self, key: str, member: str):
|
||||
bucket = self.sets.setdefault(key, set())
|
||||
before = len(bucket)
|
||||
bucket.add(member)
|
||||
return 1 if len(bucket) > before else 0
|
||||
|
||||
def expire(self, key: str, ttl: int): # noqa: ARG002
|
||||
return True
|
||||
|
||||
def test_lock_acquire_refresh_release(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
|
||||
redis_client.set.return_value = True
|
||||
self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
|
||||
redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
|
||||
|
||||
redis_client.eval.return_value = 1
|
||||
self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
|
||||
|
||||
redis_client.eval.reset_mock()
|
||||
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
|
||||
args = redis_client.eval.call_args[0]
|
||||
self.assertEqual(args[2], "lock:key")
|
||||
self.assertEqual(args[3], "owner-token")
|
||||
|
||||
def test_mobilede_segment_key_is_stable(self) -> None:
|
||||
segment = {"make_id": "BMW", "price_min": 1000, "price_max": 5000}
|
||||
|
||||
key1 = tasks._mobilede_task_segment_key(
|
||||
segment=segment,
|
||||
search_url=None,
|
||||
make_id=None,
|
||||
model_id=None,
|
||||
price_min=None,
|
||||
price_max=None,
|
||||
year_min=None,
|
||||
year_max=None,
|
||||
mileage_min=None,
|
||||
mileage_max=None,
|
||||
)
|
||||
key2 = tasks._mobilede_task_segment_key(
|
||||
segment={"price_max": 5000, "make_id": "BMW", "price_min": 1000},
|
||||
search_url=None,
|
||||
make_id=None,
|
||||
model_id=None,
|
||||
price_min=None,
|
||||
price_max=None,
|
||||
year_min=None,
|
||||
year_max=None,
|
||||
mileage_min=None,
|
||||
mileage_max=None,
|
||||
)
|
||||
|
||||
self.assertEqual(key1, key2)
|
||||
self.assertEqual(len(key1), 16)
|
||||
|
||||
def test_runtime_segment_reservation_uses_pending_cache(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
settings = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: {
|
||||
tasks.MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY: b'[{"make_id":"BMW"},{"make_id":"AUDI"}]',
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"2",
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"2",
|
||||
}.get(key)
|
||||
redis_client.incr.return_value = 2
|
||||
redis_client.ttl.return_value = -2
|
||||
|
||||
segment = tasks._reserve_mobilede_runtime_segment(
|
||||
redis_client,
|
||||
settings,
|
||||
only_new=False,
|
||||
)
|
||||
|
||||
self.assertEqual(segment, (1, {"make_id": "AUDI"}))
|
||||
|
||||
def test_preplan_splits_dense_segment_before_queueing(self) -> None:
|
||||
original_probe = tasks._mobilede_probe_segment_total
|
||||
original_children = tasks._mobilede_build_overflow_child_segments
|
||||
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
|
||||
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
|
||||
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
|
||||
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
|
||||
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
|
||||
|
||||
parent = {
|
||||
"label": "Cars",
|
||||
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
|
||||
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
}
|
||||
child_a = {**parent, "search_url": parent["search_url"] + "&ml=:100000", "mileage_max": "100000", "overflow_depth": 1}
|
||||
child_b = {**parent, "search_url": parent["search_url"] + "&ml=100001:", "mileage_min": "100001", "overflow_depth": 1}
|
||||
|
||||
try:
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 450
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 700
|
||||
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
|
||||
tasks._mobilede_probe_segment_total = lambda segment: 2000 if segment is parent or segment.get("overflow_depth") is None else 100
|
||||
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [child_a, child_b]
|
||||
|
||||
planned = tasks._mobilede_preplan_runtime_segments([parent])
|
||||
finally:
|
||||
tasks._mobilede_probe_segment_total = original_probe
|
||||
tasks._mobilede_build_overflow_child_segments = original_children
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
|
||||
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
|
||||
|
||||
self.assertEqual(len(planned), 2)
|
||||
self.assertTrue(all(item.get("total_results") == 100 for item in planned))
|
||||
self.assertTrue(all(int(item.get("max_pages") or 0) <= tasks.MOBILEDE_MAX_PAGE_NUMBER for item in planned))
|
||||
|
||||
def test_preplan_has_hard_segment_limit(self) -> None:
|
||||
original_probe = tasks._mobilede_probe_segment_total
|
||||
original_children = tasks._mobilede_build_overflow_child_segments
|
||||
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
|
||||
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
|
||||
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
|
||||
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
|
||||
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
|
||||
|
||||
parent = {
|
||||
"label": "Cars",
|
||||
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
|
||||
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
}
|
||||
|
||||
try:
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 3
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 10
|
||||
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
|
||||
tasks._mobilede_probe_segment_total = lambda segment: 5000
|
||||
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [
|
||||
{**segment, "search_url": str(segment["search_url"]) + "&a=1", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
|
||||
{**segment, "search_url": str(segment["search_url"]) + "&a=2", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
|
||||
]
|
||||
|
||||
planned = tasks._mobilede_preplan_runtime_segments([parent])
|
||||
finally:
|
||||
tasks._mobilede_probe_segment_total = original_probe
|
||||
tasks._mobilede_build_overflow_child_segments = original_children
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
|
||||
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
|
||||
|
||||
self.assertGreaterEqual(len(planned), 1)
|
||||
self.assertLessEqual(len(planned), 3)
|
||||
|
||||
def test_preplan_probe_budget_is_global(self) -> None:
|
||||
original_probe = tasks._mobilede_probe_segment_total
|
||||
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
|
||||
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
|
||||
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
|
||||
|
||||
calls = {"count": 0}
|
||||
segments = [
|
||||
{"label": f"Cars {index}", "search_url": f"https://suchen.mobile.de/fahrzeuge/search.html?x={index}"}
|
||||
for index in range(5)
|
||||
]
|
||||
|
||||
try:
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 50
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 2
|
||||
|
||||
def _probe(_segment):
|
||||
calls["count"] += 1
|
||||
return 100
|
||||
|
||||
tasks._mobilede_probe_segment_total = _probe
|
||||
|
||||
planned = tasks._mobilede_preplan_runtime_segments(segments)
|
||||
finally:
|
||||
tasks._mobilede_probe_segment_total = original_probe
|
||||
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
|
||||
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
|
||||
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
|
||||
|
||||
self.assertEqual(calls["count"], 2)
|
||||
self.assertEqual(len(planned), 5)
|
||||
|
||||
def test_overflow_root_mileage_split_keeps_high_mileage_tail_with_two_children(self) -> None:
|
||||
original_max_children = tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS
|
||||
try:
|
||||
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = 2
|
||||
children = tasks._mobilede_build_overflow_child_segments(
|
||||
segment={
|
||||
"label": "Cars",
|
||||
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=5001:10000&fr=2005:2009",
|
||||
"price_min": "5001",
|
||||
"price_max": "10000",
|
||||
"year_min": "2005",
|
||||
"year_max": "2009",
|
||||
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
},
|
||||
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
)
|
||||
finally:
|
||||
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = original_max_children
|
||||
|
||||
self.assertEqual(len(children), 2)
|
||||
self.assertEqual(children[0].get("mileage_max"), "150000")
|
||||
self.assertEqual(children[1].get("mileage_min"), "150001")
|
||||
self.assertIsNone(children[1].get("mileage_max"))
|
||||
|
||||
def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: {
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10",
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10",
|
||||
tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None,
|
||||
tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000",
|
||||
tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995",
|
||||
tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990",
|
||||
tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5",
|
||||
tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000",
|
||||
}.get(key)
|
||||
redis_client.scard.return_value = 1
|
||||
|
||||
finalized = tasks._mobilede_try_finalize_bootstrap(redis_client)
|
||||
|
||||
self.assertFalse(finalized)
|
||||
redis_client.set.assert_not_called()
|
||||
|
||||
def test_bootstrap_finalization_marks_done_when_dispatched_is_empty(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: {
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY: b"10",
|
||||
tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY: b"10",
|
||||
tasks.MOBILEDE_BOOTSTRAP_DONE_KEY: None,
|
||||
tasks.MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY: b"1000",
|
||||
tasks.MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY: b"995",
|
||||
tasks.MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY: b"990",
|
||||
tasks.MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY: b"5",
|
||||
tasks.MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY: b"7000",
|
||||
}.get(key)
|
||||
redis_client.scard.return_value = 0
|
||||
|
||||
finalized = tasks._mobilede_try_finalize_bootstrap(redis_client)
|
||||
|
||||
self.assertTrue(finalized)
|
||||
redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
|
||||
redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
|
||||
|
||||
def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None
|
||||
|
||||
should_skip = tasks._mobilede_should_skip_completed_bootstrap_segment(
|
||||
redis_client,
|
||||
segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
|
||||
only_new=False,
|
||||
bootstrap_run_active=True,
|
||||
)
|
||||
|
||||
self.assertTrue(should_skip)
|
||||
|
||||
def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None:
|
||||
exhausted = tasks._mobilede_segment_window_is_exhausted(
|
||||
{"make_id": "3500", "max_pages": 50},
|
||||
start_page=51,
|
||||
max_pages=50,
|
||||
use_cursor=False,
|
||||
)
|
||||
not_exhausted = tasks._mobilede_segment_window_is_exhausted(
|
||||
{"make_id": "3500", "max_pages": 50},
|
||||
start_page=1,
|
||||
max_pages=50,
|
||||
use_cursor=False,
|
||||
)
|
||||
cursor_window = tasks._mobilede_segment_window_is_exhausted(
|
||||
{"make_id": "3500", "max_pages": 50},
|
||||
start_page=51,
|
||||
max_pages=50,
|
||||
use_cursor=True,
|
||||
)
|
||||
|
||||
self.assertTrue(exhausted)
|
||||
self.assertFalse(not_exhausted)
|
||||
self.assertFalse(cursor_window)
|
||||
|
||||
def test_explicit_start_page_is_not_overwritten_by_segment_default(self) -> None:
|
||||
segment = {"make_id": "3500", "start_page": 1}
|
||||
|
||||
requested_start_page = 51
|
||||
if segment.get("start_page") is not None and requested_start_page <= 1:
|
||||
effective_start_page = int(segment.get("start_page") or requested_start_page)
|
||||
else:
|
||||
effective_start_page = requested_start_page
|
||||
|
||||
self.assertEqual(effective_start_page, 51)
|
||||
|
||||
def test_refresh_cycle_tracking_finalizes_on_last_segment(self) -> None:
|
||||
redis_client = self._FakeRedis()
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-a"
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2"
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "1"
|
||||
|
||||
done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment(
|
||||
redis_client,
|
||||
cycle_id="cycle-a",
|
||||
segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
|
||||
total_segments_hint=2,
|
||||
)
|
||||
|
||||
self.assertEqual(done, 2)
|
||||
self.assertEqual(total, 2)
|
||||
self.assertTrue(should_finalize)
|
||||
|
||||
def test_refresh_cycle_tracking_ignores_stale_cycle_id(self) -> None:
|
||||
redis_client = self._FakeRedis()
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-active"
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "2"
|
||||
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "0"
|
||||
|
||||
done, total, should_finalize = tasks._mobilede_track_refresh_cycle_segment(
|
||||
redis_client,
|
||||
cycle_id="cycle-old",
|
||||
segment={"make_id": "3500", "price_min": "1", "price_max": "5000"},
|
||||
total_segments_hint=2,
|
||||
)
|
||||
|
||||
self.assertEqual(done, 0)
|
||||
self.assertEqual(total, 2)
|
||||
self.assertFalse(should_finalize)
|
||||
|
||||
def test_pre_split_mileage_requires_explicit_flag(self) -> None:
|
||||
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
|
||||
try:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
|
||||
self.assertFalse(
|
||||
tasks._mobilede_should_pre_split_mileage(
|
||||
price_min=30001,
|
||||
price_max=50000,
|
||||
year_min=2010,
|
||||
year_max=2014,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
|
||||
|
||||
def test_pre_split_mileage_respects_explicit_override(self) -> None:
|
||||
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
|
||||
try:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
|
||||
self.assertTrue(
|
||||
tasks._mobilede_should_pre_split_mileage(
|
||||
price_min=30001,
|
||||
price_max=50000,
|
||||
year_min=2010,
|
||||
year_max=2017,
|
||||
)
|
||||
)
|
||||
finally:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
|
||||
|
||||
def test_filtered_multi_make_url_expands_to_make_price_year_segments(self) -> None:
|
||||
search_url = (
|
||||
"https://www.mobile.de/ru/????????????????????????-????????????????/??????????.html"
|
||||
"?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&od=up&sb=rel&ref=dsp"
|
||||
)
|
||||
|
||||
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
|
||||
original_dynamic = tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES
|
||||
try:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
|
||||
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = False
|
||||
segments = tasks._expand_mobilede_search_url_segment(
|
||||
{
|
||||
"label": "Cars",
|
||||
"search_url": search_url,
|
||||
"start_page": 1,
|
||||
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
|
||||
}
|
||||
)
|
||||
finally:
|
||||
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
|
||||
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = original_dynamic
|
||||
|
||||
make_counts: dict[str, int] = {}
|
||||
for segment in segments:
|
||||
make_id = str(segment.get("make_id") or "")
|
||||
make_counts[make_id] = make_counts.get(make_id, 0) + 1
|
||||
self.assertIsNone(segment.get("mileage_min"))
|
||||
self.assertIsNone(segment.get("mileage_max"))
|
||||
|
||||
self.assertEqual(make_counts, {"3500": 59, "11000": 59})
|
||||
self.assertEqual(len(segments), 118)
|
||||
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all(str(segment.get("max_pages")) == str(tasks.MOBILEDE_MAX_PAGE_NUMBER) for segment in segments))
|
||||
|
||||
|
||||
def test_build_runtime_segments_expands_filtered_search_urls_by_default(self) -> None:
|
||||
settings = MagicMock()
|
||||
settings.listing.filtered_search_urls = [
|
||||
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
|
||||
]
|
||||
|
||||
segments = tasks._build_mobilede_runtime_segments(settings)
|
||||
|
||||
self.assertEqual(len(segments), 118)
|
||||
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
|
||||
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
|
||||
|
||||
def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None:
|
||||
import os
|
||||
|
||||
settings = MagicMock()
|
||||
settings.listing.filtered_search_urls = [
|
||||
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
|
||||
]
|
||||
|
||||
previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
|
||||
try:
|
||||
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "true"
|
||||
segments = tasks._build_mobilede_runtime_segments(settings)
|
||||
finally:
|
||||
if previous is None:
|
||||
os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
|
||||
else:
|
||||
os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = previous
|
||||
|
||||
self.assertEqual(len(segments), 2)
|
||||
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
|
||||
self.assertTrue(all("p=" not in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments))
|
||||
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,615 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
from dataclasses import replace
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from mobilede_scraper.worker import tasks
|
||||
from mobilede_scraper.core.config import Settings, settings as base_settings
|
||||
|
||||
|
||||
class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
def test_build_listing_segments_from_runtime_config_brands(self) -> None:
|
||||
with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as fh:
|
||||
json.dump({"filters": {"brands": ["Toyota", "Lexus", "Toyota"]}}, fh)
|
||||
runtime_config_file = fh.name
|
||||
|
||||
settings = Settings(runtime_config_file=runtime_config_file)
|
||||
settings.listing.listing_segments_json = "runtime"
|
||||
|
||||
segs = tasks._build_listing_segments(settings)
|
||||
|
||||
toyota = [s for s in segs if s["make"] == "TOYOTA"]
|
||||
lexus = [s for s in segs if s["make"] == "LEXUS"]
|
||||
self.assertEqual(len(toyota), 3)
|
||||
self.assertEqual(len(lexus), 1)
|
||||
self.assertIsNone(lexus[0]["year_min"])
|
||||
os.unlink(runtime_config_file)
|
||||
|
||||
def test_lock_acquire_refresh_release(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
|
||||
# Acquire.
|
||||
redis_client.set.return_value = True
|
||||
self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
|
||||
redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
|
||||
|
||||
# Refresh.
|
||||
redis_client.eval.return_value = 1
|
||||
self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
|
||||
|
||||
# Release.
|
||||
redis_client.eval.reset_mock()
|
||||
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
|
||||
args = redis_client.eval.call_args[0]
|
||||
self.assertEqual(args[2], "lock:key")
|
||||
self.assertEqual(args[3], "owner-token")
|
||||
|
||||
def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=False):
|
||||
persistence = MagicMock()
|
||||
get_persistence.return_value = persistence
|
||||
get_redis.return_value = MagicMock()
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-123")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
persistence.create_tables.assert_called_once()
|
||||
self.assertEqual(result["status"], "skipped")
|
||||
self.assertEqual(result["reason"], "sync_already_running")
|
||||
|
||||
def test_sync_listing_task_releases_owned_lock(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 7,
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 4,
|
||||
"skipped_existing": 1,
|
||||
"elapsed_seconds": 1.25,
|
||||
}):
|
||||
persistence = MagicMock()
|
||||
get_persistence.return_value = persistence
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
stop_event = MagicMock()
|
||||
heartbeat_thread = MagicMock()
|
||||
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-123")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(make="Toyota")
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
stop_event.set.assert_called_once()
|
||||
heartbeat_thread.join.assert_called_once()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_clear_orphan_sync_listing_lock(self) -> None:
|
||||
# Нет запущенных задач → удаляет.
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = "owner-token"
|
||||
redis_client.ttl.return_value = 120
|
||||
celery_app = MagicMock()
|
||||
inspector = MagicMock()
|
||||
inspector.active.return_value = {"worker@node": []}
|
||||
inspector.reserved.return_value = {"worker@node": []}
|
||||
inspector.scheduled.return_value = {"worker@node": []}
|
||||
celery_app.control.inspect.return_value = inspector
|
||||
|
||||
self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
|
||||
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
|
||||
|
||||
# Задача активна → не удаляет.
|
||||
redis_client2 = MagicMock()
|
||||
redis_client2.get.return_value = "owner-token"
|
||||
inspector2 = MagicMock()
|
||||
inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
|
||||
inspector2.reserved.return_value = {"worker@node": []}
|
||||
inspector2.scheduled.return_value = {"worker@node": []}
|
||||
celery_app2 = MagicMock()
|
||||
celery_app2.control.inspect.return_value = inspector2
|
||||
|
||||
self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
|
||||
redis_client2.delete.assert_not_called()
|
||||
|
||||
def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \
|
||||
patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 9,
|
||||
"cars_upserted": 3,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 5,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 2.0,
|
||||
}):
|
||||
persistence = MagicMock()
|
||||
get_persistence.return_value = persistence
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
stop_event = MagicMock()
|
||||
heartbeat_thread = MagicMock()
|
||||
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-456")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(make="Honda")
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
clear_orphan.assert_called_once()
|
||||
self.assertEqual(acquire_lock.call_count, 2)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_checkpoint_save_load_roundtrip(self) -> None:
|
||||
storage: dict[str, str] = {}
|
||||
|
||||
def _fake_set(key, value, ex=None):
|
||||
storage[key] = value
|
||||
return True
|
||||
|
||||
redis_client = MagicMock()
|
||||
redis_client.set.side_effect = _fake_set
|
||||
redis_client.get.side_effect = lambda key: storage.get(key)
|
||||
|
||||
tasks._save_last_completed_segment(redis_client, 12)
|
||||
|
||||
self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
|
||||
self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
|
||||
|
||||
def test_load_checkpoint_clears_invalid_payload(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = "{not-a-number"
|
||||
|
||||
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
|
||||
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
|
||||
|
||||
def test_load_checkpoint_empty_when_missing(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
|
||||
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
|
||||
redis_client.delete.assert_not_called()
|
||||
|
||||
def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
|
||||
segments = [
|
||||
{"make": "ACURA"},
|
||||
{"make": "AUDI"},
|
||||
{"make": "BMW"},
|
||||
{"make": "EAGLE"},
|
||||
]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
"1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
)
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
"run_id": 11, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-resume-seg")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
# last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
|
||||
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
# Оставшийся чекпоинт не должен использоваться.
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
"0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
)
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
"run_id": 14, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-792")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
|
||||
self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
|
||||
clear_checkpoint.assert_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
|
||||
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
"99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
)
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
"run_id": 15, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-beyond")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 13,
|
||||
"status": "partial_success",
|
||||
"full_scan_completed": True,
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 1,
|
||||
"images_upserted": 3,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 4.0,
|
||||
"failures": [{"vehicle_url": "v", "error": "e"}],
|
||||
}), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-791")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(make="Toyota")
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "partial_success")
|
||||
# Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
|
||||
# до браузерного job + после. Главное — вызов произошёл.
|
||||
clear_checkpoint.assert_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_try_set_followup_pending_deduplicates(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.set.side_effect = [True, False]
|
||||
|
||||
self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
|
||||
self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
|
||||
|
||||
def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
|
||||
|
||||
streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
|
||||
redis_client,
|
||||
reason="max_retries_exceeded",
|
||||
)
|
||||
|
||||
self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
|
||||
self.assertFalse(should_enqueue)
|
||||
redis_client.expire.assert_called_once_with(
|
||||
tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
|
||||
tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
|
||||
)
|
||||
|
||||
def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.incr.return_value = 1
|
||||
|
||||
streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
|
||||
redis_client,
|
||||
reason="bootstrap_not_completed",
|
||||
)
|
||||
|
||||
self.assertEqual(streak, 1)
|
||||
self.assertTrue(should_enqueue)
|
||||
|
||||
def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
|
||||
patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
|
||||
patch.object(tasks, "_clear_followup_pending") as clear_pending, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 99,
|
||||
"status": "failed",
|
||||
"full_scan_completed": False,
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [{"vehicle_url": "listing", "error": "bad resume"}],
|
||||
"listing": {"vehicles_collected": 0},
|
||||
}):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
|
||||
tasks.sync_listing_task.push_request(id="task-900")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "failed")
|
||||
bump_streak.assert_called_once()
|
||||
clear_pending.assert_called()
|
||||
task_app.send_task.assert_not_called()
|
||||
|
||||
def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
|
||||
patch.object(tasks, "_release_lock_if_owner"), \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \
|
||||
patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
|
||||
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
|
||||
tasks.sync_listing_task.push_request(id="task-soft-timeout")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(make="Toyota")
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "timed_out")
|
||||
set_pending.assert_called_once()
|
||||
task_app.send_task.assert_not_called()
|
||||
|
||||
def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
|
||||
patch.object(tasks, "_release_lock_if_owner"), \
|
||||
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
|
||||
patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \
|
||||
patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 101,
|
||||
"status": "partial_success",
|
||||
"full_scan_completed": False,
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 1,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [],
|
||||
"listing": {"vehicles_collected": 2},
|
||||
}):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
|
||||
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
|
||||
tasks.sync_listing_task.push_request(id="task-breaker-stop")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "partial_success")
|
||||
# Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly.
|
||||
self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list))
|
||||
self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list))
|
||||
clear_checkpoint.assert_called_once()
|
||||
task_app.send_task.assert_not_called()
|
||||
|
||||
def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(
|
||||
tasks,
|
||||
"Settings",
|
||||
return_value=replace(
|
||||
base_settings,
|
||||
discovery=replace(base_settings.discovery, always_full_scan=True),
|
||||
),
|
||||
), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job") as run_job, \
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=[]):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
run_job.return_value = {
|
||||
"run_id": 17,
|
||||
"status": "success",
|
||||
"full_scan_completed": True,
|
||||
"cars_upserted": 1,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [],
|
||||
}
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-always-full")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
set_full_scan_done.assert_called_with(redis_client, False)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None:
|
||||
segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(
|
||||
tasks,
|
||||
"Settings",
|
||||
return_value=replace(
|
||||
base_settings,
|
||||
discovery=replace(base_settings.discovery, always_full_scan=True),
|
||||
),
|
||||
), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("mobilede_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
"0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
)
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
"run_id": 18,
|
||||
"status": "success",
|
||||
"full_scan_completed": True,
|
||||
"cars_upserted": 1,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "MobiledeScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-always-full-resume")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1)
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user