qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update scraper package

Browse Source
This commit is contained in:
qananasikq
2026-04-24 20:46:58 +03:00
parent 6aba4f0dbd
commit d484088e18
59 changed files with 2548 additions and 1272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
tests/test_db.py
View File

@@ -6,10 +6,10 @@ from pathlib import Path
from sqlalchemy import select
from iaai_scraper.core.config import Settings
from iaai_scraper.storage.db import PersistenceService
from iaai_scraper.storage.models import Car, Image, SyncRun
from iaai_scraper.storage.schemas import CarRecord, ImageRecord
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.storage.db import PersistenceService
from dubizzle_scraper.storage.models import Car, Image, SyncRun
from dubizzle_scraper.storage.schemas import CarRecord, ImageRecord
class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
@staticmethod
def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
return CarRecord(
parser_id=f"iaai:{origin_id}",
parser_id=f"dubizzle:{origin_id}",
brand="Toyota",
model="Camry",
year=2014,
price=price,
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id,
slug=f"toyota-camry-{origin_id}",
images=[
ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
fullres_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
preview_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=400&height=300",
order_index=0,
)
],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
second = self._record("888")
second.images = [
ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
fullres_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=845&height=633",
preview_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
order_index=0,
)
]
@@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
first = self._record("OLD-ID")
first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
first.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
self.persistence.upsert_car(first)
second = self._record("NEW-ID")
second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
second.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
result = self.persistence.upsert_car(second)
self.assertEqual(result["action"], "updated")

10
tests/test_listing.py
View File

@@ -2,9 +2,9 @@ from __future__ import annotations
import unittest
from iaai_scraper.browser.pace import HumanPacer
from iaai_scraper.core.config import Settings
from iaai_scraper.browser.listing import ListingCollector
from dubizzle_scraper.browser.pace import HumanPacer
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.browser.listing import ListingCollector
class _FakePage:
@@ -58,8 +58,8 @@ class TestListingUnit(unittest.TestCase):
self.assertEqual(
links,
[
("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
("https://www.dubizzle.com/VehicleDetail/45184893~US", "45184893"),
("https://www.dubizzle.com/VehicleDetail/45171480~US", "45171480"),
],
)

107
tests/test_mappers.py
View File

@@ -2,22 +2,109 @@ from __future__ import annotations
import unittest
from iaai_scraper.parsing.mapper import CarMapper
from dubizzle_scraper.parsing.mapper import CarMapper
class TestCarMapper(unittest.TestCase):
def setUp(self) -> None:
self.mapper = CarMapper()
def test_extracts_make_model_from_algolia_details(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://dubizzle.com/s/DOBI7J3",
vehicle_summary={
"id": 16810399,
"details_v2": {
"tertiary": [
{"slug": "make", "value": {"en": "Toyota", "ar": "تويوتا"}},
{"slug": "model", "value": {"en": "Land Cruiser", "ar": "لاند كروزر"}},
]
},
"category_v2": {
"names_en": ["Motors", "Used Cars", "Toyota", "Land Cruiser"],
"slug_paths": [
"motors",
"motors/used-cars",
"motors/used-cars/toyota",
"motors/used-cars/toyota/land-cruiser",
],
},
},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.brand, "Toyota")
self.assertEqual(record.model, "Land Cruiser")
def test_extracts_make_model_from_legacy_details(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://dubizzle.com/s/DOBIlegacy",
vehicle_summary={
"objectID": "item:legacy:1",
"details": {
"Make": {"en": {"label": "Make", "value": "Honda"}},
"Model": {"en": {"label": "Model", "value": "Civic"}},
},
},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.brand, "Honda")
self.assertEqual(record.model, "Civic")
def test_extracts_extended_algolia_fields(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://dubizzle.com/s/DOBGaGs",
vehicle_summary={
"id": 16345114,
"year": 2012,
"price": 59000.0,
"kilometers": 91800,
"seller_type": "DL",
"site": {"en": "Dubai"},
"category_v2": {
"names_en": ["Motors", "Used Cars", "Porsche", "Cayenne"],
"slug_paths": [
"motors",
"motors/used-cars",
"motors/used-cars/porsche",
"motors/used-cars/porsche/cayenne",
],
},
"details": {
"Body Type": {"en": {"value": "SUV"}},
"Transmission Type": {"en": {"value": "Automatic Transmission"}},
"Exterior Color": {"en": {"value": "Brown"}},
"Steering Side": {"en": {"value": "Left Hand"}},
"Engine Capacity (cc)": {"en": {"value": "4800 cc"}},
"Make": {"en": {"value": "Porsche"}},
"Model": {"en": {"value": "Cayenne"}},
},
"photo_mains": ["https://dbz-images.dubizzle.com/images/example.jpeg?impolicy=dpv"],
},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.brand, "Porsche")
self.assertEqual(record.model, "Cayenne")
self.assertEqual(record.price, 59000)
self.assertEqual(record.mileage, 91800)
self.assertEqual(record.color, "brown")
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.gearbox, "AT")
self.assertEqual(record.steering_wheel, "LEFT")
self.assertEqual(record.engine_volume, 4800)
self.assertEqual(record.country, "AE")
def test_deduplicates_images_by_image_key(self) -> None:
urls = [
"https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
"https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
"https://vis.dubizzle.com/resizer?imageKeys=1&width=200&height=150",
"https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
"https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
]
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
@@ -28,7 +115,7 @@ class TestCarMapper(unittest.TestCase):
def test_no_damage_marker_is_not_damaged(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
vehicle_summary={"make": "Ford", "model": "Focus"},
payload_insights={
"vehicle_core": {},
@@ -43,7 +130,7 @@ class TestCarMapper(unittest.TestCase):
def test_unknown_empty_values_and_normalization(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/999~US",
vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
@@ -54,7 +141,7 @@ class TestCarMapper(unittest.TestCase):
# Нормализация регистра и пробелов.
record2 = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/888~US",
vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
@@ -63,7 +150,7 @@ class TestCarMapper(unittest.TestCase):
def test_price_and_currency_parsing(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/777~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={
"vehicle_core": {},
@@ -76,7 +163,7 @@ class TestCarMapper(unittest.TestCase):
self.assertEqual(record.price, 5200)
record2 = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/778~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={
"vehicle_core": {},

10
tests/test_parser.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
import unittest
from iaai_scraper.parsing.parser import VehicleParser
from dubizzle_scraper.parsing.parser import VehicleParser
class TestVehicleParserUnit(unittest.TestCase):
@@ -29,11 +29,11 @@ class TestVehicleParserUnit(unittest.TestCase):
self.assertEqual(parsed["model"], "CAMRY")
def test_extract_image_urls_filters_and_deduplicates(self) -> None:
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
vehicle_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
payloads = [{"imageUrls": [
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
"https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.dubizzle.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
]}]
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
self.assertEqual(len(urls), 1)

14
tests/test_resilience.py
View File

@@ -4,17 +4,17 @@ import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.core.config import Settings
from iaai_scraper.worker import tasks
from dubizzle_scraper.scraper import DUBIZZLEScraper
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.worker import tasks
class TestResilience(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper:
def _make_scraper(self) -> DUBIZZLEScraper:
s = Settings()
s.log_level = "CRITICAL"
s.database.url = "sqlite://"
return IAAIScraper(s)
return DUBIZZLEScraper(s)
def test_update_task_progress_updates_global_marker(self) -> None:
redis_client = MagicMock()
@@ -60,11 +60,11 @@ class TestResilience(unittest.TestCase):
result = scraper._sync_listing_streaming(
make=None,
model=None,
lane="iaai_cars",
lane="dubizzle_cars",
limit=None,
effective_only_new=False,
started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST",
)
self.assertEqual(result["total"], 0)

76
tests/test_scraper.py
View File

@@ -5,29 +5,29 @@ from concurrent.futures import TimeoutError as FuturesTimeoutError
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from iaai_scraper.core.config import Settings
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.storage.schemas import CarRecord
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from dubizzle_scraper.scraper import DUBIZZLEScraper
from dubizzle_scraper.storage.schemas import CarRecord
def make_db_record(origin_id: str) -> dict[str, object]:
return CarRecord(
parser_id=f"iaai:{origin_id}",
parser_id=f"dubizzle:{origin_id}",
brand="Toyota",
model="Camry",
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id,
slug=f"toyota-camry-{origin_id}",
).model_dump(mode="json")
class TestScraperSync(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper:
def _make_scraper(self) -> DUBIZZLEScraper:
s = Settings()
s.log_level = "CRITICAL"
s.database.url = "sqlite://"
return IAAIScraper(s)
return DUBIZZLEScraper(s)
def test_sync_vehicle_uses_db_record(self) -> None:
scraper = self._make_scraper()
@@ -37,7 +37,7 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
result = scraper.sync_vehicle("https://www.dubizzle.com/VehicleDetail/111~US")
self.assertEqual(result["status"], "success")
self.assertIn("trace_id", result)
@@ -50,13 +50,13 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.start_sync_run = MagicMock(return_value=2)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
{"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
{"https://www.dubizzle.com/VehicleDetail/111~US"}, {"dubizzle:222"},
))
scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [
"https://www.iaai.com/VehicleDetail/111~US",
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
"https://www.dubizzle.com/VehicleDetail/111~US",
"https://www.dubizzle.com/VehicleDetail/222~US",
"https://www.dubizzle.com/VehicleDetail/333~US",
]
})
scraper.sync_batch = MagicMock(return_value={
@@ -79,7 +79,7 @@ class TestScraperSync(unittest.TestCase):
})
result2 = scraper2.sync_listing(
only_new=True,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
year_min=2020, year_max=2027,
)
self.assertEqual(result2["cars_upserted"], 10)
@@ -155,35 +155,35 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["status"], "partial_success")
def test_build_segment_listing_url(self) -> None:
base = "https://www.iaai.com/Vehiclelisting/Cars"
base = "https://www.dubizzle.com/Vehiclelisting/Cars"
self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
DUBIZZLEScraper._build_segment_listing_url(base, "TOYOTA"),
"https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
)
self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
DUBIZZLEScraper._build_segment_listing_url(base, "LAND ROVER"),
"https://www.dubizzle.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
)
self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, None), base)
self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, ""), base)
def test_guard_and_protection_detection(self) -> None:
with self.assertRaises(AntiBotDetectedError):
IAAIScraper._raise_if_blocked_or_incomplete(
DUBIZZLEScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
"access_notes": {}, "vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US",
"https://www.dubizzle.com/VehicleDetail/999~US",
)
with self.assertRaises(SiteStructureChangedError):
IAAIScraper._raise_if_blocked_or_incomplete(
DUBIZZLEScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
"access_notes": {"possible_captcha": False, "possible_antibot": False},
"vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US",
"https://www.dubizzle.com/VehicleDetail/999~US",
)
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
self.assertFalse(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
def test_close_resets_browser_state(self) -> None:
scraper = self._make_scraper()
@@ -201,7 +201,7 @@ class TestScraperSync(unittest.TestCase):
scraper = self._make_scraper()
page = MagicMock()
page_result = SimpleNamespace(
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/123~US")],
)
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper.listing_collector.open_cars_listing = MagicMock()
@@ -228,7 +228,7 @@ class TestScraperSync(unittest.TestCase):
page = MagicMock()
page_result = SimpleNamespace(
page_number=1,
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/999~US", lot_number="999")],
next_page_detected=False,
)
@@ -245,7 +245,7 @@ class TestScraperSync(unittest.TestCase):
"year_max": None,
})
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
scraper._extract_page_urls = MagicMock(return_value=["https://www.dubizzle.com/VehicleDetail/999~US"])
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 1,
"cars_failed": 0,
@@ -256,11 +256,11 @@ class TestScraperSync(unittest.TestCase):
result = scraper._sync_listing_streaming(
make=None,
model=None,
lane="iaai_cars",
lane="dubizzle_cars",
limit=None,
effective_only_new=False,
started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=EAGLE",
)
scraper.listing_collector.open_cars_listing.assert_called_once()
@@ -278,9 +278,9 @@ class TestScraperSync(unittest.TestCase):
})
urls = [
"https://www.iaai.com/VehicleDetail/111~US",
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
"https://www.dubizzle.com/VehicleDetail/111~US",
"https://www.dubizzle.com/VehicleDetail/222~US",
"https://www.dubizzle.com/VehicleDetail/333~US",
]
first_record = CarRecord.model_validate(make_db_record("111"))
@@ -298,8 +298,8 @@ class TestScraperSync(unittest.TestCase):
yield 0, first_record
raise FuturesTimeoutError()
with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
with patch("dubizzle_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
patch("dubizzle_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
patch.object(scraper, "_browser_fallback_parallel", return_value={
"records": [],
"failures": [],

30
tests/test_self_heal.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
import unittest
from unittest.mock import MagicMock, patch
from iaai_scraper.worker import self_heal
from dubizzle_scraper.worker import self_heal
class TestSelfHeal(unittest.TestCase):
@@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase):
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: {
self_heal.GLOBAL_PROGRESS_TS_KEY: None,
"iaai:state:task_progress:a": '{"ts": 100}',
"iaai:state:task_progress:b": '{"ts": 250}',
"iaai:state:task_progress:c": '{"ts": 150}',
"dubizzle:state:task_progress:a": '{"ts": 100}',
"dubizzle:state:task_progress:b": '{"ts": 250}',
"dubizzle:state:task_progress:c": '{"ts": 150}',
}.get(key)
redis_client.scan_iter.return_value = [
"iaai:state:task_progress:a",
"iaai:state:task_progress:b",
"iaai:state:task_progress:c",
"dubizzle:state:task_progress:a",
"dubizzle:state:task_progress:b",
"dubizzle:state:task_progress:c",
]
ts = self_heal._read_last_progress_ts(redis_client)
@@ -38,20 +38,20 @@ class TestSelfHeal(unittest.TestCase):
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: {
self_heal.GLOBAL_PROGRESS_TS_KEY: None,
"iaai:state:task_progress:a": "{bad-json}",
"iaai:state:task_progress:b": '{"foo": "bar"}',
"dubizzle:state:task_progress:a": "{bad-json}",
"dubizzle:state:task_progress:b": '{"foo": "bar"}',
}.get(key)
redis_client.scan_iter.return_value = [
"iaai:state:task_progress:a",
"iaai:state:task_progress:b",
"dubizzle:state:task_progress:a",
"dubizzle:state:task_progress:b",
]
ts = self_heal._read_last_progress_ts(redis_client)
self.assertIsNone(ts)
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("iaai_scraper.worker.self_heal.os.kill")
@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("dubizzle_scraper.worker.self_heal.os.kill")
@patch("builtins.open")
def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
open_mock.return_value.__enter__.return_value.read.return_value = "123"
@@ -64,8 +64,8 @@ class TestSelfHeal(unittest.TestCase):
self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("iaai_scraper.worker.self_heal.os.kill")
@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("dubizzle_scraper.worker.self_heal.os.kill")
@patch("builtins.open")
def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
open_mock.return_value.__enter__.return_value.read.return_value = "123"

33
tests/test_utils.py
View File

@@ -2,8 +2,8 @@ from __future__ import annotations
import unittest
from iaai_scraper.core.utils import deep_find_key
from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
from dubizzle_scraper.core.utils import deep_find_key
from dubizzle_scraper.core.config import parse_listing_segments, _AUTO_YEAR_SPLITS
class TestDeepFindKey(unittest.TestCase):
@@ -28,32 +28,19 @@ class TestParseListingSegments(unittest.TestCase):
self.assertEqual(parse_listing_segments("invalid"), [])
def test_auto_segments_complete_coverage(self) -> None:
"""auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
"""auto: все годовые диапазоны покрыты, без дыр и без make-фильтра."""
segs = parse_listing_segments("auto")
# Точное число сегментов.
expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
self.assertEqual(len(segs), expected)
# Все бренды из списка присутствуют.
makes_in_segments = {s["make"] for s in segs}
for make in IAAI_DEFAULT_MAKES:
self.assertIn(make, makes_in_segments)
# Крупные бренды разбиты на 3 сегмента с годами.
toyota = [s for s in segs if s["make"] == "TOYOTA"]
self.assertEqual(len(toyota), 3)
for s in toyota:
self.assertIsNotNone(s["year_min"])
# Мелкие бренды без годов.
lexus = [s for s in segs if s["make"] == "LEXUS"]
self.assertEqual(len(lexus), 1)
self.assertIsNone(lexus[0]["year_min"])
self.assertEqual(len(segs), len(_AUTO_YEAR_SPLITS))
self.assertEqual(
[(s["year_min"], s["year_max"]) for s in segs],
list(_AUTO_YEAR_SPLITS),
)
self.assertTrue(all(s["make"] is None for s in segs))
# Годовые диапазоны покрывают 1950-2027.
years = set()
for yr_min, yr_max in _YEAR_SPLITS:
for yr_min, yr_max in _AUTO_YEAR_SPLITS:
years.update(range(yr_min, yr_max + 1))
for year in range(1950, 2027):
self.assertIn(year, years)

151
tests/test_worker_tasks.py
View File

@@ -5,8 +5,8 @@ from dataclasses import replace
import unittest
from unittest.mock import MagicMock, patch
from iaai_scraper.worker import tasks
from iaai_scraper.core.config import settings as base_settings
from dubizzle_scraper.worker import tasks
from dubizzle_scraper.core.config import settings as base_settings
class TestWorkerTaskLockHelpers(unittest.TestCase):
@@ -187,13 +187,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(
base_settings.discovery,
always_full_scan=False,
mode="listing",
),
celery=replace(
base_settings.celery,
parallel_segments=False,
),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
@@ -211,7 +227,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-resume-seg")
try:
result = tasks.sync_listing_task.run()
@@ -228,6 +244,22 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(
base_settings.discovery,
always_full_scan=False,
mode="listing",
),
celery=replace(
base_settings.celery,
parallel_segments=False,
),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -235,7 +267,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
# Оставшийся чекпоинт не должен использоваться.
@@ -254,10 +286,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-792")
try:
result = tasks.sync_listing_task.run()
result = tasks.sync_listing_task.run(limit=1)
finally:
tasks.sync_listing_task.pop_request()
@@ -271,13 +303,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(
base_settings.discovery,
always_full_scan=False,
mode="listing",
),
celery=replace(
base_settings.celery,
parallel_segments=False,
),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
@@ -295,7 +343,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-beyond")
try:
result = tasks.sync_listing_task.run()
@@ -507,7 +555,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch.object(tasks, "_run_browser_job") as run_job, \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.return_value = None
@@ -554,7 +602,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
@@ -579,7 +627,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-always-full-resume")
try:
result = tasks.sync_listing_task.run()
@@ -591,6 +639,85 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
release_lock.assert_called_once()
def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None:
segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(base_settings.discovery, always_full_scan=True),
celery=replace(base_settings.celery, parallel_segments=True),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_reset_segments_progress") as reset_progress, \
patch.object(tasks, "_set_segmented_scan_active") as set_active, \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.smembers.return_value = {"0", "1"}
redis_client.get.return_value = None
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
tasks.sync_listing_task.push_request(id="task-always-hourly")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(result["segments_dispatched"], 2)
reset_progress.assert_called_once_with(redis_client, len(segments))
set_active.assert_called_once_with(redis_client, total=len(segments))
self.assertEqual(task_app.send_task.call_count, 2)
release_lock.assert_called_once()
def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None:
segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(base_settings.discovery, always_full_scan=True),
celery=replace(base_settings.celery, parallel_segments=True),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
get_redis.return_value = MagicMock()
start_heartbeat.return_value = (MagicMock(), MagicMock())
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
tasks.sync_listing_task.push_request(id="task-always-skip-duplicate")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "skipped")
self.assertEqual(result["reason"], "segmented_scan_in_progress")
task_app.send_task.assert_not_called()
release_lock.assert_called_once()
if __name__ == "__main__":
unittest.main()