qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix docker scraping

Browse Source 
improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup
This commit is contained in:
qananasikq
2026-04-13 16:35:08 +03:00
parent b9557922ed
commit a4c93a1df1
30 changed files with 2239 additions and 624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
tests/test_db.py
View File

@@ -92,16 +92,6 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(len(images), 1)
self.assertIn("imageKeys=2", images[0].fullres_image)
def test_persistence_ignores_non_db_fields(self) -> None:
record = self._record("1000")
result = self.persistence.upsert_car(record)
self.assertEqual(result["action"], "inserted")
with self.persistence.session_scope() as session:
car = session.execute(select(Car).where(Car.origin_id == "1000")).scalar_one()
self.assertEqual(car.origin_id, "1000")
def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
first_run_id = self.persistence.start_sync_run("lane-a")
second_run_id = self.persistence.start_sync_run("lane-b")

6
tests/test_listing.py
View File

@@ -31,12 +31,6 @@ class TestListingUnit(unittest.TestCase):
page = _FakePage({})
self.assertFalse(ListingCollector._has_next_page(page))
def test_constructor_with_settings_and_pacer(self) -> None:
settings = Settings()
pacer = HumanPacer(settings)
collector = ListingCollector(settings, pacer)
self.assertIsNotNone(collector)
if __name__ == "__main__":
unittest.main()

6
tests/test_parser.py
View File

@@ -13,15 +13,15 @@ class TestVehicleParserUnit(unittest.TestCase):
dom_text = """
Stock #:
45089484
VIN (Status):
1HGCM82633A123456 (OK)
Primary Damage:
Front End
Odometer:
50,123 mi (Actual)
"""
result = self.parser._parse_dom_key_value_pairs(dom_text)
self.assertEqual(result.get("lot_number"), "45089484")
self.assertEqual(result.get("vin"), "1HGCM82633A123456 (OK)")
self.assertEqual(result.get("primary_damage"), "Front End")
self.assertEqual(result.get("odometer"), "50,123 mi (Actual)")
def test_parse_title_for_year_make_model(self) -> None:
parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "")

89
tests/test_scraper.py
View File

@@ -51,15 +51,12 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=2)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set()))
scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]})
page = MagicMock()
scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
scraper._get_page = MagicMock(return_value=page)
scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called"))
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
})
result = scraper.sync_listing()
@@ -67,8 +64,7 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["cars_failed"], 0)
self.assertIn("trace_id", result)
self.assertIn("elapsed_seconds", result)
self.assertEqual(scraper.persistence.upsert_car.call_count, 1)
page.close.assert_called_once()
scraper.sync_batch.assert_called_once()
def test_sync_listing_respects_limit(self) -> None:
scraper = self._make_scraper()
@@ -76,38 +72,25 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=3)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 1})
scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
]
# Проверка пути only_new с limit.
scraper._collect_listing_iterative = MagicMock(return_value=(
["https://www.iaai.com/VehicleDetail/222~US"],
["https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US"],
{"vehicle_urls": [], "pages_collected": 1, "early_stopped": False},
0,
))
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [],
})
scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("222")})
scraper._get_page = MagicMock(return_value=MagicMock())
scraper.sync_listing(limit=1)
self.assertEqual(scraper._scrape_on_page.call_count, 1)
def test_sync_listing_does_not_count_skipped_as_upserted(self) -> None:
scraper = self._make_scraper()
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=4)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "skipped", "images_upserted": 0})
scraper.persistence.get_existing_origin_urls = MagicMock(return_value=set())
scraper.persistence.get_existing_origin_ids = MagicMock(return_value=set())
scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/444~US"]})
scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("444")})
scraper._get_page = MagicMock(return_value=MagicMock())
result = scraper.sync_listing()
self.assertEqual(result["cars_upserted"], 0)
# Должен уйти только один URL.
scraper.sync_batch.assert_called_once()
batch_urls = scraper.sync_batch.call_args[0][0]
self.assertEqual(len(batch_urls), 1)
def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None:
scraper = self._make_scraper()
@@ -115,9 +98,10 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.create_tables = MagicMock()
scraper.persistence.start_sync_run = MagicMock(return_value=5)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
scraper.persistence.get_existing_origin_urls = MagicMock(return_value={"https://www.iaai.com/VehicleDetail/111~US"})
scraper.persistence.get_existing_origin_ids = MagicMock(return_value={"222"})
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
{"https://www.iaai.com/VehicleDetail/111~US"},
{"iaai:222"},
))
scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [
@@ -126,26 +110,34 @@ class TestScraperSync(unittest.TestCase):
"https://www.iaai.com/VehicleDetail/333~US", # new
]
})
page = MagicMock()
scraper._get_page = MagicMock(return_value=page)
scraper._scrape_on_page = MagicMock(return_value={"db_record": make_db_record("333")})
# Возвращаем результат для одного нового авто.
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [],
})
result = scraper.sync_listing(only_new=True)
self.assertEqual(result["skipped_existing"], 2)
self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(scraper._scrape_on_page.call_count, 1)
scraper.persistence.get_existing_origin_urls.assert_called_once()
scraper.persistence.get_existing_origin_ids.assert_called_once()
# В batch должен попасть только новый URL.
scraper.sync_batch.assert_called_once()
batch_urls = scraper.sync_batch.call_args[0][0]
self.assertEqual(len(batch_urls), 1)
self.assertIn("333", batch_urls[0])
scraper.persistence.get_existing_urls_and_ids.assert_called_once()
def test_close_resets_browser_state(self) -> None:
scraper = self._make_scraper()
http_pool = MagicMock()
scraper._http_pool = http_pool
scraper.context = MagicMock()
scraper.browser = MagicMock()
scraper.playwright = MagicMock()
scraper.close()
http_pool.clear.assert_called_once()
self.assertIsNone(scraper._http_pool)
self.assertIsNone(scraper.context)
self.assertIsNone(scraper.browser)
self.assertIsNone(scraper.playwright)
@@ -172,6 +164,11 @@ class TestScraperSync(unittest.TestCase):
"https://www.iaai.com/VehicleDetail/999~US",
)
def test_is_protection_or_network_error_detects_known_signals(self) -> None:
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
if __name__ == "__main__":
unittest.main()

92
tests/test_worker_tasks.py Normal file
View File

@@ -0,0 +1,92 @@
from __future__ import annotations
import unittest
from unittest.mock import MagicMock, patch
from iaai_scraper.worker import tasks
class TestWorkerTaskLockHelpers(unittest.TestCase):
def test_acquire_lock_returns_true_on_success(self) -> None:
redis_client = MagicMock()
redis_client.set.return_value = True
acquired = tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)
self.assertTrue(acquired)
redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
def test_refresh_lock_if_owner_extends_ttl(self) -> None:
redis_client = MagicMock()
redis_client.eval.return_value = 1
refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)
self.assertTrue(refreshed)
redis_client.eval.assert_called_once()
def test_release_lock_if_owner_uses_owner_token(self) -> None:
redis_client = MagicMock()
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
redis_client.eval.assert_called_once()
args = redis_client.eval.call_args[0]
self.assertEqual(args[1], 1)
self.assertEqual(args[2], "lock:key")
self.assertEqual(args[3], "owner-token")
def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=False):
persistence = MagicMock()
get_persistence.return_value = persistence
get_redis.return_value = MagicMock()
tasks.sync_listing_task.push_request(id="task-123")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
persistence.create_tables.assert_called_once()
self.assertEqual(result["status"], "skipped")
self.assertEqual(result["reason"], "sync_already_running")
def test_sync_listing_task_releases_owned_lock(self) -> None:
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", return_value={
"run_id": 7,
"cars_upserted": 2,
"cars_failed": 0,
"images_upserted": 4,
"skipped_existing": 1,
"elapsed_seconds": 1.25,
}):
persistence = MagicMock()
get_persistence.return_value = persistence
redis_client = MagicMock()
get_redis.return_value = redis_client
stop_event = MagicMock()
heartbeat_thread = MagicMock()
start_heartbeat.return_value = (stop_event, heartbeat_thread)
tasks.sync_listing_task.push_request(id="task-123")
try:
result = tasks.sync_listing_task.run(make="Toyota")
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
stop_event.set.assert_called_once()
heartbeat_thread.join.assert_called_once()
release_lock.assert_called_once()
if __name__ == "__main__":
unittest.main()