qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

Browse Source
This commit is contained in:
qananasikq
2026-04-20 16:27:42 +03:00
parent 55203d33ea
commit 65d5f8e1eb
23 changed files with 3436 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

335
tests/test_worker_tasks.py
View File

@@ -6,7 +6,210 @@ from unittest.mock import MagicMock, patch
from iaai_scraper.worker import tasks
def make_settings_stub(*, parallel_segments: bool = False) -> MagicMock:
settings_stub = MagicMock()
settings_stub.celery.parallel_segments = parallel_segments
settings_stub.celery.task_soft_time_limit = 3300
settings_stub.celery.task_time_limit = 3600
settings_stub.celery.task_stall_timeout_seconds = 600
settings_stub.listing.listing_segments_json = "ignored"
settings_stub.discovery.mode = "listing"
settings_stub.discovery.hourly_mode = "rolling_refresh"
return settings_stub
class TestWorkerTaskLockHelpers(unittest.TestCase):
def test_sync_listing_task_uses_hourly_sitemap_even_when_mode_is_not_sitemap(self) -> None:
settings_stub = make_settings_stub(parallel_segments=False)
settings_stub.discovery.mode = "listing"
settings_stub.discovery.hourly_mode = "rolling_refresh"
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "Settings", return_value=settings_stub), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
"status": "success",
"run_id": None,
"cars_upserted": 25,
"cars_failed": 0,
"images_upserted": 50,
"skipped_existing": 100,
"elapsed_seconds": None,
"failures": [],
"hourly_mode": "sitemap_rolling_refresh",
"discovered_urls": 105,
"new_urls": 5,
"refresh_urls": 20,
"sold_marked": 2,
}) as hourly_sync, \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.return_value = None
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
tasks.sync_listing_task.push_request(id="task-hourly-force-sitemap")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
hourly_sync.assert_called_once()
release_lock.assert_called_once()
def test_sync_listing_task_uses_hourly_sitemap_rolling_refresh_after_bootstrap(self) -> None:
settings_stub = make_settings_stub(parallel_segments=False)
settings_stub.discovery.mode = "sitemap"
settings_stub.discovery.hourly_mode = "rolling_refresh"
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "Settings", return_value=settings_stub), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
"status": "success",
"run_id": None,
"cars_upserted": 25,
"cars_failed": 0,
"images_upserted": 50,
"skipped_existing": 100,
"elapsed_seconds": None,
"failures": [],
"hourly_mode": "sitemap_rolling_refresh",
"discovered_urls": 105,
"new_urls": 5,
"refresh_urls": 20,
"sold_marked": 2,
}) as hourly_sync, \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.return_value = None
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
tasks.sync_listing_task.push_request(id="task-hourly")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
hourly_sync.assert_called_once()
release_lock.assert_called_once()
def test_sync_listing_task_can_use_hourly_sitemap_diff_when_configured(self) -> None:
settings_stub = make_settings_stub(parallel_segments=False)
settings_stub.discovery.mode = "sitemap"
settings_stub.discovery.hourly_mode = "diff"
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "Settings", return_value=settings_stub), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_hourly_sitemap_diff_sync", return_value={
"status": "success",
"run_id": None,
"cars_upserted": 5,
"cars_failed": 0,
"images_upserted": 10,
"skipped_existing": 100,
"elapsed_seconds": None,
"failures": [],
"hourly_mode": "sitemap_diff",
"discovered_urls": 105,
"new_urls": 5,
"sold_marked": 2,
}) as hourly_sync, \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.return_value = None
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
tasks.sync_listing_task.push_request(id="task-hourly-diff")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(result["hourly_mode"], "sitemap_diff")
hourly_sync.assert_called_once()
release_lock.assert_called_once()
def test_sync_listing_task_forces_sitemap_full_scan_in_bootstrap(self) -> None:
settings_stub = make_settings_stub(parallel_segments=False)
settings_stub.discovery.mode = "listing"
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "Settings", return_value=settings_stub), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[{"make": "HONDA"}]):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.return_value = None
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_listing_mock = MagicMock(return_value={
"run_id": 99,
"status": "success",
"full_scan_completed": True,
"cars_upserted": 10,
"cars_failed": 0,
"images_upserted": 20,
"skipped_existing": 0,
"elapsed_seconds": 1.0,
"failures": [],
})
sync_listing_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
scraper = MagicMock()
scraper.sync_listing = sync_listing_mock
scraper.sync_listing_segmented = sync_listing_segmented_mock
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value = scraper
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-bootstrap-sitemap")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
sync_listing_mock.assert_called_once_with(
make=None,
model=None,
lane="iaai_cars",
limit=None,
only_new=False,
)
sync_listing_segmented_mock.assert_not_called()
release_lock.assert_called_once()
def test_lock_acquire_refresh_release(self) -> None:
redis_client = MagicMock()
@@ -175,15 +378,17 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
redis_client.delete.assert_not_called()
def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
def test_sync_listing_bootstrap_prefers_sitemap_over_segment_resume(self) -> None:
segments = [
{"make": "ACURA"},
{"make": "AUDI"},
{"make": "BMW"},
{"make": "EAGLE"},
]
settings_stub = make_settings_stub()
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "Settings", return_value=settings_stub), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -199,13 +404,15 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_segmented_mock = MagicMock(return_value={
sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
sync_listing_mock = MagicMock(return_value={
"run_id": 11, "status": "success", "full_scan_completed": True,
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
})
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
@@ -216,21 +423,42 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
# last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
sync_listing_mock.assert_called_once_with(
make=None,
model=None,
lane="iaai_cars",
limit=None,
only_new=False,
)
sync_segmented_mock.assert_not_called()
release_lock.assert_called_once()
def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
def test_sync_listing_hourly_path_ignores_checkpoint_after_full_scan_completed(self) -> None:
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
settings_stub = make_settings_stub()
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "Settings", return_value=settings_stub), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks, "_hourly_sitemap_rolling_refresh_sync", return_value={
"status": "success",
"run_id": None,
"cars_upserted": 1,
"cars_failed": 0,
"images_upserted": 0,
"skipped_existing": 0,
"elapsed_seconds": None,
"failures": [],
"hourly_mode": "sitemap_rolling_refresh",
"discovered_urls": 10,
"new_urls": 1,
"refresh_urls": 1,
"sold_marked": 0,
}) as hourly_sync, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
@@ -242,32 +470,24 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_segmented_mock = MagicMock(return_value={
"run_id": 14, "status": "success", "full_scan_completed": True,
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
})
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-792")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
tasks.sync_listing_task.push_request(id="task-792")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
self.assertEqual(result["hourly_mode"], "sitemap_rolling_refresh")
hourly_sync.assert_called_once()
clear_checkpoint.assert_called()
release_lock.assert_called_once()
def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
def test_sync_listing_bootstrap_ignores_beyond_segment_checkpoint(self) -> None:
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
settings_stub = make_settings_stub()
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "Settings", return_value=settings_stub), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -283,13 +503,15 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_segmented_mock = MagicMock(return_value={
sync_segmented_mock = MagicMock(side_effect=AssertionError("segmented path should not be used"))
sync_listing_mock = MagicMock(return_value={
"run_id": 15, "status": "success", "full_scan_completed": True,
"cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
})
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
@@ -300,7 +522,14 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
sync_listing_mock.assert_called_once_with(
make=None,
model=None,
lane="iaai_cars",
limit=None,
only_new=False,
)
sync_segmented_mock.assert_not_called()
release_lock.assert_called_once()
def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
@@ -377,28 +606,38 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
self.assertTrue(should_enqueue)
def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
settings_stub = make_settings_stub(parallel_segments=False)
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
patch.object(tasks, "_clear_followup_pending") as clear_pending, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch.object(tasks, "_run_browser_job", return_value={
"run_id": 99,
"status": "failed",
"full_scan_completed": False,
"cars_upserted": 0,
"cars_failed": 0,
"images_upserted": 0,
"skipped_existing": 0,
"elapsed_seconds": 1.0,
"failures": [{"vehicle_url": "listing", "error": "bad resume"}],
"listing": {"vehicles_collected": 0},
}):
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "Settings", return_value=settings_stub), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
patch.object(
tasks,
"_bump_bootstrap_failure_streak",
return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False),
) as bump_streak, \
patch.object(tasks, "_clear_followup_pending") as clear_pending, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch.object(
tasks,
"_run_browser_job",
return_value={
"run_id": 99,
"status": "failed",
"full_scan_completed": False,
"cars_upserted": 0,
"cars_failed": 0,
"images_upserted": 0,
"skipped_existing": 0,
"elapsed_seconds": 1.0,
"failures": [{"vehicle_url": "listing", "error": "bad resume"}],
"listing": {"vehicles_collected": 0},
},
):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.return_value = None