qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Replace page-level checkpoint with segment-level resume

Browse Source
This commit is contained in:
qananasikq
2026-04-17 17:51:23 +03:00
parent 05d1ffb5ac
commit a8c5f8aa41
4 changed files with 153 additions and 432 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
tests/test_scraper.py
View File

@@ -5,7 +5,7 @@ from types import SimpleNamespace
from unittest.mock import MagicMock
from iaai_scraper.core.config import Settings
from iaai_scraper.core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.storage.schemas import CarRecord
@@ -107,16 +107,13 @@ class TestScraperSync(unittest.TestCase):
{"make": "HONDA", "year_min": None, "year_max": None},
]
# Resume: пропускаем TOYOTA, начинаем с FORD на стр. 5.
# Resume: пропускаем TOYOTA, начинаем сразу с FORD.
result = scraper.sync_listing_segmented(
segments=segments, start_segment=1, start_page=5,
segments=segments, start_segment=1,
)
self.assertEqual(result["segments_completed"], 2) # FORD + HONDA
self.assertEqual(result["cars_upserted"], 10)
# FORD: start_page=5, HONDA: start_page=1.
self.assertEqual(call_args_log[0]["start_page"], 5)
self.assertEqual(call_args_log[1]["start_page"], 1)
# URL содержит бренд.
self.assertIn("FORD", call_args_log[0]["listing_url"])
self.assertIn("HONDA", call_args_log[1]["listing_url"])
@@ -223,7 +220,7 @@ class TestScraperSync(unittest.TestCase):
scraper.listing_collector.open_cars_listing.assert_called_once()
page.reload.assert_not_called()
def test_sync_listing_resets_stale_resume_checkpoint_and_restarts_from_page_one(self) -> None:
def test_sync_listing_always_starts_from_page_one(self) -> None:
scraper = self._make_scraper()
scraper.settings.celery.batch_size = 1
@@ -234,10 +231,11 @@ class TestScraperSync(unittest.TestCase):
next_page_detected=False,
)
progress_calls: list[int] = []
scraper._get_page_with_warmup = MagicMock(return_value=page)
scraper._reopen_listing_and_resume = MagicMock(side_effect=ListingResumeError("checkpoint page is no longer reachable"))
# Pagination-resume убран: _reopen_listing_and_resume не должен вызываться.
scraper._reopen_listing_and_resume = MagicMock(
side_effect=AssertionError("pagination resume must not be used")
)
scraper.listing_collector.open_cars_listing = MagicMock()
scraper.listing_collector.apply_filters = MagicMock(return_value={
"make": None,
@@ -261,13 +259,11 @@ class TestScraperSync(unittest.TestCase):
limit=None,
effective_only_new=False,
started_at=0.0,
start_page=50,
progress_callback=progress_calls.append,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
)
self.assertEqual(progress_calls, [0, 1])
scraper.listing_collector.open_cars_listing.assert_called_once()
scraper._reopen_listing_and_resume.assert_not_called()
scraper.sync_batch.assert_called_once()
self.assertEqual(result["cars_upserted"], 1)
self.assertEqual(result["total"], 1)

242
tests/test_worker_tasks.py
View File

@@ -1,6 +1,5 @@
from __future__ import annotations
import json
import unittest
from unittest.mock import MagicMock, patch
@@ -146,69 +145,84 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
self.assertEqual(acquire_lock.call_count, 2)
release_lock.assert_called_once()
def test_sync_listing_checkpoint_save_load_and_resume(self) -> None:
# Roundtrip: save → load.
redis_client = MagicMock()
def test_checkpoint_save_load_roundtrip(self) -> None:
storage: dict[str, str] = {}
def _fake_set(key, value, ex=None):
storage[key] = value
return True
redis_client = MagicMock()
redis_client.set.side_effect = _fake_set
redis_client.get.side_effect = lambda key: storage.get(key)
tasks._save_sync_checkpoint(
redis_client, task_id="task-1", page_number=12,
make=None, model=None, lane="iaai_cars",
)
checkpoint = tasks._load_sync_checkpoint(redis_client)
self.assertIsNotNone(checkpoint)
self.assertEqual(checkpoint["last_successful_page"], 12)
tasks._save_last_completed_segment(redis_client, 12)
# Resume: задача стартует со страницы checkpoint + 1.
self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
def test_load_checkpoint_clears_invalid_payload(self) -> None:
redis_client = MagicMock()
redis_client.get.return_value = "{not-a-number"
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
def test_load_checkpoint_empty_when_missing(self) -> None:
redis_client = MagicMock()
redis_client.get.return_value = None
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
redis_client.delete.assert_not_called()
def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
segments = [
{"make": "ACURA"},
{"make": "AUDI"},
{"make": "BMW"},
{"make": "EAGLE"},
]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client2 = MagicMock()
redis_client2.get.side_effect = lambda key: json.dumps({
"status": "in_progress", "last_successful_page": 9,
"make": None, "model": None, "lane": "iaai_cars",
}) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
get_redis.return_value = redis_client2
stop_event = MagicMock()
heartbeat_thread = MagicMock()
start_heartbeat.return_value = (stop_event, heartbeat_thread)
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
"1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
)
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_listing_mock = MagicMock(return_value={
sync_segmented_mock = MagicMock(return_value={
"run_id": 11, "status": "success", "full_scan_completed": True,
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
})
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-789")
tasks.sync_listing_task.push_request(id="task-resume-seg")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 10)
clear_checkpoint.assert_called()
# last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
release_lock.assert_called_once()
def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
@@ -218,25 +232,23 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: json.dumps({
"status": "in_progress", "last_successful_page": 25,
"make": None, "model": None, "lane": "iaai_cars",
}) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
# Оставшийся чекпоинт не должен использоваться.
redis_client.get.side_effect = lambda key: (
"0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
)
get_redis.return_value = redis_client
stop_event = MagicMock()
heartbeat_thread = MagicMock()
start_heartbeat.return_value = (stop_event, heartbeat_thread)
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_listing_mock = MagicMock(return_value={
sync_segmented_mock = MagicMock(return_value={
"run_id": 14, "status": "success", "full_scan_completed": True,
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
})
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
@@ -247,77 +259,51 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 1)
self.assertIsNone(sync_listing_mock.call_args.kwargs["progress_callback"])
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
clear_checkpoint.assert_called()
release_lock.assert_called_once()
def test_sync_listing_checkpoint_zero_page_restarts_from_first_page(self) -> None:
def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: json.dumps({
"status": "in_progress", "last_successful_page": 0,
"make": None, "model": None, "lane": "iaai_cars",
}) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
redis_client.get.side_effect = lambda key: (
"99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
)
get_redis.return_value = redis_client
stop_event = MagicMock()
heartbeat_thread = MagicMock()
start_heartbeat.return_value = (stop_event, heartbeat_thread)
start_heartbeat.return_value = (MagicMock(), MagicMock())
sync_listing_mock = MagicMock(return_value={
"run_id": 12, "status": "success", "full_scan_completed": True,
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
sync_segmented_mock = MagicMock(return_value={
"run_id": 15, "status": "success", "full_scan_completed": True,
"cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
})
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-790")
tasks.sync_listing_task.push_request(id="task-beyond")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(sync_listing_mock.call_args.kwargs["start_page"], 1)
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
release_lock.assert_called_once()
def test_save_sync_checkpoint_sets_ttl(self) -> None:
redis_client = MagicMock()
tasks._save_sync_checkpoint(
redis_client,
task_id="task-1",
page_number=3,
make=None,
model=None,
lane="iaai_cars",
)
redis_client.set.assert_called_once()
self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
def test_load_sync_checkpoint_clears_invalid_payload(self) -> None:
redis_client = MagicMock()
redis_client.get.return_value = "{broken-json"
checkpoint = tasks._load_sync_checkpoint(redis_client)
self.assertIsNone(checkpoint)
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
def test_sync_listing_task_clears_checkpoint_on_partial_success_when_full_scan_completed(self) -> None:
def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
@@ -341,9 +327,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
redis_client = MagicMock()
redis_client.get.return_value = None
get_redis.return_value = redis_client
stop_event = MagicMock()
heartbeat_thread = MagicMock()
start_heartbeat.return_value = (stop_event, heartbeat_thread)
start_heartbeat.return_value = (MagicMock(), MagicMock())
tasks.sync_listing_task.push_request(id="task-791")
try:
@@ -352,7 +336,9 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "partial_success")
clear_checkpoint.assert_called_once_with(redis_client)
# Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
# до браузерного job + после. Главное — вызов произошёл.
clear_checkpoint.assert_called()
release_lock.assert_called_once()
def test_try_set_followup_pending_deduplicates(self) -> None:
@@ -417,9 +403,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
redis_client = MagicMock()
redis_client.get.return_value = None
get_redis.return_value = redis_client
stop_event = MagicMock()
heartbeat_thread = MagicMock()
start_heartbeat.return_value = (stop_event, heartbeat_thread)
start_heartbeat.return_value = (MagicMock(), MagicMock())
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
tasks.sync_listing_task.push_request(id="task-900")
@@ -433,86 +417,6 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
clear_pending.assert_called()
task_app.send_task.assert_not_called()
def test_bump_checkpoint_resume_failure_deletes_after_second_failure(self) -> None:
redis_client = MagicMock()
checkpoint = {
"status": "in_progress",
"last_successful_page": 9,
"resume_failures": 1,
"lane": "iaai_cars",
}
failures, deleted = tasks._bump_checkpoint_resume_failure(
redis_client,
checkpoint,
reason="resume failed",
)
self.assertEqual(failures, 2)
self.assertTrue(deleted)
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
def test_bump_checkpoint_resume_failure_persists_first_failure(self) -> None:
redis_client = MagicMock()
checkpoint = {
"status": "in_progress",
"last_successful_page": 9,
"resume_failures": 0,
"lane": "iaai_cars",
}
failures, deleted = tasks._bump_checkpoint_resume_failure(
redis_client,
checkpoint,
reason="resume failed",
)
self.assertEqual(failures, 1)
self.assertFalse(deleted)
redis_client.set.assert_called_once()
def test_sync_listing_task_deletes_checkpoint_after_second_resume_failure(self) -> None:
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_bump_checkpoint_resume_failure", return_value=(2, True)) as bump_failures, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch.object(tasks.sync_listing_task, "retry", side_effect=tasks.sync_listing_task.MaxRetriesExceededError()), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: json.dumps({
"status": "in_progress",
"last_successful_page": 9,
"resume_failures": 1,
"make": None,
"model": None,
"lane": "iaai_cars",
}) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
get_redis.return_value = redis_client
stop_event = MagicMock()
heartbeat_thread = MagicMock()
start_heartbeat.return_value = (stop_event, heartbeat_thread)
with patch.object(tasks, "IAAIScraper") as scraper_cls:
scraper_ctx = MagicMock()
scraper_ctx.__enter__.return_value.sync_listing = MagicMock(side_effect=RuntimeError("Failed to resume listing at page 10"))
scraper_ctx.__exit__.return_value = None
scraper_cls.return_value = scraper_ctx
tasks.sync_listing_task.push_request(id="task-793")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "failed")
bump_failures.assert_called_once()
release_lock.assert_called_once()
if __name__ == "__main__":
unittest.main()