add helper scripts
This commit is contained in:
qananasikq
723
tests/test_worker_tasks.py
Normal file
723
tests/test_worker_tasks.py
Normal file
@@ -0,0 +1,723 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from dataclasses import replace
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from dubizzle_scraper.worker import tasks
|
||||
from dubizzle_scraper.core.config import settings as base_settings
|
||||
|
||||
|
||||
class TestWorkerTaskLockHelpers(unittest.TestCase):
|
||||
def test_lock_acquire_refresh_release(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
|
||||
# Acquire.
|
||||
redis_client.set.return_value = True
|
||||
self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
|
||||
redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
|
||||
|
||||
# Refresh.
|
||||
redis_client.eval.return_value = 1
|
||||
self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
|
||||
|
||||
# Release.
|
||||
redis_client.eval.reset_mock()
|
||||
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
|
||||
args = redis_client.eval.call_args[0]
|
||||
self.assertEqual(args[2], "lock:key")
|
||||
self.assertEqual(args[3], "owner-token")
|
||||
|
||||
def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=False):
|
||||
persistence = MagicMock()
|
||||
get_persistence.return_value = persistence
|
||||
get_redis.return_value = MagicMock()
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-123")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
persistence.create_tables.assert_called_once()
|
||||
self.assertEqual(result["status"], "skipped")
|
||||
self.assertEqual(result["reason"], "sync_already_running")
|
||||
|
||||
def test_sync_listing_task_releases_owned_lock(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 7,
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 4,
|
||||
"skipped_existing": 1,
|
||||
"elapsed_seconds": 1.25,
|
||||
}):
|
||||
persistence = MagicMock()
|
||||
get_persistence.return_value = persistence
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
stop_event = MagicMock()
|
||||
heartbeat_thread = MagicMock()
|
||||
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-123")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(make="Toyota")
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
stop_event.set.assert_called_once()
|
||||
heartbeat_thread.join.assert_called_once()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_clear_orphan_sync_listing_lock(self) -> None:
|
||||
# Нет запущенных задач → удаляет.
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = "owner-token"
|
||||
redis_client.ttl.return_value = 120
|
||||
celery_app = MagicMock()
|
||||
inspector = MagicMock()
|
||||
inspector.active.return_value = {"worker@node": []}
|
||||
inspector.reserved.return_value = {"worker@node": []}
|
||||
inspector.scheduled.return_value = {"worker@node": []}
|
||||
celery_app.control.inspect.return_value = inspector
|
||||
|
||||
self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
|
||||
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
|
||||
|
||||
# Задача активна → не удаляет.
|
||||
redis_client2 = MagicMock()
|
||||
redis_client2.get.return_value = "owner-token"
|
||||
inspector2 = MagicMock()
|
||||
inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
|
||||
inspector2.reserved.return_value = {"worker@node": []}
|
||||
inspector2.scheduled.return_value = {"worker@node": []}
|
||||
celery_app2 = MagicMock()
|
||||
celery_app2.control.inspect.return_value = inspector2
|
||||
|
||||
self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
|
||||
redis_client2.delete.assert_not_called()
|
||||
|
||||
def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \
|
||||
patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 9,
|
||||
"cars_upserted": 3,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 5,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 2.0,
|
||||
}):
|
||||
persistence = MagicMock()
|
||||
get_persistence.return_value = persistence
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
stop_event = MagicMock()
|
||||
heartbeat_thread = MagicMock()
|
||||
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-456")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(make="Honda")
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
clear_orphan.assert_called_once()
|
||||
self.assertEqual(acquire_lock.call_count, 2)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_checkpoint_save_load_roundtrip(self) -> None:
|
||||
storage: dict[str, str] = {}
|
||||
|
||||
def _fake_set(key, value, ex=None):
|
||||
storage[key] = value
|
||||
return True
|
||||
|
||||
redis_client = MagicMock()
|
||||
redis_client.set.side_effect = _fake_set
|
||||
redis_client.get.side_effect = lambda key: storage.get(key)
|
||||
|
||||
tasks._save_last_completed_segment(redis_client, 12)
|
||||
|
||||
self.assertEqual(tasks._load_last_completed_segment(redis_client), 12)
|
||||
self.assertEqual(redis_client.set.call_args.kwargs["ex"], tasks.SYNC_LISTING_CHECKPOINT_TTL_SECONDS)
|
||||
|
||||
def test_load_checkpoint_clears_invalid_payload(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = "{not-a-number"
|
||||
|
||||
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
|
||||
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_CHECKPOINT_KEY)
|
||||
|
||||
def test_load_checkpoint_empty_when_missing(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
|
||||
self.assertIsNone(tasks._load_last_completed_segment(redis_client))
|
||||
redis_client.delete.assert_not_called()
|
||||
|
||||
def test_sync_listing_resumes_from_next_segment_during_bootstrap(self) -> None:
|
||||
segments = [
|
||||
{"make": "ACURA"},
|
||||
{"make": "AUDI"},
|
||||
{"make": "BMW"},
|
||||
{"make": "EAGLE"},
|
||||
]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(
|
||||
tasks,
|
||||
"Settings",
|
||||
return_value=replace(
|
||||
base_settings,
|
||||
discovery=replace(
|
||||
base_settings.discovery,
|
||||
always_full_scan=False,
|
||||
mode="listing",
|
||||
),
|
||||
celery=replace(
|
||||
base_settings.celery,
|
||||
parallel_segments=False,
|
||||
),
|
||||
),
|
||||
), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
"1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
)
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
"run_id": 11, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-resume-seg")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
# last_completed=1 → start_segment=2 (AUDI завершён, возобновляем с BMW).
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 2)
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_ignores_checkpoint_after_full_scan_completed(self) -> None:
|
||||
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(
|
||||
tasks,
|
||||
"Settings",
|
||||
return_value=replace(
|
||||
base_settings,
|
||||
discovery=replace(
|
||||
base_settings.discovery,
|
||||
always_full_scan=False,
|
||||
mode="listing",
|
||||
),
|
||||
celery=replace(
|
||||
base_settings.celery,
|
||||
parallel_segments=False,
|
||||
),
|
||||
),
|
||||
), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
# Оставшийся чекпоинт не должен использоваться.
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
"0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
)
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
"run_id": 14, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 1, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-792")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(limit=1)
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
|
||||
self.assertIsNone(sync_segmented_mock.call_args.kwargs["progress_callback"])
|
||||
clear_checkpoint.assert_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_checkpoint_beyond_segments_restarts_from_zero(self) -> None:
|
||||
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(
|
||||
tasks,
|
||||
"Settings",
|
||||
return_value=replace(
|
||||
base_settings,
|
||||
discovery=replace(
|
||||
base_settings.discovery,
|
||||
always_full_scan=False,
|
||||
mode="listing",
|
||||
),
|
||||
celery=replace(
|
||||
base_settings.celery,
|
||||
parallel_segments=False,
|
||||
),
|
||||
),
|
||||
), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
"99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
)
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
"run_id": 15, "status": "success", "full_scan_completed": True,
|
||||
"cars_upserted": 0, "cars_failed": 0, "images_upserted": 0,
|
||||
"skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-beyond")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 0)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_clears_checkpoint_on_hourly_run(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 13,
|
||||
"status": "partial_success",
|
||||
"full_scan_completed": True,
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 1,
|
||||
"images_upserted": 3,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 4.0,
|
||||
"failures": [{"vehicle_url": "v", "error": "e"}],
|
||||
}), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-791")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(make="Toyota")
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "partial_success")
|
||||
# Hourly-ветка (full_scan_done=True) всегда удаляет оставшийся чекпоинт
|
||||
# до браузерного job + после. Главное — вызов произошёл.
|
||||
clear_checkpoint.assert_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_try_set_followup_pending_deduplicates(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.set.side_effect = [True, False]
|
||||
|
||||
self.assertTrue(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
|
||||
self.assertFalse(tasks._try_set_followup_pending(redis_client, ttl_seconds=120))
|
||||
|
||||
def test_bump_bootstrap_failure_streak_opens_circuit_breaker(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.incr.return_value = tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT
|
||||
|
||||
streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
|
||||
redis_client,
|
||||
reason="max_retries_exceeded",
|
||||
)
|
||||
|
||||
self.assertEqual(streak, tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT)
|
||||
self.assertFalse(should_enqueue)
|
||||
redis_client.expire.assert_called_once_with(
|
||||
tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY,
|
||||
tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS,
|
||||
)
|
||||
|
||||
def test_bump_bootstrap_failure_streak_allows_retry_before_limit(self) -> None:
|
||||
redis_client = MagicMock()
|
||||
redis_client.incr.return_value = 1
|
||||
|
||||
streak, should_enqueue = tasks._bump_bootstrap_failure_streak(
|
||||
redis_client,
|
||||
reason="bootstrap_not_completed",
|
||||
)
|
||||
|
||||
self.assertEqual(streak, 1)
|
||||
self.assertTrue(should_enqueue)
|
||||
|
||||
def test_sync_listing_task_does_not_enqueue_followup_after_bootstrap_error_limit(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
|
||||
patch.object(tasks, "_bump_bootstrap_failure_streak", return_value=(tasks.SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT, False)) as bump_streak, \
|
||||
patch.object(tasks, "_clear_followup_pending") as clear_pending, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 99,
|
||||
"status": "failed",
|
||||
"full_scan_completed": False,
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [{"vehicle_url": "listing", "error": "bad resume"}],
|
||||
"listing": {"vehicles_collected": 0},
|
||||
}):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
|
||||
tasks.sync_listing_task.push_request(id="task-900")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "failed")
|
||||
bump_streak.assert_called_once()
|
||||
clear_pending.assert_called()
|
||||
task_app.send_task.assert_not_called()
|
||||
|
||||
def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
|
||||
patch.object(tasks, "_release_lock_if_owner"), \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \
|
||||
patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
|
||||
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
|
||||
tasks.sync_listing_task.push_request(id="task-soft-timeout")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run(make="Toyota")
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "timed_out")
|
||||
set_pending.assert_called_once()
|
||||
task_app.send_task.assert_not_called()
|
||||
|
||||
def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
|
||||
patch.object(tasks, "_release_lock_if_owner"), \
|
||||
patch.object(tasks, "_try_set_followup_pending", return_value=True), \
|
||||
patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \
|
||||
patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
|
||||
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job", return_value={
|
||||
"run_id": 101,
|
||||
"status": "partial_success",
|
||||
"full_scan_completed": False,
|
||||
"cars_upserted": 2,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 1,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [],
|
||||
"listing": {"vehicles_collected": 2},
|
||||
}):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
|
||||
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
|
||||
tasks.sync_listing_task.push_request(id="task-breaker-stop")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "partial_success")
|
||||
# Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly.
|
||||
self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list))
|
||||
self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list))
|
||||
clear_checkpoint.assert_called_once()
|
||||
task_app.send_task.assert_not_called()
|
||||
|
||||
def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None:
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(
|
||||
tasks,
|
||||
"Settings",
|
||||
return_value=replace(
|
||||
base_settings,
|
||||
discovery=replace(base_settings.discovery, always_full_scan=True),
|
||||
),
|
||||
), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch.object(tasks, "_run_browser_job") as run_job, \
|
||||
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
run_job.return_value = {
|
||||
"run_id": 17,
|
||||
"status": "success",
|
||||
"full_scan_completed": True,
|
||||
"cars_upserted": 1,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [],
|
||||
}
|
||||
|
||||
tasks.sync_listing_task.push_request(id="task-always-full")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
set_full_scan_done.assert_called_with(redis_client, False)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None:
|
||||
segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(
|
||||
tasks,
|
||||
"Settings",
|
||||
return_value=replace(
|
||||
base_settings,
|
||||
discovery=replace(base_settings.discovery, always_full_scan=True),
|
||||
),
|
||||
), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
|
||||
patch.object(tasks.sync_listing_task, "update_state"), \
|
||||
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.get.side_effect = lambda key: (
|
||||
"0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
|
||||
)
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
sync_segmented_mock = MagicMock(return_value={
|
||||
"run_id": 18,
|
||||
"status": "success",
|
||||
"full_scan_completed": True,
|
||||
"cars_upserted": 1,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"skipped_existing": 0,
|
||||
"elapsed_seconds": 1.0,
|
||||
"failures": [],
|
||||
})
|
||||
scraper_ctx = MagicMock()
|
||||
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
|
||||
scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
|
||||
scraper_ctx.__exit__.return_value = None
|
||||
|
||||
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
|
||||
tasks.sync_listing_task.push_request(id="task-always-full-resume")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1)
|
||||
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None:
|
||||
segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(
|
||||
tasks,
|
||||
"Settings",
|
||||
return_value=replace(
|
||||
base_settings,
|
||||
discovery=replace(base_settings.discovery, always_full_scan=True),
|
||||
celery=replace(base_settings.celery, parallel_segments=True),
|
||||
),
|
||||
), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks, "_reset_segments_progress") as reset_progress, \
|
||||
patch.object(tasks, "_set_segmented_scan_active") as set_active, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
redis_client = MagicMock()
|
||||
redis_client.smembers.return_value = {"0", "1"}
|
||||
redis_client.get.return_value = None
|
||||
get_redis.return_value = redis_client
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
|
||||
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
tasks.sync_listing_task.push_request(id="task-always-hourly")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "success")
|
||||
self.assertEqual(result["segments_dispatched"], 2)
|
||||
reset_progress.assert_called_once_with(redis_client, len(segments))
|
||||
set_active.assert_called_once_with(redis_client, total=len(segments))
|
||||
self.assertEqual(task_app.send_task.call_count, 2)
|
||||
release_lock.assert_called_once()
|
||||
|
||||
def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None:
|
||||
segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
|
||||
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
||||
patch.object(tasks, "_get_redis") as get_redis, \
|
||||
patch.object(
|
||||
tasks,
|
||||
"Settings",
|
||||
return_value=replace(
|
||||
base_settings,
|
||||
discovery=replace(base_settings.discovery, always_full_scan=True),
|
||||
celery=replace(base_settings.celery, parallel_segments=True),
|
||||
),
|
||||
), \
|
||||
patch.object(tasks, "_acquire_lock", return_value=True), \
|
||||
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
||||
patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \
|
||||
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
||||
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
||||
patch.object(tasks.sync_listing_task, "update_state"):
|
||||
get_persistence.return_value = MagicMock()
|
||||
get_redis.return_value = MagicMock()
|
||||
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
||||
|
||||
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
|
||||
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
|
||||
tasks.sync_listing_task.push_request(id="task-always-skip-duplicate")
|
||||
try:
|
||||
result = tasks.sync_listing_task.run()
|
||||
finally:
|
||||
tasks.sync_listing_task.pop_request()
|
||||
|
||||
self.assertEqual(result["status"], "skipped")
|
||||
self.assertEqual(result["reason"], "segmented_scan_in_progress")
|
||||
task_app.send_task.assert_not_called()
|
||||
release_lock.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user