190 lines
8.4 KiB
Python
190 lines
8.4 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from openlane_scraper.worker import tasks
|
|
|
|
|
|
class TestWorkerTaskLockHelpers(unittest.TestCase):
|
|
def test_lock_acquire_refresh_release(self) -> None:
|
|
redis_client = MagicMock()
|
|
|
|
# Acquire.
|
|
redis_client.set.return_value = True
|
|
self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120))
|
|
redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
|
|
|
|
# Refresh.
|
|
redis_client.eval.return_value = 1
|
|
self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120))
|
|
|
|
# Release.
|
|
redis_client.eval.reset_mock()
|
|
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
|
|
args = redis_client.eval.call_args[0]
|
|
self.assertEqual(args[2], "lock:key")
|
|
self.assertEqual(args[3], "owner-token")
|
|
|
|
def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
|
|
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
|
patch.object(tasks, "_get_redis") as get_redis, \
|
|
patch.object(tasks, "_acquire_lock", return_value=False):
|
|
persistence = MagicMock()
|
|
get_persistence.return_value = persistence
|
|
get_redis.return_value = MagicMock()
|
|
|
|
tasks.sync_listing_task.push_request(id="task-123")
|
|
try:
|
|
result = tasks.sync_listing_task.run()
|
|
finally:
|
|
tasks.sync_listing_task.pop_request()
|
|
|
|
persistence.create_tables.assert_called_once()
|
|
self.assertEqual(result["status"], "skipped")
|
|
self.assertEqual(result["reason"], "sync_already_running")
|
|
|
|
def test_sync_listing_task_releases_owned_lock(self) -> None:
|
|
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
|
patch.object(tasks, "_get_redis") as get_redis, \
|
|
patch.object(tasks, "_acquire_lock", return_value=True), \
|
|
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
|
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
|
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
|
patch.object(tasks.sync_listing_task, "update_state"), \
|
|
patch("openlane_scraper.worker.tasks.time.sleep"), \
|
|
patch.object(tasks, "_run_browser_job", return_value={
|
|
"run_id": 7,
|
|
"status": "success",
|
|
"cars_upserted": 2,
|
|
"cars_failed": 0,
|
|
"images_upserted": 4,
|
|
"skipped_existing": 1,
|
|
"elapsed_seconds": 1.25,
|
|
}):
|
|
persistence = MagicMock()
|
|
get_persistence.return_value = persistence
|
|
redis_client = MagicMock()
|
|
redis_client.get.return_value = None
|
|
get_redis.return_value = redis_client
|
|
stop_event = MagicMock()
|
|
heartbeat_thread = MagicMock()
|
|
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
|
|
|
tasks.sync_listing_task.push_request(id="task-123")
|
|
try:
|
|
result = tasks.sync_listing_task.run()
|
|
finally:
|
|
tasks.sync_listing_task.pop_request()
|
|
|
|
self.assertEqual(result["status"], "success")
|
|
stop_event.set.assert_called_once()
|
|
heartbeat_thread.join.assert_called_once()
|
|
release_lock.assert_called_once()
|
|
|
|
def test_clear_orphan_sync_listing_lock(self) -> None:
|
|
# Нет запущенных задач → удаляет.
|
|
redis_client = MagicMock()
|
|
redis_client.get.return_value = "owner-token"
|
|
redis_client.ttl.return_value = 120
|
|
celery_app = MagicMock()
|
|
inspector = MagicMock()
|
|
inspector.active.return_value = {"worker@node": []}
|
|
inspector.reserved.return_value = {"worker@node": []}
|
|
inspector.scheduled.return_value = {"worker@node": []}
|
|
celery_app.control.inspect.return_value = inspector
|
|
|
|
self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app))
|
|
redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY)
|
|
|
|
# Задача активна → не удаляет.
|
|
redis_client2 = MagicMock()
|
|
redis_client2.get.return_value = "owner-token"
|
|
inspector2 = MagicMock()
|
|
inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]}
|
|
inspector2.reserved.return_value = {"worker@node": []}
|
|
inspector2.scheduled.return_value = {"worker@node": []}
|
|
celery_app2 = MagicMock()
|
|
celery_app2.control.inspect.return_value = inspector2
|
|
|
|
self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2))
|
|
redis_client2.delete.assert_not_called()
|
|
|
|
def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None:
|
|
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
|
patch.object(tasks, "_get_redis") as get_redis, \
|
|
patch.object(tasks, "_acquire_lock", side_effect=[False, True]) as acquire_lock, \
|
|
patch.object(tasks, "_clear_orphan_sync_listing_lock", return_value=True) as clear_orphan, \
|
|
patch.object(tasks, "_is_full_scan_done", return_value=True), \
|
|
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
|
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
|
patch.object(tasks.sync_listing_task, "update_state"), \
|
|
patch("openlane_scraper.worker.tasks.time.sleep"), \
|
|
patch.object(tasks, "_run_browser_job", return_value={
|
|
"run_id": 9,
|
|
"status": "success",
|
|
"cars_upserted": 3,
|
|
"cars_failed": 0,
|
|
"images_upserted": 5,
|
|
"skipped_existing": 0,
|
|
"elapsed_seconds": 2.0,
|
|
}):
|
|
persistence = MagicMock()
|
|
get_persistence.return_value = persistence
|
|
redis_client = MagicMock()
|
|
redis_client.get.return_value = None
|
|
get_redis.return_value = redis_client
|
|
stop_event = MagicMock()
|
|
heartbeat_thread = MagicMock()
|
|
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
|
|
|
tasks.sync_listing_task.push_request(id="task-456")
|
|
try:
|
|
result = tasks.sync_listing_task.run()
|
|
finally:
|
|
tasks.sync_listing_task.pop_request()
|
|
|
|
self.assertEqual(result["status"], "success")
|
|
clear_orphan.assert_called_once()
|
|
self.assertEqual(acquire_lock.call_count, 2)
|
|
release_lock.assert_called_once()
|
|
|
|
def test_bootstrap_forces_full_scan(self) -> None:
|
|
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
|
patch.object(tasks, "_get_redis") as get_redis, \
|
|
patch.object(tasks, "_acquire_lock", return_value=True), \
|
|
patch.object(tasks, "_is_full_scan_done", return_value=False), \
|
|
patch.object(tasks, "_set_full_scan_done") as set_done, \
|
|
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
|
patch.object(tasks, "_release_lock_if_owner"), \
|
|
patch.object(tasks.sync_listing_task, "update_state"), \
|
|
patch("openlane_scraper.worker.tasks.time.sleep"), \
|
|
patch.object(tasks, "_run_browser_job", return_value={
|
|
"run_id": 11,
|
|
"status": "success",
|
|
"cars_upserted": 5,
|
|
"cars_failed": 0,
|
|
"images_upserted": 10,
|
|
"skipped_existing": 0,
|
|
"elapsed_seconds": 3.0,
|
|
}):
|
|
persistence = MagicMock()
|
|
get_persistence.return_value = persistence
|
|
redis_client = MagicMock()
|
|
get_redis.return_value = redis_client
|
|
start_heartbeat.return_value = (MagicMock(), MagicMock())
|
|
|
|
tasks.sync_listing_task.push_request(id="task-bootstrap")
|
|
try:
|
|
result = tasks.sync_listing_task.run(only_new=True)
|
|
finally:
|
|
tasks.sync_listing_task.pop_request()
|
|
|
|
self.assertEqual(result["status"], "success")
|
|
# Bootstrap should set full_scan_done to True on success.
|
|
set_done.assert_called_once_with(redis_client, True)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|