a4c93a1df1
improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
92 lines
3.5 KiB
Python
92 lines
3.5 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from iaai_scraper.worker import tasks
|
|
|
|
|
|
class TestWorkerTaskLockHelpers(unittest.TestCase):
|
|
def test_acquire_lock_returns_true_on_success(self) -> None:
|
|
redis_client = MagicMock()
|
|
redis_client.set.return_value = True
|
|
|
|
acquired = tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)
|
|
|
|
self.assertTrue(acquired)
|
|
redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120)
|
|
|
|
def test_refresh_lock_if_owner_extends_ttl(self) -> None:
|
|
redis_client = MagicMock()
|
|
redis_client.eval.return_value = 1
|
|
|
|
refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)
|
|
|
|
self.assertTrue(refreshed)
|
|
redis_client.eval.assert_called_once()
|
|
|
|
def test_release_lock_if_owner_uses_owner_token(self) -> None:
|
|
redis_client = MagicMock()
|
|
|
|
tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token")
|
|
|
|
redis_client.eval.assert_called_once()
|
|
args = redis_client.eval.call_args[0]
|
|
self.assertEqual(args[1], 1)
|
|
self.assertEqual(args[2], "lock:key")
|
|
self.assertEqual(args[3], "owner-token")
|
|
|
|
def test_sync_listing_task_skips_when_lock_not_acquired(self) -> None:
|
|
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
|
patch.object(tasks, "_get_redis") as get_redis, \
|
|
patch.object(tasks, "_acquire_lock", return_value=False):
|
|
persistence = MagicMock()
|
|
get_persistence.return_value = persistence
|
|
get_redis.return_value = MagicMock()
|
|
|
|
tasks.sync_listing_task.push_request(id="task-123")
|
|
try:
|
|
result = tasks.sync_listing_task.run()
|
|
finally:
|
|
tasks.sync_listing_task.pop_request()
|
|
|
|
persistence.create_tables.assert_called_once()
|
|
self.assertEqual(result["status"], "skipped")
|
|
self.assertEqual(result["reason"], "sync_already_running")
|
|
|
|
def test_sync_listing_task_releases_owned_lock(self) -> None:
|
|
with patch.object(tasks, "_get_persistence") as get_persistence, \
|
|
patch.object(tasks, "_get_redis") as get_redis, \
|
|
patch.object(tasks, "_acquire_lock", return_value=True), \
|
|
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
|
|
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
|
|
patch.object(tasks, "_run_browser_job", return_value={
|
|
"run_id": 7,
|
|
"cars_upserted": 2,
|
|
"cars_failed": 0,
|
|
"images_upserted": 4,
|
|
"skipped_existing": 1,
|
|
"elapsed_seconds": 1.25,
|
|
}):
|
|
persistence = MagicMock()
|
|
get_persistence.return_value = persistence
|
|
redis_client = MagicMock()
|
|
get_redis.return_value = redis_client
|
|
stop_event = MagicMock()
|
|
heartbeat_thread = MagicMock()
|
|
start_heartbeat.return_value = (stop_event, heartbeat_thread)
|
|
|
|
tasks.sync_listing_task.push_request(id="task-123")
|
|
try:
|
|
result = tasks.sync_listing_task.run(make="Toyota")
|
|
finally:
|
|
tasks.sync_listing_task.pop_request()
|
|
|
|
self.assertEqual(result["status"], "success")
|
|
stop_event.set.assert_called_once()
|
|
heartbeat_thread.join.assert_called_once()
|
|
release_lock.assert_called_once()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main() |