qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update scraper package

Browse Source
This commit is contained in:
qananasikq
2026-04-24 20:46:58 +03:00
parent 6aba4f0dbd
commit d484088e18
59 changed files with 2548 additions and 1272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

151
tests/test_worker_tasks.py
View File

@@ -5,8 +5,8 @@ from dataclasses import replace
import unittest
from unittest.mock import MagicMock, patch
from iaai_scraper.worker import tasks
from iaai_scraper.core.config import settings as base_settings
from dubizzle_scraper.worker import tasks
from dubizzle_scraper.core.config import settings as base_settings
class TestWorkerTaskLockHelpers(unittest.TestCase):
@@ -187,13 +187,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(
base_settings.discovery,
always_full_scan=False,
mode="listing",
),
celery=replace(
base_settings.celery,
parallel_segments=False,
),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
@@ -211,7 +227,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-resume-seg")
try:
result = tasks.sync_listing_task.run()
@@ -228,6 +244,22 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(
base_settings.discovery,
always_full_scan=False,
mode="listing",
),
celery=replace(
base_settings.celery,
parallel_segments=False,
),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -235,7 +267,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
# Оставшийся чекпоинт не должен использоваться.
@@ -254,10 +286,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-792")
try:
result = tasks.sync_listing_task.run()
result = tasks.sync_listing_task.run(limit=1)
finally:
tasks.sync_listing_task.pop_request()
@@ -271,13 +303,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(
base_settings.discovery,
always_full_scan=False,
mode="listing",
),
celery=replace(
base_settings.celery,
parallel_segments=False,
),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
@@ -295,7 +343,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-beyond")
try:
result = tasks.sync_listing_task.run()
@@ -507,7 +555,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch.object(tasks, "_run_browser_job") as run_job, \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.return_value = None
@@ -554,7 +602,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
@@ -579,7 +627,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-always-full-resume")
try:
result = tasks.sync_listing_task.run()
@@ -591,6 +639,85 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
release_lock.assert_called_once()
def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None:
segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(base_settings.discovery, always_full_scan=True),
celery=replace(base_settings.celery, parallel_segments=True),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_reset_segments_progress") as reset_progress, \
patch.object(tasks, "_set_segmented_scan_active") as set_active, \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.smembers.return_value = {"0", "1"}
redis_client.get.return_value = None
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
tasks.sync_listing_task.push_request(id="task-always-hourly")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(result["segments_dispatched"], 2)
reset_progress.assert_called_once_with(redis_client, len(segments))
set_active.assert_called_once_with(redis_client, total=len(segments))
self.assertEqual(task_app.send_task.call_count, 2)
release_lock.assert_called_once()
def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None:
segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(base_settings.discovery, always_full_scan=True),
celery=replace(base_settings.celery, parallel_segments=True),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
get_redis.return_value = MagicMock()
start_heartbeat.return_value = (MagicMock(), MagicMock())
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
tasks.sync_listing_task.push_request(id="task-always-skip-duplicate")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "skipped")
self.assertEqual(result["reason"], "segmented_scan_in_progress")
task_app.send_task.assert_not_called()
release_lock.assert_called_once()
if __name__ == "__main__":
unittest.main()