131 lines
5.7 KiB
Python
131 lines
5.7 KiB
Python
from __future__ import annotations
|
|
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from mobilede_scraper.core.runtime_config import RuntimeMobileDeEnrichmentConfig
|
|
from mobilede_scraper.worker import tasks
|
|
from mobilede_scraper.worker.celery_app import MOBILEDE_ENRICH_IMAGES_TASK, beat_schedule, celery_app
|
|
from mobilede_scraper.worker.constants import MOBILEDE_IMAGES_QUEUE
|
|
|
|
|
|
class TestMobileDeEnrichment(unittest.TestCase):
|
|
def test_runtime_config_defaults_and_overrides(self) -> None:
|
|
defaults = RuntimeMobileDeEnrichmentConfig.from_dict(None)
|
|
self.assertTrue(defaults.enabled)
|
|
self.assertEqual(defaults.batch_size, 10)
|
|
self.assertEqual(defaults.max_existing_images, 1)
|
|
self.assertEqual(defaults.staleness_hours, 168)
|
|
|
|
configured = RuntimeMobileDeEnrichmentConfig.from_dict({
|
|
"enabled": False,
|
|
"batch_size": 5,
|
|
"max_existing_images": 2,
|
|
"staleness_hours": 24,
|
|
"delay_seconds": 0.25,
|
|
})
|
|
self.assertFalse(configured.enabled)
|
|
self.assertEqual(configured.batch_size, 5)
|
|
self.assertEqual(configured.max_existing_images, 2)
|
|
self.assertEqual(configured.staleness_hours, 24)
|
|
self.assertEqual(configured.delay_seconds, 0.25)
|
|
|
|
def test_celery_routes_and_schedules_enrichment_on_images_queue(self) -> None:
|
|
routes = celery_app.conf.task_routes
|
|
self.assertEqual(routes[MOBILEDE_ENRICH_IMAGES_TASK]["queue"], MOBILEDE_IMAGES_QUEUE)
|
|
schedule = beat_schedule["periodic-mobilede-enrich-images"]
|
|
self.assertEqual(schedule["task"], MOBILEDE_ENRICH_IMAGES_TASK)
|
|
self.assertEqual(schedule["options"]["queue"], MOBILEDE_IMAGES_QUEUE)
|
|
self.assertGreater(schedule["options"]["expires"], 0)
|
|
|
|
def test_task_queues_only_never_fetched_candidates_by_default(self) -> None:
|
|
enrichment = SimpleNamespace(
|
|
enabled=True,
|
|
batch_size=2,
|
|
max_existing_images=1,
|
|
staleness_hours=48,
|
|
delay_seconds=0.0,
|
|
)
|
|
runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
|
|
redis_client = MagicMock()
|
|
redis_client.scard.return_value = 0
|
|
persistence = MagicMock()
|
|
persistence.get_active_cars_batch_for_image_enrich.return_value = [
|
|
(1, "mobile.de:111", "https://suchen.mobile.de/fahrzeuge/details.html?id=111", 3),
|
|
(2, "mobile.de:222", "https://suchen.mobile.de/fahrzeuge/details.html?id=222", 0),
|
|
]
|
|
|
|
with (
|
|
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
|
|
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
|
|
patch.object(tasks, "_get_redis", return_value=redis_client),
|
|
patch.object(tasks, "_get_persistence", return_value=persistence),
|
|
patch.object(
|
|
tasks,
|
|
"_queue_mobilede_detail_listing_ids",
|
|
return_value={"queued": 2, "duplicate": 0, "full": 0},
|
|
) as queue_details,
|
|
):
|
|
result = tasks.mobilede_enrich_images_batch_task.run()
|
|
self.assertEqual(result["status"], "success")
|
|
self.assertEqual(result["candidates"], 2)
|
|
self.assertEqual(result["queued"], 2)
|
|
selector_kwargs = persistence.get_active_cars_batch_for_image_enrich.call_args.kwargs
|
|
self.assertEqual(selector_kwargs["limit"], 2)
|
|
self.assertEqual(selector_kwargs["max_existing_images"], 1)
|
|
self.assertIsNone(selector_kwargs["stale_before"])
|
|
queue_details.assert_called_once()
|
|
|
|
def test_task_stops_when_pending_buffer_is_full(self) -> None:
|
|
enrichment = SimpleNamespace(
|
|
enabled=True,
|
|
batch_size=1,
|
|
max_existing_images=1,
|
|
staleness_hours=48,
|
|
delay_seconds=0.0,
|
|
)
|
|
runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
|
|
redis_client = MagicMock()
|
|
redis_client.scard.return_value = 750
|
|
persistence = MagicMock()
|
|
|
|
with (
|
|
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
|
|
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
|
|
patch.object(tasks, "_get_redis", return_value=redis_client),
|
|
patch.object(tasks, "_get_persistence", return_value=persistence),
|
|
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_MAX_PENDING": "750"}),
|
|
):
|
|
result = tasks.mobilede_enrich_images_batch_task.run()
|
|
|
|
self.assertEqual(result["status"], "full")
|
|
self.assertEqual(result["pending"], 750)
|
|
persistence.get_active_cars_batch_for_image_enrich.assert_not_called()
|
|
|
|
def test_task_skips_when_enrichment_is_disabled(self) -> None:
|
|
enrichment = SimpleNamespace(
|
|
enabled=False,
|
|
batch_size=1,
|
|
max_existing_images=1,
|
|
staleness_hours=48,
|
|
delay_seconds=0.0,
|
|
)
|
|
runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
|
|
|
|
with (
|
|
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
|
|
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
|
|
patch.object(tasks, "_get_redis") as get_redis,
|
|
patch.object(tasks, "_get_persistence") as get_persistence,
|
|
):
|
|
result = tasks.mobilede_enrich_images_batch_task.run()
|
|
|
|
self.assertEqual(result["status"], "disabled")
|
|
get_redis.assert_not_called()
|
|
get_persistence.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|