143 lines
6.5 KiB
Python
143 lines
6.5 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timedelta, timezone
|
|
from types import SimpleNamespace
|
|
import unittest
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from mobilede_scraper.core.runtime_config import RuntimeMobileDeEnrichmentConfig
|
|
from mobilede_scraper.worker import tasks
|
|
from mobilede_scraper.worker.celery_app import MOBILEDE_ENRICH_IMAGES_TASK, beat_schedule, celery_app
|
|
from mobilede_scraper.worker.constants import MOBILEDE_IMAGES_QUEUE
|
|
|
|
|
|
class TestMobileDeEnrichment(unittest.TestCase):
|
|
def test_runtime_config_defaults_and_overrides(self) -> None:
|
|
defaults = RuntimeMobileDeEnrichmentConfig.from_dict(None)
|
|
self.assertTrue(defaults.enabled)
|
|
self.assertEqual(defaults.batch_size, 10)
|
|
self.assertEqual(defaults.max_existing_images, 1)
|
|
self.assertEqual(defaults.staleness_hours, 168)
|
|
|
|
configured = RuntimeMobileDeEnrichmentConfig.from_dict({
|
|
"enabled": False,
|
|
"batch_size": 5,
|
|
"max_existing_images": 2,
|
|
"staleness_hours": 24,
|
|
"delay_seconds": 0.25,
|
|
})
|
|
self.assertFalse(configured.enabled)
|
|
self.assertEqual(configured.batch_size, 5)
|
|
self.assertEqual(configured.max_existing_images, 2)
|
|
self.assertEqual(configured.staleness_hours, 24)
|
|
self.assertEqual(configured.delay_seconds, 0.25)
|
|
|
|
def test_celery_routes_and_schedules_enrichment_on_images_queue(self) -> None:
|
|
routes = celery_app.conf.task_routes
|
|
self.assertEqual(routes[MOBILEDE_ENRICH_IMAGES_TASK]["queue"], MOBILEDE_IMAGES_QUEUE)
|
|
schedule = beat_schedule["periodic-mobilede-enrich-images"]
|
|
self.assertEqual(schedule["task"], MOBILEDE_ENRICH_IMAGES_TASK)
|
|
self.assertEqual(schedule["options"]["queue"], MOBILEDE_IMAGES_QUEUE)
|
|
self.assertGreater(schedule["options"]["expires"], 0)
|
|
|
|
def test_task_uses_runtime_staleness_and_continues_after_failure(self) -> None:
|
|
enrichment = SimpleNamespace(
|
|
enabled=True,
|
|
batch_size=1,
|
|
max_existing_images=1,
|
|
staleness_hours=48,
|
|
delay_seconds=0.0,
|
|
)
|
|
runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
|
|
redis_client = MagicMock()
|
|
redis_client.set.return_value = True
|
|
redis_client.eval.return_value = 1
|
|
persistence = MagicMock()
|
|
persistence.get_active_cars_batch_for_image_enrich.return_value = [
|
|
(1, "mobile.de:111", "https://suchen.mobile.de/fahrzeuge/details.html?id=111", 0),
|
|
(2, "mobile.de:222", "https://suchen.mobile.de/fahrzeuge/details.html?id=222", 1),
|
|
]
|
|
scraper = MagicMock()
|
|
scraper.sync_detail.side_effect = [{"upsert": {"action": "updated"}}, RuntimeError("blocked")]
|
|
|
|
before = datetime.now(timezone.utc) - timedelta(hours=48, seconds=2)
|
|
with (
|
|
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
|
|
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
|
|
patch.object(tasks, "_get_redis", return_value=redis_client),
|
|
patch.object(tasks, "_get_persistence", return_value=persistence),
|
|
patch.object(tasks, "MobileDeScraper", return_value=scraper),
|
|
):
|
|
result = tasks.mobilede_enrich_images_batch_task.run()
|
|
after = datetime.now(timezone.utc) - timedelta(hours=48) + timedelta(seconds=2)
|
|
|
|
self.assertEqual(result["status"], "success")
|
|
self.assertEqual(result["candidates"], 2)
|
|
self.assertEqual(result["enriched"], 1)
|
|
self.assertEqual(result["failed"], 1)
|
|
selector_kwargs = persistence.get_active_cars_batch_for_image_enrich.call_args.kwargs
|
|
self.assertNotIn("limit", selector_kwargs)
|
|
self.assertEqual(selector_kwargs["max_existing_images"], 1)
|
|
self.assertGreaterEqual(selector_kwargs["stale_before"], before)
|
|
self.assertLessEqual(selector_kwargs["stale_before"], after)
|
|
self.assertEqual(scraper.sync_detail.call_count, 2)
|
|
self.assertGreaterEqual(redis_client.eval.call_count, 3)
|
|
|
|
def test_task_stops_when_enrichment_lock_ownership_is_lost(self) -> None:
|
|
enrichment = SimpleNamespace(
|
|
enabled=True,
|
|
batch_size=1,
|
|
max_existing_images=1,
|
|
staleness_hours=48,
|
|
delay_seconds=0.0,
|
|
)
|
|
runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
|
|
redis_client = MagicMock()
|
|
redis_client.set.return_value = True
|
|
redis_client.eval.side_effect = [0, 0]
|
|
persistence = MagicMock()
|
|
persistence.get_active_cars_batch_for_image_enrich.return_value = [
|
|
(1, "mobile.de:111", "https://suchen.mobile.de/fahrzeuge/details.html?id=111", 0),
|
|
]
|
|
scraper = MagicMock()
|
|
|
|
with (
|
|
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
|
|
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
|
|
patch.object(tasks, "_get_redis", return_value=redis_client),
|
|
patch.object(tasks, "_get_persistence", return_value=persistence),
|
|
patch.object(tasks, "MobileDeScraper", return_value=scraper),
|
|
):
|
|
result = tasks.mobilede_enrich_images_batch_task.run()
|
|
|
|
self.assertEqual(result["status"], "lock_lost")
|
|
self.assertEqual(result["enriched"], 0)
|
|
scraper.sync_detail.assert_not_called()
|
|
|
|
def test_task_skips_when_enrichment_lock_is_held(self) -> None:
|
|
enrichment = SimpleNamespace(
|
|
enabled=True,
|
|
batch_size=1,
|
|
max_existing_images=1,
|
|
staleness_hours=48,
|
|
delay_seconds=0.0,
|
|
)
|
|
runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
|
|
redis_client = MagicMock()
|
|
redis_client.set.return_value = False
|
|
|
|
with (
|
|
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
|
|
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
|
|
patch.object(tasks, "_get_redis", return_value=redis_client),
|
|
patch.object(tasks, "_get_persistence") as get_persistence,
|
|
):
|
|
result = tasks.mobilede_enrich_images_batch_task.run()
|
|
|
|
self.assertEqual(result["status"], "locked")
|
|
get_persistence.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|