qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Queue gallery enrichment jobs

Browse Source
This commit is contained in:
qananasikq
2026-08-19 12:06:00 +03:00
parent 33b85ccecd
commit 2913199d1e
8 changed files with 487 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_mobilede_enrichment.py
+25 -37
View File
@@ -1,6 +1,5 @@
from __future__ import annotations
from datetime import datetime, timedelta, timezone
from types import SimpleNamespace
import unittest
from unittest.mock import MagicMock, patch
@@ -40,50 +39,45 @@ class TestMobileDeEnrichment(unittest.TestCase):
self.assertEqual(schedule["options"]["queue"], MOBILEDE_IMAGES_QUEUE)
self.assertGreater(schedule["options"]["expires"], 0)
def test_task_uses_runtime_staleness_and_continues_after_failure(self) -> None:
def test_task_queues_only_never_fetched_candidates_by_default(self) -> None:
enrichment = SimpleNamespace(
enabled=True,
batch_size=1,
batch_size=2,
max_existing_images=1,
staleness_hours=48,
delay_seconds=0.0,
)
runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
redis_client = MagicMock()
redis_client.set.return_value = True
redis_client.eval.return_value = 1
redis_client.scard.return_value = 0
persistence = MagicMock()
persistence.get_active_cars_batch_for_image_enrich.return_value = [
(1, "mobile.de:111", "https://suchen.mobile.de/fahrzeuge/details.html?id=111", 0),
(2, "mobile.de:222", "https://suchen.mobile.de/fahrzeuge/details.html?id=222", 1),
(1, "mobile.de:111", "https://suchen.mobile.de/fahrzeuge/details.html?id=111", 3),
(2, "mobile.de:222", "https://suchen.mobile.de/fahrzeuge/details.html?id=222", 0),
]
scraper = MagicMock()
scraper.sync_detail.side_effect = [{"upsert": {"action": "updated"}}, RuntimeError("blocked")]
before = datetime.now(timezone.utc) - timedelta(hours=48, seconds=2)
with (
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
patch.object(tasks, "_get_redis", return_value=redis_client),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks, "MobileDeScraper", return_value=scraper),
patch.object(
tasks,
"_queue_mobilede_detail_listing_ids",
return_value={"queued": 2, "duplicate": 0, "full": 0},
) as queue_details,
):
result = tasks.mobilede_enrich_images_batch_task.run()
after = datetime.now(timezone.utc) - timedelta(hours=48) + timedelta(seconds=2)
self.assertEqual(result["status"], "success")
self.assertEqual(result["candidates"], 2)
self.assertEqual(result["enriched"], 1)
self.assertEqual(result["failed"], 1)
self.assertEqual(result["queued"], 2)
selector_kwargs = persistence.get_active_cars_batch_for_image_enrich.call_args.kwargs
self.assertNotIn("limit", selector_kwargs)
self.assertEqual(selector_kwargs["limit"], 2)
self.assertEqual(selector_kwargs["max_existing_images"], 1)
self.assertGreaterEqual(selector_kwargs["stale_before"], before)
self.assertLessEqual(selector_kwargs["stale_before"], after)
self.assertEqual(scraper.sync_detail.call_count, 2)
self.assertGreaterEqual(redis_client.eval.call_count, 3)
self.assertIsNone(selector_kwargs["stale_before"])
queue_details.assert_called_once()
def test_task_stops_when_enrichment_lock_ownership_is_lost(self) -> None:
def test_task_stops_when_pending_buffer_is_full(self) -> None:
enrichment = SimpleNamespace(
enabled=True,
batch_size=1,
@@ -93,48 +87,42 @@ class TestMobileDeEnrichment(unittest.TestCase):
)
runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
redis_client = MagicMock()
redis_client.set.return_value = True
redis_client.eval.side_effect = [0, 0]
redis_client.scard.return_value = 750
persistence = MagicMock()
persistence.get_active_cars_batch_for_image_enrich.return_value = [
(1, "mobile.de:111", "https://suchen.mobile.de/fahrzeuge/details.html?id=111", 0),
]
scraper = MagicMock()
with (
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
patch.object(tasks, "_get_redis", return_value=redis_client),
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks, "MobileDeScraper", return_value=scraper),
patch.dict("os.environ", {"MOBILEDE_DETAIL_QUEUE_MAX_PENDING": "750"}),
):
result = tasks.mobilede_enrich_images_batch_task.run()
self.assertEqual(result["status"], "lock_lost")
self.assertEqual(result["enriched"], 0)
scraper.sync_detail.assert_not_called()
self.assertEqual(result["status"], "full")
self.assertEqual(result["pending"], 750)
persistence.get_active_cars_batch_for_image_enrich.assert_not_called()
def test_task_skips_when_enrichment_lock_is_held(self) -> None:
def test_task_skips_when_enrichment_is_disabled(self) -> None:
enrichment = SimpleNamespace(
enabled=True,
enabled=False,
batch_size=1,
max_existing_images=1,
staleness_hours=48,
delay_seconds=0.0,
)
runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
redis_client = MagicMock()
redis_client.set.return_value = False
with (
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
patch.object(tasks, "_get_redis", return_value=redis_client),
patch.object(tasks, "_get_redis") as get_redis,
patch.object(tasks, "_get_persistence") as get_persistence,
):
result = tasks.mobilede_enrich_images_batch_task.run()
self.assertEqual(result["status"], "locked")
self.assertEqual(result["status"], "disabled")
get_redis.assert_not_called()
get_persistence.assert_not_called()