Schedule stale gallery enrichment

2026-08-18 10:40:26 +03:00
parent 19007dafc8
commit ca1fab1034
15 changed files with 460 additions and 82 deletions
@@ -217,6 +217,39 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
        self.assertEqual(car.body_type, "COUPE")
        self.assertEqual([image.fullres_image for image in images], ["https://example.test/refreshed.jpg"])

+    def test_detail_timestamp_is_set_and_preserved_by_search_update(self) -> None:
+        search = self._record("mobile.de:detail-ts")
+        search.details_confirmed = False
+        search.images_confirmed = False
+        search.preserve_existing_details = True
+        self.persistence.upsert_car(search)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
+            self.assertIsNone(car.details_fetched_at)
+
+        detail = self._record("mobile.de:detail-ts", price=1500)
+        before = datetime.now(timezone.utc)
+        self.persistence.upsert_car(detail)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
+            fetched_at = self._as_utc(car.details_fetched_at)
+
+        self.assertIsNotNone(fetched_at)
+        self.assertGreaterEqual(fetched_at, before)
+
+        search_again = self._record("mobile.de:detail-ts", price=2000)
+        search_again.details_confirmed = False
+        search_again.images_confirmed = False
+        search_again.preserve_existing_details = True
+        self.persistence.upsert_car(search_again)
+
+        with self.persistence.session_scope() as session:
+            car = session.execute(select(Car).where(Car.origin_id == search.origin_id)).scalar_one()
+
+        self.assertEqual(self._as_utc(car.details_fetched_at), fetched_at)
+
    def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
        first_run_id = self.persistence.start_sync_run("lane-a")
        second_run_id = self.persistence.start_sync_run("lane-b")
@@ -328,6 +361,45 @@ class TestPersistenceServiceIntegration(unittest.TestCase):

        self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])

+    def test_image_enrich_selector_prefers_unfetched_then_oldest_stale(self) -> None:
+        unfetched = self._record("mobile.de:unfetched")
+        unfetched.details_confirmed = False
+        stale_old = self._record("mobile.de:stale-old")
+        stale_new = self._record("mobile.de:stale-new")
+        fresh = self._record("mobile.de:fresh")
+        for record in (unfetched, stale_old, stale_new, fresh):
+            record.images = []
+            self.persistence.upsert_car(record)
+
+        now = datetime.now(timezone.utc)
+        with self.persistence.session_scope() as session:
+            cars = {
+                car.origin_id: car
+                for car in session.execute(select(Car)).scalars().all()
+            }
+            cars[stale_old.origin_id].details_fetched_at = now - timedelta(days=10)
+            cars[stale_new.origin_id].details_fetched_at = now - timedelta(days=8)
+            cars[fresh.origin_id].details_fetched_at = now - timedelta(days=1)
+
+        candidates = self.persistence.get_active_cars_batch_for_image_enrich(
+            limit=10,
+            max_existing_images=1,
+            stale_before=now - timedelta(days=7),
+        )
+
+        self.assertEqual(
+            [origin_id for _id, origin_id, _url, _count in candidates],
+            [unfetched.origin_id, stale_old.origin_id, stale_new.origin_id],
+        )
+        self.assertEqual(
+            self.persistence.get_active_cars_batch_for_image_enrich(
+                limit=0,
+                max_existing_images=1,
+                stale_before=now,
+            ),
+            [],
+        )
+

 if __name__ == "__main__":
    unittest.main()
@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+from datetime import datetime, timedelta, timezone
+from types import SimpleNamespace
+import unittest
+from unittest.mock import MagicMock, patch
+
+from mobilede_scraper.core.runtime_config import RuntimeMobileDeEnrichmentConfig
+from mobilede_scraper.worker import tasks
+from mobilede_scraper.worker.celery_app import MOBILEDE_ENRICH_IMAGES_TASK, beat_schedule, celery_app
+from mobilede_scraper.worker.constants import MOBILEDE_IMAGES_QUEUE
+
+
+class TestMobileDeEnrichment(unittest.TestCase):
+    def test_runtime_config_defaults_and_overrides(self) -> None:
+        defaults = RuntimeMobileDeEnrichmentConfig.from_dict(None)
+        self.assertTrue(defaults.enabled)
+        self.assertEqual(defaults.batch_size, 10)
+        self.assertEqual(defaults.max_existing_images, 1)
+        self.assertEqual(defaults.staleness_hours, 168)
+
+        configured = RuntimeMobileDeEnrichmentConfig.from_dict({
+            "enabled": False,
+            "batch_size": 5,
+            "max_existing_images": 2,
+            "staleness_hours": 24,
+            "delay_seconds": 0.25,
+        })
+        self.assertFalse(configured.enabled)
+        self.assertEqual(configured.batch_size, 5)
+        self.assertEqual(configured.max_existing_images, 2)
+        self.assertEqual(configured.staleness_hours, 24)
+        self.assertEqual(configured.delay_seconds, 0.25)
+
+    def test_celery_routes_and_schedules_enrichment_on_images_queue(self) -> None:
+        routes = celery_app.conf.task_routes
+        self.assertEqual(routes[MOBILEDE_ENRICH_IMAGES_TASK]["queue"], MOBILEDE_IMAGES_QUEUE)
+        schedule = beat_schedule["periodic-mobilede-enrich-images"]
+        self.assertEqual(schedule["task"], MOBILEDE_ENRICH_IMAGES_TASK)
+        self.assertEqual(schedule["options"]["queue"], MOBILEDE_IMAGES_QUEUE)
+        self.assertGreater(schedule["options"]["expires"], 0)
+
+    def test_task_uses_runtime_staleness_and_continues_after_failure(self) -> None:
+        enrichment = SimpleNamespace(
+            enabled=True,
+            batch_size=1,
+            max_existing_images=1,
+            staleness_hours=48,
+            delay_seconds=0.0,
+        )
+        runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
+        redis_client = MagicMock()
+        redis_client.set.return_value = True
+        redis_client.eval.return_value = 1
+        persistence = MagicMock()
+        persistence.get_active_cars_batch_for_image_enrich.return_value = [
+            (1, "mobile.de:111", "https://suchen.mobile.de/fahrzeuge/details.html?id=111", 0),
+            (2, "mobile.de:222", "https://suchen.mobile.de/fahrzeuge/details.html?id=222", 1),
+        ]
+        scraper = MagicMock()
+        scraper.sync_detail.side_effect = [{"upsert": {"action": "updated"}}, RuntimeError("blocked")]
+
+        before = datetime.now(timezone.utc) - timedelta(hours=48, seconds=2)
+        with (
+            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
+            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
+            patch.object(tasks, "_get_redis", return_value=redis_client),
+            patch.object(tasks, "_get_persistence", return_value=persistence),
+            patch.object(tasks, "MobileDeScraper", return_value=scraper),
+        ):
+            result = tasks.mobilede_enrich_images_batch_task.run()
+        after = datetime.now(timezone.utc) - timedelta(hours=48) + timedelta(seconds=2)
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["candidates"], 2)
+        self.assertEqual(result["enriched"], 1)
+        self.assertEqual(result["failed"], 1)
+        selector_kwargs = persistence.get_active_cars_batch_for_image_enrich.call_args.kwargs
+        self.assertNotIn("limit", selector_kwargs)
+        self.assertEqual(selector_kwargs["max_existing_images"], 1)
+        self.assertGreaterEqual(selector_kwargs["stale_before"], before)
+        self.assertLessEqual(selector_kwargs["stale_before"], after)
+        self.assertEqual(scraper.sync_detail.call_count, 2)
+        redis_client.eval.assert_called()
+
+    def test_task_skips_when_enrichment_lock_is_held(self) -> None:
+        enrichment = SimpleNamespace(
+            enabled=True,
+            batch_size=1,
+            max_existing_images=1,
+            staleness_hours=48,
+            delay_seconds=0.0,
+        )
+        runtime_config = SimpleNamespace(mobilede=SimpleNamespace(enrichment=enrichment))
+        redis_client = MagicMock()
+        redis_client.set.return_value = False
+
+        with (
+            patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
+            patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
+            patch.object(tasks, "_get_redis", return_value=redis_client),
+            patch.object(tasks, "_get_persistence") as get_persistence,
+        ):
+            result = tasks.mobilede_enrich_images_batch_task.run()
+
+        self.assertEqual(result["status"], "locked")
+        get_persistence.assert_not_called()
+
+
+if __name__ == "__main__":
+    unittest.main()
@@ -5,7 +5,7 @@ from types import SimpleNamespace
 import unittest
 from unittest.mock import MagicMock, patch

-from mobilede_scraper.worker import tasks
+from mobilede_scraper.worker import planner, tasks


 class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
@@ -553,7 +553,13 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
        previous = os.environ.get("MOBILEDE_FILTERED_URL_FAST_START")
        try:
            os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
-            segments = tasks._build_mobilede_runtime_segments(settings)
+            with (
+                patch.object(planner, "_mobilede_try_keep_root_segment_unsplit", return_value=None),
+                patch.object(planner, "_expand_mobilede_search_url_segment_by_probe", return_value=None),
+                patch.object(planner, "_mobilede_probe_total", return_value=None),
+                patch.object(planner, "_mobilede_preplan_runtime_segments", side_effect=lambda items: items),
+            ):
+                segments = tasks._build_mobilede_runtime_segments(settings)
        finally:
            if previous is None:
                os.environ.pop("MOBILEDE_FILTERED_URL_FAST_START", None)
@@ -578,7 +584,13 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
        try:
            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
            os.environ["MOBILEDE_FILTERED_URL_FAST_START"] = "false"
-            segments = tasks._build_mobilede_runtime_segments(settings)
+            with (
+                patch.object(planner, "_mobilede_try_keep_root_segment_unsplit", return_value=None),
+                patch.object(planner, "_expand_mobilede_search_url_segment_by_probe", return_value=None),
+                patch.object(planner, "_mobilede_probe_total", return_value=None),
+                patch.object(planner, "_mobilede_preplan_runtime_segments", side_effect=lambda items: items),
+            ):
+                segments = tasks._build_mobilede_runtime_segments(settings)
        finally:
            tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
            if previous_fast_start is None: