from __future__ import annotations import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch from dubizzle_scraper.scraper import DUBIZZLEScraper from dubizzle_scraper.core.config import Settings from dubizzle_scraper.worker import tasks class TestResilience(unittest.TestCase): def _make_scraper(self) -> DUBIZZLEScraper: s = Settings() s.log_level = "CRITICAL" s.database.url = "sqlite://" return DUBIZZLEScraper(s) def test_update_task_progress_updates_global_marker(self) -> None: redis_client = MagicMock() pipe = MagicMock() redis_client.pipeline.return_value = pipe tasks._update_task_progress( redis_client, task_id="task-abc", stage="batch_upserted", ttl_seconds=180, cars_upserted=10, ) redis_client.pipeline.assert_called_once() self.assertEqual(pipe.set.call_count, 2) first_call = pipe.set.call_args_list[0] second_call = pipe.set.call_args_list[1] self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc")) self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY) pipe.execute.assert_called_once() def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None: scraper = self._make_scraper() scraper.settings.celery.batch_size = 50 page = MagicMock() empty_page_result = SimpleNamespace( page_number=1, vehicle_links=[], next_page_detected=True, ) scraper._open_listing_for_stream = MagicMock(return_value=( page, {"make": None, "model": None, "year_min": None, "year_max": None}, )) scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result) scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, [])) scraper.sync_batch = MagicMock() result = scraper._sync_listing_streaming( make=None, model=None, lane="dubizzle_cars", limit=None, effective_only_new=False, started_at=0.0, listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST", ) self.assertEqual(result["total"], 0) self.assertEqual(result["cars_upserted"], 0) self.assertEqual(result["cars_failed"], 0) self.assertEqual(result["listing"]["pages_collected"], 1) scraper._recover_empty_listing_page.assert_called_once() scraper.sync_batch.assert_not_called() if __name__ == "__main__": unittest.main()