80 lines
2.3 KiB
Python
80 lines
2.3 KiB
Python
from __future__ import annotations
|
|
|
|
import unittest
|
|
from types import SimpleNamespace
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
from iaai_scraper.scraper import IAAIScraper
|
|
from iaai_scraper.core.config import Settings
|
|
from iaai_scraper.worker import tasks
|
|
|
|
|
|
class TestResilience(unittest.TestCase):
|
|
def _make_scraper(self) -> IAAIScraper:
|
|
s = Settings()
|
|
s.log_level = "CRITICAL"
|
|
s.database.url = "sqlite://"
|
|
return IAAIScraper(s)
|
|
|
|
def test_update_task_progress_updates_global_marker(self) -> None:
|
|
redis_client = MagicMock()
|
|
pipe = MagicMock()
|
|
redis_client.pipeline.return_value = pipe
|
|
|
|
tasks._update_task_progress(
|
|
redis_client,
|
|
task_id="task-abc",
|
|
stage="batch_upserted",
|
|
ttl_seconds=180,
|
|
cars_upserted=10,
|
|
)
|
|
|
|
redis_client.pipeline.assert_called_once()
|
|
self.assertEqual(pipe.set.call_count, 2)
|
|
first_call = pipe.set.call_args_list[0]
|
|
second_call = pipe.set.call_args_list[1]
|
|
|
|
self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc"))
|
|
self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY)
|
|
pipe.execute.assert_called_once()
|
|
|
|
def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None:
|
|
scraper = self._make_scraper()
|
|
scraper.settings.celery.batch_size = 50
|
|
|
|
page = MagicMock()
|
|
empty_page_result = SimpleNamespace(
|
|
page_number=1,
|
|
vehicle_links=[],
|
|
next_page_detected=True,
|
|
)
|
|
|
|
scraper._open_listing_for_stream = MagicMock(return_value=(
|
|
page,
|
|
{"make": None, "model": None, "year_min": None, "year_max": None},
|
|
))
|
|
scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result)
|
|
scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, []))
|
|
scraper.sync_batch = MagicMock()
|
|
|
|
result = scraper._sync_listing_streaming(
|
|
make=None,
|
|
model=None,
|
|
lane="iaai_cars",
|
|
limit=None,
|
|
effective_only_new=False,
|
|
started_at=0.0,
|
|
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
|
|
)
|
|
|
|
self.assertEqual(result["total"], 0)
|
|
self.assertEqual(result["cars_upserted"], 0)
|
|
self.assertEqual(result["cars_failed"], 0)
|
|
self.assertEqual(result["listing"]["pages_collected"], 1)
|
|
scraper._recover_empty_listing_page.assert_called_once()
|
|
scraper.sync_batch.assert_not_called()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|