update scraper package

This commit is contained in:
qananasikq
2026-04-24 20:46:58 +03:00
parent 6aba4f0dbd
commit d484088e18
59 changed files with 2548 additions and 1272 deletions

View File

@@ -4,17 +4,17 @@ import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.core.config import Settings
from iaai_scraper.worker import tasks
from dubizzle_scraper.scraper import DUBIZZLEScraper
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.worker import tasks
class TestResilience(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper:
def _make_scraper(self) -> DUBIZZLEScraper:
s = Settings()
s.log_level = "CRITICAL"
s.database.url = "sqlite://"
return IAAIScraper(s)
return DUBIZZLEScraper(s)
def test_update_task_progress_updates_global_marker(self) -> None:
redis_client = MagicMock()
@@ -60,11 +60,11 @@ class TestResilience(unittest.TestCase):
result = scraper._sync_listing_streaming(
make=None,
model=None,
lane="iaai_cars",
lane="dubizzle_cars",
limit=None,
effective_only_new=False,
started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST",
)
self.assertEqual(result["total"], 0)