From 3c3aea8eb39f9f91b4413b7e2ca5256062b315f0 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Wed, 15 Apr 2026 21:58:10 +0300 Subject: [PATCH] tune vps config and update tests --- .env.example | 57 +++++++++ .gitignore | 3 +- tests/test_listing.py | 24 ++-- tests/test_mappers.py | 24 ++-- tests/test_parser.py | 24 +--- tests/test_scraper.py | 238 +++++++++++++++++-------------------- tests/test_utils.py | 62 ++++++++-- tests/test_worker_tasks.py | 109 +++++++---------- 8 files changed, 282 insertions(+), 259 deletions(-) create mode 100644 .env.example diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..e7c6f52 --- /dev/null +++ b/.env.example @@ -0,0 +1,57 @@ +IAAI_HEADLESS=true +IAAI_LOG_LEVEL=INFO + +IAAI_CAPTURE_SAME_ORIGIN_ONLY=true +IAAI_MAX_CAPTURED_REQUESTS=40 +IAAI_MAX_CAPTURED_JSON_RESPONSES=20 + +IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars +IAAI_LISTING_SEGMENTS=auto +IAAI_MAX_PAGES_PER_RUN=999999 +IAAI_MAX_VEHICLES_PER_RUN=999999 +IAAI_PAGE_LINK_LIMIT=999999 +IAAI_INCLUDE_PAGINATION=true +IAAI_COLLECT_CURRENT_PAGE_ONLY=false + +IAAI_HUMAN_PACE_ENABLED=true +IAAI_PARALLEL_TABS=10 +CELERY_BATCH_SIZE=100 +IAAI_AFTER_LISTING_OPEN_MIN_S=0.2 +IAAI_AFTER_LISTING_OPEN_MAX_S=0.5 +IAAI_AFTER_FILTER_ACTION_MIN_S=0.2 +IAAI_AFTER_FILTER_ACTION_MAX_S=0.5 +IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02 +IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08 +IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02 +IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08 +IAAI_BETWEEN_VEHICLES_MIN_S=0.02 +IAAI_BETWEEN_VEHICLES_MAX_S=0.08 +IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2 +IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5 + +IAAI_SYNC_ONLY_NEW=false +IAAI_TOKENS_FILE=/data/tokens.json +IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json + +IAAI_MAX_RETRIES=5 +IAAI_RETRY_DELAY_SECONDS=4 +IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 +IAAI_RETRY_JITTER_SECONDS=0.5 +IAAI_TIMEOUT_MS=90000 +IAAI_FALLBACK_NAV_TIMEOUT_MS=30000 + +IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper +IAAI_DATABASE_ECHO=false +IAAI_DATABASE_POOL_SIZE=5 +IAAI_DATABASE_MAX_OVERFLOW=5 + +IAAI_REDIS_URL=redis://redis:6379/0 + +CELERY_BROKER_URL=redis://redis:6379/0 +CELERY_RESULT_BACKEND=redis://redis:6379/0 +CELERY_TASK_SOFT_TIME_LIMIT=86400 +CELERY_TASK_TIME_LIMIT=86520 +CELERY_BROKER_VISIBILITY_TIMEOUT=90000 +CELERY_WORKER_CONCURRENCY=1 +CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 +CELERY_BEAT_SYNC_LIMIT=0 diff --git a/.gitignore b/.gitignore index b628404..0c27e35 100644 --- a/.gitignore +++ b/.gitignore @@ -3,8 +3,7 @@ __pycache__/ .coverage coverage.xml .env* -!.env -.env.example +!.env.example .venv/ venv/ *.pyc diff --git a/tests/test_listing.py b/tests/test_listing.py index 106ca31..ecc2ddb 100644 --- a/tests/test_listing.py +++ b/tests/test_listing.py @@ -30,23 +30,19 @@ class _FakePage: class TestListingUnit(unittest.TestCase): - def test_has_next_page_true_for_known_selector(self) -> None: - page = _FakePage({"a[aria-label*='Next']": 1}) - self.assertTrue(ListingCollector._has_next_page(page)) - - def test_has_next_page_false_when_no_selectors(self) -> None: - page = _FakePage({}) - self.assertFalse(ListingCollector._has_next_page(page)) - - def test_has_next_page_true_for_numeric_pagination_fallback(self) -> None: + def test_pagination_detection_and_page_number(self) -> None: + # Есть кнопка Next → True. + self.assertTrue(ListingCollector._has_next_page(_FakePage({"a[aria-label*='Next']": 1}))) + # Нет селекторов → False. + self.assertFalse(ListingCollector._has_next_page(_FakePage({}))) + # Числовая пагинация через JS → True. page = _FakePage({}) page._evaluate_result = True self.assertTrue(ListingCollector._has_next_page(page)) - - def test_get_current_page_number_from_text_counter(self) -> None: - page = _FakePage({}) - page._evaluate_values = [5] - self.assertEqual(ListingCollector._get_current_page_number(page), 5) + # Номер текущей страницы. + page2 = _FakePage({}) + page2._evaluate_values = [5] + self.assertEqual(ListingCollector._get_current_page_number(page2), 5) def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None: collector = ListingCollector(Settings(), HumanPacer(Settings())) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 4ac34be..621a5fc 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -41,30 +41,27 @@ class TestCarMapper(unittest.TestCase): self.assertFalse(record.is_damaged) - def test_unknown_and_empty_values_fallbacks(self) -> None: + def test_unknown_empty_values_and_normalization(self) -> None: record = self.mapper.map_to_car_record( vehicle_url="https://www.iaai.com/VehicleDetail/999~US", vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, ) - self.assertEqual(record.brand, "UNKNOWN") self.assertEqual(record.model, "UNKNOWN") - self.assertIsNone(record.steering_wheel if record.steering_wheel not in {"LEFT", None} else None) self.assertEqual(record.drive, "NA") self.assertEqual(record.gearbox, "NA") - def test_mapper_handles_case_and_spaces(self) -> None: - record = self.mapper.map_to_car_record( + # Нормализация регистра и пробелов. + record2 = self.mapper.map_to_car_record( vehicle_url="https://www.iaai.com/VehicleDetail/888~US", vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, ) + self.assertEqual(record2.drive, "FWD") + self.assertEqual(record2.gearbox, "AT") - self.assertEqual(record.drive, "FWD") - self.assertEqual(record.gearbox, "AT") - - def test_price_parsing_dirty_formats(self) -> None: + def test_price_and_currency_parsing(self) -> None: record = self.mapper.map_to_car_record( vehicle_url="https://www.iaai.com/VehicleDetail/777~US", vehicle_summary={"make": "Toyota", "model": "Corolla"}, @@ -76,11 +73,9 @@ class TestCarMapper(unittest.TestCase): "images": {}, }, ) - self.assertEqual(record.price, 5200) - def test_currency_detection_from_symbol(self) -> None: - record = self.mapper.map_to_car_record( + record2 = self.mapper.map_to_car_record( vehicle_url="https://www.iaai.com/VehicleDetail/778~US", vehicle_summary={"make": "Toyota", "model": "Corolla"}, payload_insights={ @@ -91,9 +86,8 @@ class TestCarMapper(unittest.TestCase): "images": {}, }, ) - - self.assertEqual(record.currency, "EUR") - self.assertEqual(record.price, 4500) + self.assertEqual(record2.currency, "EUR") + self.assertEqual(record2.price, 4500) if __name__ == "__main__": diff --git a/tests/test_parser.py b/tests/test_parser.py index 9884e17..72a2b6c 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -9,7 +9,7 @@ class TestVehicleParserUnit(unittest.TestCase): def setUp(self) -> None: self.parser = VehicleParser() - def test_parse_dom_key_value_pairs_extracts_known_fields(self) -> None: + def test_parse_dom_pairs_and_title(self) -> None: dom_text = """ Stock #: 45089484 @@ -23,41 +23,27 @@ class TestVehicleParserUnit(unittest.TestCase): self.assertEqual(result.get("primary_damage"), "Front End") self.assertEqual(result.get("odometer"), "50,123 mi (Actual)") - def test_parse_title_for_year_make_model(self) -> None: parsed = self.parser._parse_title_for_year_make_model("2014 TOYOTA CAMRY for sale", "") self.assertEqual(parsed["year"], "2014") self.assertEqual(parsed["make"], "TOYOTA") self.assertEqual(parsed["model"], "CAMRY") - def test_extract_image_urls_filters_other_vehicle(self) -> None: - vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" - payloads = [ - { - "imageUrls": [ - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", - ] - } - ] - urls = self.parser._extract_image_urls(payloads, "", vehicle_url) - self.assertEqual(len(urls), 1) - self.assertIn("45089484", urls[0]) - - def test_extract_image_urls_deduplicates_same_url(self) -> None: + def test_extract_image_urls_filters_and_deduplicates(self) -> None: vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" payloads = [{"imageUrls": [ "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", ]}] urls = self.parser._extract_image_urls(payloads, "", vehicle_url) self.assertEqual(len(urls), 1) + self.assertIn("45089484", urls[0]) - def test_dom_hints_detect_captcha_and_antibot(self) -> None: + def test_dom_hints_and_access_notes_detect_antibot(self) -> None: hints = self.parser._dom_hints("Please verify you are human. CAPTCHA. Incapsula access denied.") self.assertTrue(hints["has_captcha_text"]) self.assertTrue(hints["has_antibot_text"]) - def test_access_notes_reflect_antibot_signals(self) -> None: summary = {"vin": "", "image_urls": [], "note": "Incapsula access denied. Verify you are human."} notes = self.parser._build_access_notes(summary, []) self.assertTrue(notes["possible_captcha"]) diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 65bc6be..1381346 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -28,104 +28,129 @@ class TestScraperSync(unittest.TestCase): s.database.url = "sqlite://" return IAAIScraper(s) - def test_sync_vehicle_uses_db_record_without_remapping(self) -> None: + def test_sync_vehicle_uses_db_record(self) -> None: scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=1) scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) - scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) - scraper.car_mapper.map_to_car_record = MagicMock(side_effect=AssertionError("should not be called")) result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") self.assertEqual(result["status"], "success") self.assertIn("trace_id", result) - self.assertIn("elapsed_seconds", result) - self.assertEqual(scraper.persistence.upsert_car.call_count, 1) + scraper.persistence.upsert_car.assert_called_once() - def test_sync_listing_uses_db_record_without_remapping(self) -> None: + def test_only_new_routing_legacy_and_streaming(self) -> None: + # Legacy path: only_new без listing_url. scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=2) scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(set(), set())) - - scraper.collect_listing = MagicMock(return_value={"vehicle_urls": ["https://www.iaai.com/VehicleDetail/222~US"]}) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [], + scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( + {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"}, + )) + scraper.collect_listing = MagicMock(return_value={ + "vehicle_urls": [ + "https://www.iaai.com/VehicleDetail/111~US", + "https://www.iaai.com/VehicleDetail/222~US", + "https://www.iaai.com/VehicleDetail/333~US", + ] }) - - result = scraper.sync_listing() - + scraper.sync_batch = MagicMock(return_value={ + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], + }) + result = scraper.sync_listing(only_new=True) + self.assertEqual(result["skipped_existing"], 2) self.assertEqual(result["cars_upserted"], 1) - self.assertEqual(result["cars_failed"], 0) - self.assertIn("trace_id", result) - self.assertIn("elapsed_seconds", result) - scraper.sync_batch.assert_called_once() - def test_sync_listing_respects_limit(self) -> None: + # Streaming path: only_new + listing_url. + scraper2 = self._make_scraper() + scraper2.persistence.create_tables = MagicMock() + scraper2.persistence.start_sync_run = MagicMock(return_value=6) + scraper2.persistence.finish_sync_run = MagicMock() + scraper2.collect_listing = MagicMock(side_effect=AssertionError("legacy path should not be used")) + scraper2._sync_listing_streaming = MagicMock(return_value={ + "listing": {"vehicles_collected": 10, "early_stopped": False, "truncated_by_time_budget": False}, + "total": 10, "skipped_existing": 0, "cars_upserted": 10, "cars_failed": 0, + "images_upserted": 20, "failures": [], "all_listing_origin_urls": set(), + }) + result2 = scraper2.sync_listing( + only_new=True, + listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + year_min=2020, year_max=2027, + ) + self.assertEqual(result2["cars_upserted"], 10) + scraper2._sync_listing_streaming.assert_called_once() + scraper2.collect_listing.assert_not_called() + + def test_segmented_sync_calls_per_segment_and_resumes(self) -> None: scraper = self._make_scraper() - scraper.persistence.create_tables = MagicMock() scraper.persistence.start_sync_run = MagicMock(return_value=3) scraper.persistence.finish_sync_run = MagicMock() - # Проверка пути only_new с limit. - scraper._collect_listing_iterative = MagicMock(return_value=( - ["https://www.iaai.com/VehicleDetail/222~US"], - ["https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/VehicleDetail/333~US"], - {"vehicle_urls": [], "pages_collected": 1, "early_stopped": False}, - 0, - )) - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 1, "failures": [], - }) + call_args_log: list[dict] = [] + def _fake_sync_listing(**kwargs): + call_args_log.append(kwargs) + return { + "status": "success", "cars_upserted": 5, "cars_failed": 0, + "images_upserted": 10, "skipped_existing": 0, + "listing": {"vehicles_collected": 50}, "failures": [], + } - scraper.sync_listing(limit=1) + scraper.sync_listing = MagicMock(side_effect=_fake_sync_listing) + segments = [ + {"make": "TOYOTA", "year_min": 2020, "year_max": 2027}, + {"make": "FORD", "year_min": None, "year_max": None}, + {"make": "HONDA", "year_min": None, "year_max": None}, + ] - # Должен уйти только один URL. - scraper.sync_batch.assert_called_once() - batch_urls = scraper.sync_batch.call_args[0][0] - self.assertEqual(len(batch_urls), 1) + # Resume: пропускаем TOYOTA, начинаем с FORD на стр. 5. + result = scraper.sync_listing_segmented( + segments=segments, start_segment=1, start_page=5, + ) - def test_sync_listing_only_new_filters_existing_by_url_and_origin_id(self) -> None: - scraper = self._make_scraper() + self.assertEqual(result["segments_completed"], 2) # FORD + HONDA + self.assertEqual(result["cars_upserted"], 10) + # FORD: start_page=5, HONDA: start_page=1. + self.assertEqual(call_args_log[0]["start_page"], 5) + self.assertEqual(call_args_log[1]["start_page"], 1) + # URL содержит бренд. + self.assertIn("FORD", call_args_log[0]["listing_url"]) + self.assertIn("HONDA", call_args_log[1]["listing_url"]) - scraper.persistence.create_tables = MagicMock() - scraper.persistence.start_sync_run = MagicMock(return_value=5) - scraper.persistence.finish_sync_run = MagicMock() - scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( - {"https://www.iaai.com/VehicleDetail/111~US"}, - {"iaai:222"}, - )) + def test_build_segment_listing_url(self) -> None: + base = "https://www.iaai.com/Vehiclelisting/Cars" + self.assertEqual( + IAAIScraper._build_segment_listing_url(base, "TOYOTA"), + "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + ) + self.assertEqual( + IAAIScraper._build_segment_listing_url(base, "LAND ROVER"), + "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER", + ) + self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base) + self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base) - scraper.collect_listing = MagicMock(return_value={ - "vehicle_urls": [ - "https://www.iaai.com/VehicleDetail/111~US", # exists by URL - "https://www.iaai.com/VehicleDetail/222~US", # exists by ID - "https://www.iaai.com/VehicleDetail/333~US", # new - ] - }) - # Возвращаем результат для одного нового авто. - scraper.sync_batch = MagicMock(return_value={ - "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, "failures": [], - }) - - result = scraper.sync_listing(only_new=True) - - self.assertEqual(result["skipped_existing"], 2) - self.assertEqual(result["cars_upserted"], 1) - # В batch должен попасть только новый URL. - scraper.sync_batch.assert_called_once() - batch_urls = scraper.sync_batch.call_args[0][0] - self.assertEqual(len(batch_urls), 1) - self.assertIn("333", batch_urls[0]) - scraper.persistence.get_existing_urls_and_ids.assert_called_once() + def test_guard_and_protection_detection(self) -> None: + with self.assertRaises(AntiBotDetectedError): + IAAIScraper._raise_if_blocked_or_incomplete( + {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, + "access_notes": {}, "vehicle_summary": {}}, + "https://www.iaai.com/VehicleDetail/999~US", + ) + with self.assertRaises(SiteStructureChangedError): + IAAIScraper._raise_if_blocked_or_incomplete( + {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, + "access_notes": {"possible_captcha": False, "possible_antibot": False}, + "vehicle_summary": {}}, + "https://www.iaai.com/VehicleDetail/999~US", + ) + self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) + self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) + self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) def test_close_resets_browser_state(self) -> None: scraper = self._make_scraper() @@ -134,85 +159,34 @@ class TestScraperSync(unittest.TestCase): scraper.context = MagicMock() scraper.browser = MagicMock() scraper.playwright = MagicMock() - scraper.close() - http_pool.clear.assert_called_once() - self.assertIsNone(scraper._http_pool) self.assertIsNone(scraper.context) self.assertIsNone(scraper.browser) - self.assertIsNone(scraper.playwright) - def test_guard_raises_on_antibot_signals(self) -> None: - with self.assertRaises(AntiBotDetectedError): - IAAIScraper._raise_if_blocked_or_incomplete( - { - "dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, - "access_notes": {}, - "vehicle_summary": {}, - }, - "https://www.iaai.com/VehicleDetail/999~US", - ) - - def test_guard_raises_on_empty_vehicle_page(self) -> None: - with self.assertRaises(SiteStructureChangedError): - IAAIScraper._raise_if_blocked_or_incomplete( - { - "dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, - "access_notes": {"possible_captcha": False, "possible_antibot": False}, - "vehicle_summary": {}, - }, - "https://www.iaai.com/VehicleDetail/999~US", - ) - - def test_is_protection_or_network_error_detects_known_signals(self) -> None: - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) - self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) - - def test_recover_empty_listing_page_reload_recovers_links(self) -> None: + def test_recover_empty_listing_page(self) -> None: scraper = self._make_scraper() page = MagicMock() page_result = SimpleNamespace( vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")], ) scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) - - all_raw_urls: list[str] = [] - seen_urls: set[str] = set() - - recovered_result, recovered_urls = scraper._recover_empty_listing_page( - page, - page_number=5, - all_raw_urls=all_raw_urls, - seen_urls=seen_urls, - ) - - page.reload.assert_called_once() - self.assertIs(recovered_result, page_result) - self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/123~US"]) - self.assertEqual(all_raw_urls, ["https://www.iaai.com/VehicleDetail/123~US"]) - - def test_recover_empty_listing_page_reopens_listing_for_first_page(self) -> None: - scraper = self._make_scraper() - page = MagicMock() - page_result = SimpleNamespace( - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/456~US")], - ) scraper.listing_collector.open_cars_listing = MagicMock() - scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) - recovered_result, recovered_urls = scraper._recover_empty_listing_page( - page, - page_number=1, - all_raw_urls=[], - seen_urls=set(), + # Страница > 1: reload. + _, urls = scraper._recover_empty_listing_page( + page, page_number=5, all_raw_urls=[], seen_urls=set(), ) + page.reload.assert_called_once() + self.assertEqual(len(urls), 1) - scraper.listing_collector.open_cars_listing.assert_called_once_with(page) + # Страница 1: переоткрытие листинга. + page.reset_mock() + _, urls = scraper._recover_empty_listing_page( + page, page_number=1, all_raw_urls=[], seen_urls=set(), + ) + scraper.listing_collector.open_cars_listing.assert_called_once() page.reload.assert_not_called() - self.assertIs(recovered_result, page_result) - self.assertEqual(recovered_urls, ["https://www.iaai.com/VehicleDetail/456~US"]) if __name__ == "__main__": diff --git a/tests/test_utils.py b/tests/test_utils.py index fff40fa..2927c1e 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -3,28 +3,72 @@ from __future__ import annotations import unittest from iaai_scraper.core.utils import deep_find_key +from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS class TestDeepFindKey(unittest.TestCase): - def test_finds_key_in_nested_structure(self) -> None: + def test_finds_nested_and_respects_depth(self) -> None: payload = { "root": { "target": "a", "nested": [{"target": "b"}, {"x": 1}], } } + self.assertEqual(deep_find_key(payload, {"target"}), ["a", "b"]) - found = deep_find_key(payload, {"target"}) - self.assertEqual(found, ["a", "b"]) + deep_payload = {"l1": {"l2": {"l3": {"target": "value"}}}} + self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=2), []) + self.assertEqual(deep_find_key(deep_payload, {"target"}, max_depth=8), ["value"]) - def test_respects_max_depth(self) -> None: - payload = {"l1": {"l2": {"l3": {"target": "value"}}}} - found_too_shallow = deep_find_key(payload, {"target"}, max_depth=2) - found_enough_depth = deep_find_key(payload, {"target"}, max_depth=8) +class TestParseListingSegments(unittest.TestCase): + def test_empty_and_invalid_return_empty(self) -> None: + self.assertEqual(parse_listing_segments(""), []) + self.assertEqual(parse_listing_segments(" "), []) + self.assertEqual(parse_listing_segments("invalid"), []) - self.assertEqual(found_too_shallow, []) - self.assertEqual(found_enough_depth, ["value"]) + def test_auto_segments_complete_coverage(self) -> None: + """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр.""" + segs = parse_listing_segments("auto") + + # Точное число сегментов. + expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES)) + self.assertEqual(len(segs), expected) + + # Все бренды из списка присутствуют. + makes_in_segments = {s["make"] for s in segs} + for make in IAAI_DEFAULT_MAKES: + self.assertIn(make, makes_in_segments) + + # Крупные бренды разбиты на 3 сегмента с годами. + toyota = [s for s in segs if s["make"] == "TOYOTA"] + self.assertEqual(len(toyota), 3) + for s in toyota: + self.assertIsNotNone(s["year_min"]) + + # Мелкие бренды без годов. + lexus = [s for s in segs if s["make"] == "LEXUS"] + self.assertEqual(len(lexus), 1) + self.assertIsNone(lexus[0]["year_min"]) + + # Годовые диапазоны покрывают 1950-2027. + years = set() + for yr_min, yr_max in _YEAR_SPLITS: + years.update(range(yr_min, yr_max + 1)) + for year in range(1950, 2027): + self.assertIn(year, years) + + def test_json_input_formats(self) -> None: + # Массив строк. + segs = parse_listing_segments('["toyota", "ford"]') + self.assertEqual(len(segs), 2) + self.assertEqual(segs[0]["make"], "TOYOTA") + + # Массив объектов с годами. + segs = parse_listing_segments('[{"make":"BMW","year_min":2020,"year_max":2025}]') + self.assertEqual(segs[0]["make"], "BMW") + self.assertEqual(segs[0]["year_min"], 2020) + self.assertEqual(segs[0]["year_max"], 2025) if __name__ == "__main__": diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py index 12dd2e4..f255d22 100644 --- a/tests/test_worker_tasks.py +++ b/tests/test_worker_tasks.py @@ -8,32 +8,22 @@ from iaai_scraper.worker import tasks class TestWorkerTaskLockHelpers(unittest.TestCase): - def test_acquire_lock_returns_true_on_success(self) -> None: + def test_lock_acquire_refresh_release(self) -> None: redis_client = MagicMock() + + # Acquire. redis_client.set.return_value = True - - acquired = tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120) - - self.assertTrue(acquired) + self.assertTrue(tasks._acquire_lock(redis_client, "lock:key", "owner-token", 120)) redis_client.set.assert_called_once_with("lock:key", "owner-token", nx=True, ex=120) - def test_refresh_lock_if_owner_extends_ttl(self) -> None: - redis_client = MagicMock() + # Refresh. redis_client.eval.return_value = 1 + self.assertTrue(tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120)) - refreshed = tasks._refresh_lock_if_owner(redis_client, "lock:key", "owner-token", 120) - - self.assertTrue(refreshed) - redis_client.eval.assert_called_once() - - def test_release_lock_if_owner_uses_owner_token(self) -> None: - redis_client = MagicMock() - + # Release. + redis_client.eval.reset_mock() tasks._release_lock_if_owner(redis_client, "lock:key", "owner-token") - - redis_client.eval.assert_called_once() args = redis_client.eval.call_args[0] - self.assertEqual(args[1], 1) self.assertEqual(args[2], "lock:key") self.assertEqual(args[3], "owner-token") @@ -62,6 +52,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 7, "cars_upserted": 2, @@ -90,7 +81,8 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): heartbeat_thread.join.assert_called_once() release_lock.assert_called_once() - def test_clear_orphan_sync_listing_lock_deletes_when_no_tasks_running(self) -> None: + def test_clear_orphan_sync_listing_lock(self) -> None: + # Нет запущенных задач → удаляет. redis_client = MagicMock() redis_client.get.return_value = "owner-token" redis_client.ttl.return_value = 120 @@ -101,27 +93,21 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): inspector.scheduled.return_value = {"worker@node": []} celery_app.control.inspect.return_value = inspector - cleared = tasks._clear_orphan_sync_listing_lock(redis_client, celery_app) - - self.assertTrue(cleared) + self.assertTrue(tasks._clear_orphan_sync_listing_lock(redis_client, celery_app)) redis_client.delete.assert_called_once_with(tasks.SYNC_LISTING_LOCK_KEY) - def test_clear_orphan_sync_listing_lock_keeps_when_task_detected(self) -> None: - redis_client = MagicMock() - redis_client.get.return_value = "owner-token" - celery_app = MagicMock() - inspector = MagicMock() - inspector.active.return_value = { - "worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}] - } - inspector.reserved.return_value = {"worker@node": []} - inspector.scheduled.return_value = {"worker@node": []} - celery_app.control.inspect.return_value = inspector + # Задача активна → не удаляет. + redis_client2 = MagicMock() + redis_client2.get.return_value = "owner-token" + inspector2 = MagicMock() + inspector2.active.return_value = {"worker@node": [{"name": tasks.SYNC_LISTING_TASK_NAME}]} + inspector2.reserved.return_value = {"worker@node": []} + inspector2.scheduled.return_value = {"worker@node": []} + celery_app2 = MagicMock() + celery_app2.control.inspect.return_value = inspector2 - cleared = tasks._clear_orphan_sync_listing_lock(redis_client, celery_app) - - self.assertFalse(cleared) - redis_client.delete.assert_not_called() + self.assertFalse(tasks._clear_orphan_sync_listing_lock(redis_client2, celery_app2)) + redis_client2.delete.assert_not_called() def test_sync_listing_task_recovers_orphan_lock_and_runs(self) -> None: with patch.object(tasks, "_get_persistence") as get_persistence, \ @@ -131,6 +117,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks, "_run_browser_job", return_value={ "run_id": 9, "cars_upserted": 3, @@ -159,28 +146,22 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): self.assertEqual(acquire_lock.call_count, 2) release_lock.assert_called_once() - def test_sync_listing_checkpoint_roundtrip(self) -> None: + def test_sync_listing_checkpoint_save_load_and_resume(self) -> None: + # Roundtrip: save → load. redis_client = MagicMock() storage: dict[str, str] = {} redis_client.set.side_effect = lambda key, value: storage.__setitem__(key, value) redis_client.get.side_effect = lambda key: storage.get(key) tasks._save_sync_checkpoint( - redis_client, - task_id="task-1", - page_number=12, - make=None, - model=None, - lane="iaai_cars", + redis_client, task_id="task-1", page_number=12, + make=None, model=None, lane="iaai_cars", ) - checkpoint = tasks._load_sync_checkpoint(redis_client) - self.assertIsNotNone(checkpoint) self.assertEqual(checkpoint["last_successful_page"], 12) - self.assertEqual(checkpoint["status"], "in_progress") - def test_sync_listing_task_resumes_from_checkpoint_page(self) -> None: + # Resume: задача стартует со страницы checkpoint + 1. with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ patch.object(tasks, "_acquire_lock", return_value=True), \ @@ -188,32 +169,24 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ - patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()): - persistence = MagicMock() - get_persistence.return_value = persistence - redis_client = MagicMock() - redis_client.get.side_effect = lambda key: json.dumps({ - "status": "in_progress", - "last_successful_page": 9, - "make": None, - "model": None, - "lane": "iaai_cars", + patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ + patch.object(tasks.sync_listing_task, "update_state"), \ + patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]): + get_persistence.return_value = MagicMock() + redis_client2 = MagicMock() + redis_client2.get.side_effect = lambda key: json.dumps({ + "status": "in_progress", "last_successful_page": 9, + "make": None, "model": None, "lane": "iaai_cars", }) if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None - get_redis.return_value = redis_client + get_redis.return_value = redis_client2 stop_event = MagicMock() heartbeat_thread = MagicMock() start_heartbeat.return_value = (stop_event, heartbeat_thread) sync_listing_mock = MagicMock(return_value={ - "run_id": 11, - "status": "success", - "full_scan_completed": True, - "cars_upserted": 1, - "cars_failed": 0, - "images_upserted": 0, - "skipped_existing": 0, - "elapsed_seconds": 1.0, - "failures": [], + "run_id": 11, "status": "success", "full_scan_completed": True, + "cars_upserted": 1, "cars_failed": 0, "images_upserted": 0, + "skipped_existing": 0, "elapsed_seconds": 1.0, "failures": [], }) scraper_ctx = MagicMock() scraper_ctx.__enter__.return_value.sync_listing = sync_listing_mock