diff --git a/iaai_scraper/scraper.py b/iaai_scraper/scraper.py index 6036bf6..96fb399 100644 --- a/iaai_scraper/scraper.py +++ b/iaai_scraper/scraper.py @@ -10,6 +10,7 @@ from playwright.sync_api import TimeoutError as PlaywrightTimeoutError from .browser import BrowserFactory, HumanPacer, NetworkCapture from .core.config import Settings, settings +from .core.exceptions import AntiBotDetectedError, SiteStructureChangedError from .core.logs import set_trace_id, setup_logging from .core.retry import retryable from .core.runtime_config import RuntimeConfig @@ -26,6 +27,22 @@ VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) class IAAIScraper: + @staticmethod + def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None: + dom_hints = parsed.get("dom_hints", {}) or {} + access_notes = parsed.get("access_notes", {}) or {} + summary = parsed.get("vehicle_summary", {}) or {} + + if dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text"): + raise AntiBotDetectedError(f"IAAI anti-bot detected for {vehicle_url}") + + if access_notes.get("possible_captcha") or access_notes.get("possible_antibot"): + raise AntiBotDetectedError(f"IAAI blocked or challenged request for {vehicle_url}") + + has_identity = bool(summary.get("lot_number") or summary.get("vin") or summary.get("make") or summary.get("model")) + if not has_identity: + raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") + @staticmethod def _extract_origin_id_from_url(vehicle_url: str) -> str | None: match = VEHICLE_ID_RE.search(vehicle_url) @@ -152,6 +169,7 @@ class IAAIScraper: dom_text = "" network_dump = capture.export() parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump) + self._raise_if_blocked_or_incomplete(parsed, vehicle_url) db_record = self.car_mapper.map_to_car_record( vehicle_url=vehicle_url, @@ -255,8 +273,6 @@ class IAAIScraper: try: listing = self.collect_listing(make=make, model=model) vehicle_urls = list(listing.get("vehicle_urls", [])) - if limit is not None: - vehicle_urls = vehicle_urls[:max(0, limit)] effective_only_new = self.settings.sync_only_new if only_new is None else only_new if effective_only_new: @@ -279,6 +295,9 @@ class IAAIScraper: logger.info("Filtering already known vehicles: skipped %d", skipped_existing) vehicle_urls = [url for url in vehicle_urls if url not in known_urls] + if limit is not None: + vehicle_urls = vehicle_urls[:max(0, limit)] + total = len(vehicle_urls) logger.info("Starting sync: %d vehicles to process", total) @@ -294,6 +313,7 @@ class IAAIScraper: # Применяем фильтры из runtime_config (include/exclude/price/mileage/flags) if not self.runtime_config.filters.is_empty(): + vehicle_summary = scrape_result.get("vehicle_summary", {}) or {} filter_values = { "brand": record.brand, "model": record.model, @@ -302,9 +322,11 @@ class IAAIScraper: "color": record.color, "drive": record.drive, "gearbox": record.gearbox, + "location": vehicle_summary.get("location"), "price": record.price, "mileage": record.mileage, "is_damaged": record.is_damaged, + "run_and_drive": vehicle_summary.get("run_and_drive"), } if not self.runtime_config.filters.matches(filter_values): logger.info(