qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix limit after only_new filter

Browse Source
This commit is contained in:
qananasikq
2026-04-10 15:10:38 +03:00
parent 6d1702faae
commit 7b1501008e
1 changed files with 24 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
iaai_scraper/scraper.py
View File

@@ -10,6 +10,7 @@ from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
from .browser import BrowserFactory, HumanPacer, NetworkCapture
from .core.config import Settings, settings
from .core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from .core.logs import set_trace_id, setup_logging
from .core.retry import retryable
from .core.runtime_config import RuntimeConfig
@@ -26,6 +27,22 @@ VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
class IAAIScraper:
@staticmethod
def _raise_if_blocked_or_incomplete(parsed: dict, vehicle_url: str) -> None:
dom_hints = parsed.get("dom_hints", {}) or {}
access_notes = parsed.get("access_notes", {}) or {}
summary = parsed.get("vehicle_summary", {}) or {}
if dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text"):
raise AntiBotDetectedError(f"IAAI anti-bot detected for {vehicle_url}")
if access_notes.get("possible_captcha") or access_notes.get("possible_antibot"):
raise AntiBotDetectedError(f"IAAI blocked or challenged request for {vehicle_url}")
has_identity = bool(summary.get("lot_number") or summary.get("vin") or summary.get("make") or summary.get("model"))
if not has_identity:
raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}")
@staticmethod
def _extract_origin_id_from_url(vehicle_url: str) -> str | None:
match = VEHICLE_ID_RE.search(vehicle_url)
@@ -152,6 +169,7 @@ class IAAIScraper:
dom_text = ""
network_dump = capture.export()
parsed = self.vehicle_parser.normalize(vehicle_url, html, dom_text, network_dump)
self._raise_if_blocked_or_incomplete(parsed, vehicle_url)
db_record = self.car_mapper.map_to_car_record(
vehicle_url=vehicle_url,
@@ -255,8 +273,6 @@ class IAAIScraper:
try:
listing = self.collect_listing(make=make, model=model)
vehicle_urls = list(listing.get("vehicle_urls", []))
if limit is not None:
vehicle_urls = vehicle_urls[:max(0, limit)]
effective_only_new = self.settings.sync_only_new if only_new is None else only_new
if effective_only_new:
@@ -279,6 +295,9 @@ class IAAIScraper:
logger.info("Filtering already known vehicles: skipped %d", skipped_existing)
vehicle_urls = [url for url in vehicle_urls if url not in known_urls]
if limit is not None:
vehicle_urls = vehicle_urls[:max(0, limit)]
total = len(vehicle_urls)
logger.info("Starting sync: %d vehicles to process", total)
@@ -294,6 +313,7 @@ class IAAIScraper:
# Применяем фильтры из runtime_config (include/exclude/price/mileage/flags)
if not self.runtime_config.filters.is_empty():
vehicle_summary = scrape_result.get("vehicle_summary", {}) or {}
filter_values = {
"brand": record.brand,
"model": record.model,
@@ -302,9 +322,11 @@ class IAAIScraper:
"color": record.color,
"drive": record.drive,
"gearbox": record.gearbox,
"location": vehicle_summary.get("location"),
"price": record.price,
"mileage": record.mileage,
"is_damaged": record.is_damaged,
"run_and_drive": vehicle_summary.get("run_and_drive"),
}
if not self.runtime_config.filters.matches(filter_values):
logger.info(