qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fast parser

Browse Source
This commit is contained in:
qananasikq
2026-04-24 12:39:05 +03:00
parent 3a3222c7dc
commit fc1bea562a
19 changed files with 2482 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

222
iaai_scraper/scraper.py
View File

@@ -22,6 +22,7 @@ from playwright.sync_api import Error as PlaywrightError
from playwright.sync_api import BrowserContext, Page, sync_playwright
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
from .browser.fast_client import DETAIL_MARKER, IAAIFastClient, is_challenge_response, parse_product_details_vm
from .browser import BrowserFactory, HumanPacer, NetworkCapture
from .core.config import Settings, settings, parse_listing_segments
from .core.exceptions import AntiBotDetectedError, ListingResumeError, SiteStructureChangedError
@@ -29,6 +30,8 @@ from .core.logs import set_trace_id, setup_logging
from .core.retry import retryable
from .core.runtime_config import RuntimeConfig
from .core.utils import save_to_json
from .fast_sync import FastSyncEngine
from .parsing.fast_mapper import FastCarMapper
from .parsing.mapper import CarMapper
from .parsing.parser import VehicleParser
from .storage.db import PersistenceService
@@ -205,6 +208,8 @@ class IAAIScraper:
self.pacer = HumanPacer(self.settings)
self.listing_collector = ListingCollector(self.settings, self.pacer)
self.vehicle_parser = VehicleParser()
self.fast_client = IAAIFastClient(self.settings)
self.fast_mapper = FastCarMapper()
self.car_mapper = CarMapper()
self.persistence = PersistenceService(self.settings)
self._shutdown_requested = False
@@ -259,6 +264,8 @@ class IAAIScraper:
pass
def __enter__(self) -> "IAAIScraper":
if self.settings.scraping_profile.http_first and not self.settings.scraping_profile.browser_fallback_enabled:
return self
if self.playwright is None:
self.playwright = sync_playwright().start()
if self.browser is None:
@@ -549,6 +556,33 @@ class IAAIScraper:
known_origin_ids: set[str] | None = None,
max_duration_seconds: float | None = None,
):
try:
max_pages = self.settings.listing.max_pages_per_run
vehicles = list(self.fast_client.iter_listing_vehicles(make=make, max_pages=max_pages))
vehicle_urls = [f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}" for vehicle in vehicles]
if model:
model_key = model.casefold()
vehicle_urls = [url for url in vehicle_urls if model_key in url.casefold()]
if known_origin_ids:
filtered_urls = []
for url in vehicle_urls:
origin_id = self._extract_db_origin_id_from_url(url)
if origin_id and origin_id in known_origin_ids:
continue
filtered_urls.append(url)
vehicle_urls = filtered_urls
return {
"status": "ok",
"mode": "fast_hidden_payload",
"vehicle_urls": vehicle_urls,
"vehicles_collected": len(vehicle_urls),
"pages": [],
"early_stopped": False,
"truncated_by_time_budget": False,
}
except Exception as exc:
logger.warning("Fast listing collection failed, falling back to browser listing: %s", exc)
page = self._get_page_with_warmup()
try:
@@ -1179,12 +1213,47 @@ class IAAIScraper:
@retryable(max_attempts=3, jitter_seconds=0.25)
def scrape_vehicle_detail(self, vehicle_url: str):
try:
return self._scrape_vehicle_detail_fast(vehicle_url)
except Exception as exc:
logger.warning("Fast detail scrape failed for %s, falling back to browser: %s", vehicle_url, exc)
page = self._get_page()
try:
return self._scrape_on_page(page, vehicle_url)
finally:
page.close()
def _scrape_vehicle_detail_fast(self, vehicle_url: str) -> dict[str, Any]:
trace_id = self._new_trace_id("scrape-fast")
started_at = time.perf_counter()
origin_id = self._extract_origin_id_from_url(vehicle_url)
if not origin_id:
raise RuntimeError(f"Could not extract IAAI inventory id from URL: {vehicle_url}")
detail_payload = self.fast_client.fetch_vehicle_detail_payload(origin_id)
db_record = self.fast_mapper.map_payload_to_record(
detail_payload=detail_payload,
vehicle_url=self._normalize_vehicle_url(vehicle_url),
)
vehicle_summary = self.fast_mapper.payload_to_summary(detail_payload, vehicle_url)
if not (vehicle_summary.get("make") or vehicle_summary.get("lot_number")):
raise SiteStructureChangedError(f"ProductDetailsVM returned no vehicle identity for {vehicle_url}")
return {
"trace_id": trace_id,
"source_url": vehicle_url,
"fetched_at_epoch": int(time.time()),
"elapsed_seconds": round(time.perf_counter() - started_at, 3),
"network": {"requests": [], "json_responses": [], "capture_limits": {}},
"vehicle_summary": vehicle_summary,
"payload_insights": {"source": "ProductDetailsVM"},
"embedded_json": [detail_payload],
"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
"access_notes": {"mode": "fast_hidden_payload"},
"db_record": db_record.model_dump(mode="json"),
}
_JS_EXTRACT = """
() => {
try {
@@ -1404,6 +1473,14 @@ class IAAIScraper:
Использует json.JSONDecoder.raw_decode для быстрого поиска JSON
вместо посимвольного сканирования скобок.
"""
try:
payload = parse_product_details_vm(html)
db_record = IAAIScraper._fast_payload_to_js_data(payload)
if db_record:
return db_record
except Exception:
pass
search = "inventoryView"
pos = html.find(search)
if pos < 0:
@@ -1481,6 +1558,61 @@ class IAAIScraper:
"imageKeys": img_keys,
}
@staticmethod
def _fast_payload_to_js_data(payload: dict[str, Any]) -> dict[str, Any] | None:
inventory_view = payload.get("inventoryView") if isinstance(payload, dict) else None
if not isinstance(inventory_view, dict):
return None
attr = inventory_view.get("attributes")
if not isinstance(attr, dict):
return None
image_dimensions = inventory_view.get("imageDimensions")
image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
keys_container = image_dimensions.get("keys")
keys_container = keys_container if isinstance(keys_container, dict) else {}
image_keys = keys_container.get("$values")
image_keys = image_keys if isinstance(image_keys, list) else []
auction_info = payload.get("auctionInformation")
auction_info = auction_info if isinstance(auction_info, dict) else {}
bid = auction_info.get("biddingInformation")
bid = bid if isinstance(bid, dict) else {}
prebid = auction_info.get("prebidInformation")
prebid = prebid if isinstance(prebid, dict) else {}
return {
"ok": True,
"SalvageId": attr.get("SalvageId", ""),
"StockNumber": attr.get("Id") or attr.get("StockNumber", ""),
"Year": attr.get("Year", ""),
"Make": attr.get("Make", ""),
"Model": attr.get("Model", ""),
"Series": attr.get("Series", ""),
"BodyStyleName": attr.get("BodyStyleName", ""),
"Cylinders": attr.get("Cylinders", ""),
"DriveLineTypeDesc": attr.get("DriveLineTypeDesc", ""),
"EngineSize": (attr.get("EngineInformation") or attr.get("EngineSize") or "").strip(),
"FuelTypeCode": attr.get("FuelTypeCode", ""),
"Transmission": attr.get("Transmission", ""),
"ExteriorColor": attr.get("ExteriorColor", ""),
"PrimaryDamageDesc": attr.get("PrimaryDamageDesc", ""),
"SecondaryDamageDesc": attr.get("SecondaryDamageDesc", ""),
"ODOValue": attr.get("ODOValue", ""),
"ODOBrand": attr.get("ODOBrand", ""),
"RunAndDrive": attr.get("RunAndDrive", ""),
"Keys": attr.get("Keys", ""),
"BranchName": attr.get("BranchName", ""),
"AuctionDateTime": attr.get("AuctionDateTime", ""),
"Title": attr.get("Title", ""),
"TitleBrand": attr.get("TitleBrand", ""),
"TitleCode": attr.get("TitleCode", ""),
"EstRepairCost": attr.get("EstRepairCost", ""),
"VehicleGrade": attr.get("VehicleGrade", ""),
"highBidAmount": prebid.get("highBidAmount") or bid.get("highBidAmount", ""),
"buyNowPrice": prebid.get("buyNowPrice") or bid.get("buyNowPrice", ""),
"acv": attr.get("ProviderACV", ""),
"BranchNumber": attr.get("BranchNumber", ""),
"imageKeys": [item.get("k", "") for item in image_keys if isinstance(item, dict) and item.get("k")],
}
def _scrape_via_context_request(self, vehicle_url: str) -> CarRecord:
if not self.context:
self._new_context()
@@ -1558,6 +1690,8 @@ class IAAIScraper:
if response.status >= 400:
raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}")
html = response.data.decode("utf-8", errors="ignore")
if is_challenge_response(status_code=int(response.status), body_text=html, expected_marker=DETAIL_MARKER):
raise AntiBotDetectedError(f"IAAI challenge detected for {vehicle_url}")
js_data = self._extract_iaai_json_from_html(html, vehicle_url)
if not js_data:
@@ -1937,7 +2071,7 @@ class IAAIScraper:
)
# ── Фаза 2: браузерный fallback ──
if fallback_urls:
if fallback_urls and self.settings.scraping_profile.browser_fallback_enabled:
logger.info(
"Fallback browser mode: %d/%d URLs, single page",
len(fallback_urls), total,
@@ -1966,6 +2100,10 @@ class IAAIScraper:
batch_failures=cars_failed + len(failures),
protection_events=protection_events,
)
elif fallback_urls:
cars_failed += len(fallback_urls)
failures.extend({"vehicle_url": url, "error": "HTTP fast-path failed; browser fallback disabled"} for url, _ in fallback_urls)
logger.warning("Browser fallback disabled by profile; %d URLs marked failed", len(fallback_urls))
# ── Фаза 2.5: пост-фильтр по runtime_config ──
filters_cfg = self.runtime_config.filters
@@ -2034,6 +2172,7 @@ class IAAIScraper:
except Exception as exc:
logger.error("Batch upsert failed: %s", exc)
cars_failed += len(records)
failures.append({"vehicle_url": "db_batch", "error": str(exc)})
self._report_progress(
"sync_batch_db_upsert_failed",
batch_total=total,
@@ -2110,25 +2249,55 @@ class IAAIScraper:
is_partial_scan = True
failures: list[dict[str, str]] = []
listing: dict = {}
stream_result: dict[str, Any] = {}
try:
effective_only_new = self.settings.sync_only_new if only_new is None else only_new
stream_result = self._sync_listing_streaming(
make=make,
model=model,
lane=lane,
limit=limit,
effective_only_new=effective_only_new,
started_at=started_at,
listing_url=listing_url,
year_min=year_min,
year_max=year_max,
)
if self.settings.scraping_profile.http_first:
if year_min is not None or year_max is not None:
logger.warning(
"Fast HTTP-first profile ignores year split %s-%s and uses iaai-fast hidden-payload flow",
year_min,
year_max,
)
fast_engine = FastSyncEngine(
client=self.fast_client,
mapper=self.fast_mapper,
persistence=self.persistence,
batch_size=self.settings.celery.batch_size,
fetch_concurrency=self.settings.fetch_concurrency,
report_progress=self._report_progress,
)
stream_result = fast_engine.sync_listing(
runtime_config=self.runtime_config,
make=make,
model=model,
lane=lane,
limit=limit,
only_new=effective_only_new,
listing_url=listing_url,
max_pages=self.settings.listing.max_pages_per_run,
skip_mark_sold=skip_mark_sold,
)
else:
# Stable profile keeps the legacy browser streaming path.
stream_result = self._sync_listing_streaming(
make=make,
model=model,
lane=lane,
limit=limit,
effective_only_new=effective_only_new,
started_at=started_at,
listing_url=listing_url,
year_min=year_min,
year_max=year_max,
)
listing = stream_result["listing"]
total = stream_result["total"]
skipped_existing = stream_result["skipped_existing"]
cars_upserted = stream_result["cars_upserted"]
cars_failed = stream_result["cars_failed"]
cars_filtered = int(stream_result.get("cars_filtered", 0))
images_upserted = stream_result["images_upserted"]
protection_events = int(stream_result.get("protection_events", 0))
failures.extend(stream_result["failures"])
@@ -2138,10 +2307,13 @@ class IAAIScraper:
# Помечаем проданные авто, исчезнувшие из листинга (только при полном скане).
is_partial_scan = (
effective_only_new
or (limit is not None and limit > 0)
(limit is not None and limit > 0)
or make is not None
or model is not None
or listing_url is not None
or listing.get("early_stopped", False)
or listing.get("truncated_by_time_budget", False)
or bool(stream_result.get("full_scan_completed") is False)
)
if skip_mark_sold:
logger.debug("Skipping mark_sold: caller requested")
@@ -2153,8 +2325,8 @@ class IAAIScraper:
except Exception as exc:
logger.warning("Failed to mark sold cars: %s", exc)
elif is_partial_scan:
logger.debug("Skipping mark_sold: partial/incremental scan (only_new=%s, limit=%s, early_stopped=%s)",
effective_only_new, limit, listing.get("early_stopped", False))
logger.debug("Skipping mark_sold: partial scan (only_new=%s, limit=%s, make=%s, model=%s, listing_url=%s, early_stopped=%s)",
effective_only_new, limit, make, model, listing_url, listing.get("early_stopped", False))
except Exception as exc:
if not failures:
failures.append({"vehicle_url": "collect_listing", "error": str(exc)})
@@ -2189,7 +2361,7 @@ class IAAIScraper:
"run_id": run_id,
# partial_success допустим — отдельные машины могли не спарситься,
# это не повод повторять весь bootstrap.
"full_scan_completed": (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected,
"full_scan_completed": bool(stream_result.get("full_scan_completed", (not is_partial_scan) and status in ("success", "partial_success") and not anti_bot_detected)),
"only_new_effective": effective_only_new,
"listing": listing,
"total_discovered": discovered_total,
@@ -2242,7 +2414,7 @@ class IAAIScraper:
# Runtime-фильтры применяются позже (на уровне конкретных карточек),
# но не должны сужать сам обход сегментов.
logger.warning(
logger.info(
"Starting segmented sync: %d segments, resume from segment=%d",
len(segments), start_segment,
)
@@ -2257,13 +2429,13 @@ class IAAIScraper:
# иначе прогон становится частичным.
self._reload_runtime_config()
seg_url = self._build_segment_listing_url(base_url, seg_make) if seg_make else None
seg_url = None if self.settings.scraping_profile.http_first else (self._build_segment_listing_url(base_url, seg_make) if seg_make else None)
seg_label = f"{seg_make or 'ALL'}"
if seg_year_min is not None or seg_year_max is not None:
seg_label += f" ({seg_year_min}-{seg_year_max})"
logger.warning(
logger.info(
"Segment %d/%d: %s",
seg_idx + 1, len(segments), seg_label,
)
@@ -2292,7 +2464,7 @@ class IAAIScraper:
segments_total=len(segments),
)
result = self.sync_listing(
make=None if seg_url else seg_make,
make=seg_make,
model=None,
lane=lane,
only_new=only_new,
@@ -2320,8 +2492,9 @@ class IAAIScraper:
# Даже если сегмент завершился неполно (например, страница/пагинация сломалась),
# в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше.
if not segment_done:
completed_all = False
skipped_segments += 1
logger.warning(
logger.info(
"Segment %d/%d incomplete: %s — advancing checkpoint and continuing",
seg_idx + 1, len(segments), seg_label,
)
@@ -2337,7 +2510,7 @@ class IAAIScraper:
seg_idx, exc_info=True,
)
logger.warning(
logger.info(
"Segment %d/%d done: %s → upserted=%d, failed=%d, collected=%d",
seg_idx + 1, len(segments), seg_label,
result.get("cars_upserted", 0),
@@ -2354,6 +2527,7 @@ class IAAIScraper:
)
except Exception as exc:
logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc)
completed_all = False
all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)})
skipped_segments += 1
segment_results.append({
@@ -2388,7 +2562,7 @@ class IAAIScraper:
elapsed = round(time.perf_counter() - started_at, 3)
status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed"
logger.warning(
logger.info(
"Segmented sync done: %d/%d segments, upserted=%d, failed=%d, discovered=%d, elapsed=%.1fs",
len(segment_results), len(segments),
total_cars_upserted, total_cars_failed, total_discovered, elapsed,