47 lines
1.6 KiB
Python
47 lines
1.6 KiB
Python
from playwright.sync_api import sync_playwright
|
|
from iaai_scraper.scraper import IAAIScraper
|
|
from iaai_scraper.parsing.parser import VehicleParser
|
|
import json
|
|
import re
|
|
|
|
url = 'https://www.iaai.com/VehicleDetail/45394480~US'
|
|
out = {}
|
|
with sync_playwright() as p:
|
|
browser = p.chromium.launch(headless=True)
|
|
page = browser.new_page()
|
|
page.goto(url, wait_until='commit', timeout=60000)
|
|
try:
|
|
page.wait_for_load_state('domcontentloaded', timeout=5000)
|
|
except Exception:
|
|
pass
|
|
try:
|
|
page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=1000)
|
|
except Exception:
|
|
pass
|
|
html = page.content()
|
|
try:
|
|
dom_text = page.evaluate("() => document.body?.textContent || ''")
|
|
except Exception:
|
|
dom_text = ''
|
|
title = page.title()
|
|
js_data = {}
|
|
try:
|
|
js_data = page.evaluate(IAAIScraper._JS_EXTRACT) or {}
|
|
except Exception:
|
|
js_data = {'eval_error': True}
|
|
hints = VehicleParser._dom_hints(dom_text)
|
|
out = {
|
|
'final_url': page.url,
|
|
'title': title,
|
|
'html_len': len(html),
|
|
'dom_text_len': len(dom_text),
|
|
'selector_present': any(token in html for token in ['VehicleDetailViewModel', 'veh-details', 'vehicle-details', 'vehicleDetailPage']),
|
|
'js_ok': bool(js_data.get('ok')),
|
|
'js_keys': sorted(list(js_data.keys()))[:20],
|
|
'dom_hints': hints,
|
|
'dom_text_preview': re.sub(r'\s+', ' ', dom_text)[:1500],
|
|
'html_preview': re.sub(r'\s+', ' ', html)[:2000],
|
|
}
|
|
browser.close()
|
|
print(json.dumps(out, ensure_ascii=False))
|