add helper scripts
This commit is contained in:
30
scripts/probe_listing_shape.py
Normal file
30
scripts/probe_listing_shape.py
Normal file
@@ -0,0 +1,30 @@
|
||||
from playwright.sync_api import sync_playwright
|
||||
import re
|
||||
|
||||
url = "https://www.dubizzle.com/Vehiclelisting/Cars?Make=FORD"
|
||||
|
||||
with sync_playwright() as p:
|
||||
browser = p.chromium.launch(headless=True, args=["--headless=new"])
|
||||
page = browser.new_page()
|
||||
page.goto(url, wait_until="commit", timeout=60000)
|
||||
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=5000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
title = page.title()
|
||||
text = (page.evaluate("() => document.body ? document.body.innerText : ''") or "")
|
||||
html = page.content()
|
||||
combined = (text + "\n" + html).lower()
|
||||
|
||||
print("TITLE=", title)
|
||||
print("URL=", page.url)
|
||||
print("HAS_INCAPSULA=", "incapsula" in combined)
|
||||
print("HAS_CAPTCHA=", "captcha" in combined)
|
||||
print("HAS_CHALLENGE=", "challenge" in combined)
|
||||
print("HAS_VEHICLEDETAIL=", "/vehicledetail/" in combined)
|
||||
print("HAS_MOTORS_USED_CARS=", "/motors/used-cars/" in combined)
|
||||
print("TEXT_PREVIEW=", re.sub(r"\s+", " ", text)[:1000])
|
||||
|
||||
browser.close()
|
||||
Reference in New Issue
Block a user