From 3ff51fb93b8b9c06b2aa5f307b7fb980fe73d5e9 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Fri, 24 Apr 2026 20:47:18 +0300 Subject: [PATCH] add helper scripts --- .gitignore | 1 + scripts/db_activity_check.sql | 19 ++++++++ scripts/probe_listing_shape.py | 30 ++++++++++++ scripts/probe_vehicle_page_shape.py | 46 ++++++++++++++++++ scripts/sync_from_seed_urls.py | 75 +++++++++++++++++++++++++++++ 5 files changed, 171 insertions(+) create mode 100644 scripts/db_activity_check.sql create mode 100644 scripts/probe_listing_shape.py create mode 100644 scripts/probe_vehicle_page_shape.py create mode 100644 scripts/sync_from_seed_urls.py diff --git a/.gitignore b/.gitignore index 0c27e35..1b10c3c 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ venv/ *.pyd *.log *.db +iaai_sync_*.tar.gz storage_state.json .vscode/ *.egg-info/ diff --git a/scripts/db_activity_check.sql b/scripts/db_activity_check.sql new file mode 100644 index 0000000..3bc9005 --- /dev/null +++ b/scripts/db_activity_check.sql @@ -0,0 +1,19 @@ +select 'seen_1h' as metric, count(*)::text as value from cars where last_seen_at >= now() - interval '1 hour' +union all +select 'seen_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours' +union all +select 'old_rows_touched_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours' and id <= (select greatest(max(id) - 5000, 0) from cars) +union all +select 'min_recent_id_1h', coalesce(min(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour' +union all +select 'max_recent_id_1h', coalesce(max(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour' +union all +select 'top_5_recent_old_ids_24h', coalesce(string_agg(id::text, ', ' order by last_seen_at desc), 'none') +from ( + select id, last_seen_at + from cars + where last_seen_at >= now() - interval '24 hours' + and id <= (select greatest(max(id) - 5000, 0) from cars) + order by last_seen_at desc + limit 5 +) t; diff --git a/scripts/probe_listing_shape.py b/scripts/probe_listing_shape.py new file mode 100644 index 0000000..f39a91a --- /dev/null +++ b/scripts/probe_listing_shape.py @@ -0,0 +1,30 @@ +from playwright.sync_api import sync_playwright +import re + +url = "https://www.dubizzle.com/Vehiclelisting/Cars?Make=FORD" + +with sync_playwright() as p: + browser = p.chromium.launch(headless=True, args=["--headless=new"]) + page = browser.new_page() + page.goto(url, wait_until="commit", timeout=60000) + + try: + page.wait_for_load_state("domcontentloaded", timeout=5000) + except Exception: + pass + + title = page.title() + text = (page.evaluate("() => document.body ? document.body.innerText : ''") or "") + html = page.content() + combined = (text + "\n" + html).lower() + + print("TITLE=", title) + print("URL=", page.url) + print("HAS_INCAPSULA=", "incapsula" in combined) + print("HAS_CAPTCHA=", "captcha" in combined) + print("HAS_CHALLENGE=", "challenge" in combined) + print("HAS_VEHICLEDETAIL=", "/vehicledetail/" in combined) + print("HAS_MOTORS_USED_CARS=", "/motors/used-cars/" in combined) + print("TEXT_PREVIEW=", re.sub(r"\s+", " ", text)[:1000]) + + browser.close() diff --git a/scripts/probe_vehicle_page_shape.py b/scripts/probe_vehicle_page_shape.py new file mode 100644 index 0000000..e59801a --- /dev/null +++ b/scripts/probe_vehicle_page_shape.py @@ -0,0 +1,46 @@ +from playwright.sync_api import sync_playwright +from dubizzle_scraper.scraper import DUBIZZLEScraper +from dubizzle_scraper.parsing.parser import VehicleParser +import json +import re + +url = 'https://www.dubizzle.com/VehicleDetail/45394480~US' +out = {} +with sync_playwright() as p: + browser = p.chromium.launch(headless=True) + page = browser.new_page() + page.goto(url, wait_until='commit', timeout=60000) + try: + page.wait_for_load_state('domcontentloaded', timeout=5000) + except Exception: + pass + try: + page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=1000) + except Exception: + pass + html = page.content() + try: + dom_text = page.evaluate("() => document.body?.textContent || ''") + except Exception: + dom_text = '' + title = page.title() + js_data = {} + try: + js_data = page.evaluate(DUBIZZLEScraper._JS_EXTRACT) or {} + except Exception: + js_data = {'eval_error': True} + hints = VehicleParser._dom_hints(dom_text) + out = { + 'final_url': page.url, + 'title': title, + 'html_len': len(html), + 'dom_text_len': len(dom_text), + 'selector_present': any(token in html for token in ['VehicleDetailViewModel', 'veh-details', 'vehicle-details', 'vehicleDetailPage']), + 'js_ok': bool(js_data.get('ok')), + 'js_keys': sorted(list(js_data.keys()))[:20], + 'dom_hints': hints, + 'dom_text_preview': re.sub(r'\s+', ' ', dom_text)[:1500], + 'html_preview': re.sub(r'\s+', ' ', html)[:2000], + } + browser.close() +print(json.dumps(out, ensure_ascii=False)) diff --git a/scripts/sync_from_seed_urls.py b/scripts/sync_from_seed_urls.py new file mode 100644 index 0000000..a1ece79 --- /dev/null +++ b/scripts/sync_from_seed_urls.py @@ -0,0 +1,75 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path + +from dubizzle_scraper.scraper import DUBIZZLEScraper + + +def _load_urls(path: Path) -> list[str]: + raw = path.read_text(encoding="utf-8") + payload = json.loads(raw) + + urls: list[str] = [] + if isinstance(payload, list): + urls = [str(item).strip() for item in payload if str(item).strip()] + elif isinstance(payload, dict): + candidates = payload.get("urls") or payload.get("vehicle_urls") or [] + if isinstance(candidates, list): + urls = [str(item).strip() for item in candidates if str(item).strip()] + + # дедуп по порядку + deduped: list[str] = [] + seen: set[str] = set() + for url in urls: + if url in seen: + continue + seen.add(url) + deduped.append(url) + return deduped + + +def main() -> None: + parser = argparse.ArgumentParser(description="Sync cars from pre-collected seed URLs") + parser.add_argument("--input", default="artifacts/json/seed_urls.json", help="JSON file with list of vehicle URLs") + parser.add_argument("--lane", default="dubizzle_cars", help="Target lane") + parser.add_argument("--batch-size", type=int, default=100, help="Batch size for sync_batch") + args = parser.parse_args() + + input_path = Path(args.input) + if not input_path.exists(): + raise SystemExit(f"Input file not found: {input_path}") + + urls = _load_urls(input_path) + if not urls: + raise SystemExit("No URLs found in input JSON") + + total_upserted = 0 + total_failed = 0 + total_images = 0 + failures: list[dict[str, str]] = [] + + with DUBIZZLEScraper() as scraper: + for i in range(0, len(urls), max(1, args.batch_size)): + chunk = urls[i:i + max(1, args.batch_size)] + result = scraper.sync_batch(chunk, lane=args.lane) + total_upserted += int(result.get("cars_upserted", 0)) + total_failed += int(result.get("cars_failed", 0)) + total_images += int(result.get("images_upserted", 0)) + failures.extend(result.get("failures", [])) + print(f"[{i + 1}-{i + len(chunk)}] upserted={result.get('cars_upserted', 0)} failed={result.get('cars_failed', 0)}") + + summary = { + "input": str(input_path), + "urls_total": len(urls), + "cars_upserted": total_upserted, + "cars_failed": total_failed, + "images_upserted": total_images, + "failures_count": len(failures), + } + print(json.dumps(summary, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main()