dubizzle project

scripts/db_activity_check.sql

@@ -0,0 +1,19 @@
+select 'seen_1h' as metric, count(*)::text as value from cars where last_seen_at >= now() - interval '1 hour'
+union all
+select 'seen_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours'
+union all
+select 'old_rows_touched_24h', count(*)::text from cars where last_seen_at >= now() - interval '24 hours' and id <= (select greatest(max(id) - 5000, 0) from cars)
+union all
+select 'min_recent_id_1h', coalesce(min(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour'
+union all
+select 'max_recent_id_1h', coalesce(max(id)::text, 'null') from cars where last_seen_at >= now() - interval '1 hour'
+union all
+select 'top_5_recent_old_ids_24h', coalesce(string_agg(id::text, ', ' order by last_seen_at desc), 'none')
+from (
+    select id, last_seen_at
+    from cars
+    where last_seen_at >= now() - interval '24 hours'
+      and id <= (select greatest(max(id) - 5000, 0) from cars)
+    order by last_seen_at desc
+    limit 5
+) t;

scripts/probe_listing_shape.py

@@ -0,0 +1,30 @@
+from playwright.sync_api import sync_playwright
+import re
+
+url = "https://www.dubizzle.com/Vehiclelisting/Cars?Make=FORD"
+
+with sync_playwright() as p:
+    browser = p.chromium.launch(headless=True, args=["--headless=new"])
+    page = browser.new_page()
+    page.goto(url, wait_until="commit", timeout=60000)
+
+    try:
+        page.wait_for_load_state("domcontentloaded", timeout=5000)
+    except Exception:
+        pass
+
+    title = page.title()
+    text = (page.evaluate("() => document.body ? document.body.innerText : ''") or "")
+    html = page.content()
+    combined = (text + "\n" + html).lower()
+
+    print("TITLE=", title)
+    print("URL=", page.url)
+    print("HAS_INCAPSULA=", "incapsula" in combined)
+    print("HAS_CAPTCHA=", "captcha" in combined)
+    print("HAS_CHALLENGE=", "challenge" in combined)
+    print("HAS_VEHICLEDETAIL=", "/vehicledetail/" in combined)
+    print("HAS_MOTORS_USED_CARS=", "/motors/used-cars/" in combined)
+    print("TEXT_PREVIEW=", re.sub(r"\s+", " ", text)[:1000])
+
+    browser.close()

scripts/probe_vehicle_page_shape.py

@@ -0,0 +1,46 @@
+from playwright.sync_api import sync_playwright
+from dubizzle_scraper.scraper import DUBIZZLEScraper
+from dubizzle_scraper.parsing.parser import VehicleParser
+import json
+import re
+
+url = 'https://www.dubizzle.com/VehicleDetail/45394480~US'
+out = {}
+with sync_playwright() as p:
+    browser = p.chromium.launch(headless=True)
+    page = browser.new_page()
+    page.goto(url, wait_until='commit', timeout=60000)
+    try:
+        page.wait_for_load_state('domcontentloaded', timeout=5000)
+    except Exception:
+        pass
+    try:
+        page.wait_for_selector("#VehicleDetailViewModel, .veh-details, .vehicle-details, [data-uname='vehicleDetailPage']", timeout=1000)
+    except Exception:
+        pass
+    html = page.content()
+    try:
+        dom_text = page.evaluate("() => document.body?.textContent || ''")
+    except Exception:
+        dom_text = ''
+    title = page.title()
+    js_data = {}
+    try:
+        js_data = page.evaluate(DUBIZZLEScraper._JS_EXTRACT) or {}
+    except Exception:
+        js_data = {'eval_error': True}
+    hints = VehicleParser._dom_hints(dom_text)
+    out = {
+        'final_url': page.url,
+        'title': title,
+        'html_len': len(html),
+        'dom_text_len': len(dom_text),
+        'selector_present': any(token in html for token in ['VehicleDetailViewModel', 'veh-details', 'vehicle-details', 'vehicleDetailPage']),
+        'js_ok': bool(js_data.get('ok')),
+        'js_keys': sorted(list(js_data.keys()))[:20],
+        'dom_hints': hints,
+        'dom_text_preview': re.sub(r'\s+', ' ', dom_text)[:1500],
+        'html_preview': re.sub(r'\s+', ' ', html)[:2000],
+    }
+    browser.close()
+print(json.dumps(out, ensure_ascii=False))

scripts/sync_from_seed_urls.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+
+from dubizzle_scraper.scraper import DUBIZZLEScraper
+
+
+def _load_urls(path: Path) -> list[str]:
+    raw = path.read_text(encoding="utf-8")
+    payload = json.loads(raw)
+
+    urls: list[str] = []
+    if isinstance(payload, list):
+        urls = [str(item).strip() for item in payload if str(item).strip()]
+    elif isinstance(payload, dict):
+        candidates = payload.get("urls") or payload.get("vehicle_urls") or []
+        if isinstance(candidates, list):
+            urls = [str(item).strip() for item in candidates if str(item).strip()]
+
+    # дедуп по порядку
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        if url in seen:
+            continue
+        seen.add(url)
+        deduped.append(url)
+    return deduped
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Sync cars from pre-collected seed URLs")
+    parser.add_argument("--input", default="artifacts/json/seed_urls.json", help="JSON file with list of vehicle URLs")
+    parser.add_argument("--lane", default="dubizzle_cars", help="Target lane")
+    parser.add_argument("--batch-size", type=int, default=100, help="Batch size for sync_batch")
+    args = parser.parse_args()
+
+    input_path = Path(args.input)
+    if not input_path.exists():
+        raise SystemExit(f"Input file not found: {input_path}")
+
+    urls = _load_urls(input_path)
+    if not urls:
+        raise SystemExit("No URLs found in input JSON")
+
+    total_upserted = 0
+    total_failed = 0
+    total_images = 0
+    failures: list[dict[str, str]] = []
+
+    with DUBIZZLEScraper() as scraper:
+        for i in range(0, len(urls), max(1, args.batch_size)):
+            chunk = urls[i:i + max(1, args.batch_size)]
+            result = scraper.sync_batch(chunk, lane=args.lane)
+            total_upserted += int(result.get("cars_upserted", 0))
+            total_failed += int(result.get("cars_failed", 0))
+            total_images += int(result.get("images_upserted", 0))
+            failures.extend(result.get("failures", []))
+            print(f"[{i + 1}-{i + len(chunk)}] upserted={result.get('cars_upserted', 0)} failed={result.get('cars_failed', 0)}")
+
+    summary = {
+        "input": str(input_path),
+        "urls_total": len(urls),
+        "cars_upserted": total_upserted,
+        "cars_failed": total_failed,
+        "images_upserted": total_images,
+        "failures_count": len(failures),
+    }
+    print(json.dumps(summary, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()