qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix dubizzle listing extraction

Browse Source
This commit is contained in:
qananasikq
2026-05-08 13:31:22 +03:00
parent 42bd4c64d6
commit 046e9f125f
2 changed files with 281 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

260
dubizzle_scraper/scraper.py
View File

@@ -7,6 +7,7 @@ import signal
import time
import uuid
import html as html_module
from collections.abc import Mapping
from concurrent.futures import ThreadPoolExecutor, as_completed
from concurrent.futures import TimeoutError as FuturesTimeoutError
from datetime import datetime, timezone
@@ -44,6 +45,10 @@ VEHICLE_ID_RE = re.compile(
)
HTML_TAG_RE = re.compile(r"<[^>]+>")
SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?</\1>", re.IGNORECASE | re.DOTALL)
NEXT_DATA_RE = re.compile(
r'<script[^>]+id=["\']__NEXT_DATA__["\'][^>]*type=["\']application/json["\'][^>]*>(.*?)</script>',
re.IGNORECASE | re.DOTALL,
)
# Таймауты операций страницы.
_PAGE_COLLECT_TIMEOUT_S = 90
@@ -797,6 +802,11 @@ class DUBIZZLEScraper:
total = len(records) if effective_only_new else len(all_raw_urls)
all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url}
all_listing_origin_ids = {
origin_id
for origin_id in (self._extract_db_origin_id_from_url(url) for url in all_raw_urls)
if origin_id
}
return {
"listing": {
"status": "ok",
@@ -818,6 +828,7 @@ class DUBIZZLEScraper:
"protection_events": 0,
"failures": failures,
"all_listing_origin_urls": all_listing_origin_urls,
"all_listing_origin_ids": all_listing_origin_ids,
}
def _sync_listing_streaming(
@@ -952,6 +963,11 @@ class DUBIZZLEScraper:
all_raw_urls = list(algolia_result.vehicle_urls)
all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url}
all_listing_origin_ids = {
str(origin_id).strip()
for origin_id in algolia_result.origin_ids_by_url.values()
if str(origin_id).strip()
}
pages_info = list(algolia_result.pages)
effective_urls = list(all_raw_urls)
@@ -1065,6 +1081,7 @@ class DUBIZZLEScraper:
"protection_events": protection_events,
"failures": failures,
"all_listing_origin_urls": all_listing_origin_urls,
"all_listing_origin_ids": all_listing_origin_ids,
}
def _sync_listing_streaming_sitemap_fallback(
@@ -1107,6 +1124,11 @@ class DUBIZZLEScraper:
failures.append({"vehicle_url": f"sitemap_batch_{i}", "error": str(exc)})
all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_urls if url}
all_listing_origin_ids = {
origin_id
for origin_id in (self._extract_db_origin_id_from_url(url) for url in all_urls)
if origin_id
}
return {
"listing": {
"status": "ok",
@@ -1128,6 +1150,7 @@ class DUBIZZLEScraper:
"protection_events": 0,
"failures": failures,
"all_listing_origin_urls": all_listing_origin_urls,
"all_listing_origin_ids": all_listing_origin_ids,
}
def _sync_listing_streaming_browser_fallback(
@@ -1275,6 +1298,11 @@ class DUBIZZLEScraper:
page.close()
all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url}
all_listing_origin_ids = {
origin_id
for origin_id in (self._extract_db_origin_id_from_url(url) for url in all_raw_urls)
if origin_id
}
total = len(all_raw_urls) if not effective_only_new else max(0, len(all_raw_urls) - skipped_existing)
return {
"listing": {
@@ -1296,6 +1324,7 @@ class DUBIZZLEScraper:
"protection_events": 0,
"failures": failures,
"all_listing_origin_urls": all_listing_origin_urls,
"all_listing_origin_ids": all_listing_origin_ids,
}
def _get_page(self) -> Page:
@@ -1314,6 +1343,48 @@ class DUBIZZLEScraper:
_JS_EXTRACT = """
() => {
try {
const nextData = window.__NEXT_DATA__;
const listing = nextData?.props?.pageProps?.reduxWrapperActionsGIPP
?.find((entry) => entry && entry.payload && entry.payload.listing)
?.payload?.listing;
if (listing && listing.details) {
const detailSections = listing.details;
const normalizeSection = (items) => Array.isArray(items)
? items.map((item) => ({
label: item?.label || '',
value: item?.value ?? '',
slug: item?.slug || '',
}))
: [];
return {
ok: true,
source: 'next_data',
listing: {
name: listing.name || '',
description: listing.description || listing.long_description || '',
absolute_url: listing.absolute_url || {},
short_url: listing.short_url || '',
price: listing.price || {},
location: listing.location || {},
posted_timestamp: listing.posted_timestamp || null,
tracking: listing.tracking || {},
categories: Array.isArray(listing.categories) ? listing.categories : [],
details: {
make_model_trim: normalizeSection(detailSections.make_model_trim),
primary: normalizeSection(detailSections.primary),
secondary: normalizeSection(detailSections.secondary),
rental_details: normalizeSection(detailSections.rental_details),
requirements: normalizeSection(detailSections.requirements),
},
photos: Array.isArray(listing.photos_combined)
? listing.photos_combined.map((photo) => photo?.url || photo?.large || photo?.medium || photo?.small || '').filter(Boolean)
: Array.isArray(listing.photos)
? listing.photos.map((photo) => photo?.url || photo?.large || photo?.medium || photo?.small || '').filter(Boolean)
: [],
},
};
}
const scripts = document.querySelectorAll('script:not([src])');
for (const s of scripts) {
const t = s.textContent || '';
@@ -1383,6 +1454,142 @@ class DUBIZZLEScraper:
if not js_data or not js_data.get("ok"):
return {}
if js_data.get("source") == "next_data":
listing = js_data.get("listing") or {}
details = listing.get("details") or {}
categories = listing.get("categories") or []
absolute_url = listing.get("absolute_url") or {}
location = listing.get("location") or {}
tracking = listing.get("tracking") or {}
def detail_value_by_slug(target_slug: str) -> Any:
target = target_slug.strip().lower()
for section_items in details.values():
if not isinstance(section_items, list):
continue
for item in section_items:
if not isinstance(item, Mapping):
continue
if str(item.get("slug") or "").strip().lower() == target:
return item.get("value")
return None
def pick_photo_url(photo: Any) -> str:
if isinstance(photo, str):
return photo.strip()
if not isinstance(photo, Mapping):
return ""
for key in ("url", "main", "large", "medium", "small", "micro"):
value = photo.get(key)
if value:
return str(value).strip()
return ""
detail_sections: dict[str, list[dict[str, Any]]] = {}
flat_details: dict[str, dict[str, dict[str, Any]]] = {}
for section_name, items in details.items():
if not isinstance(items, list):
continue
normalized_items: list[dict[str, Any]] = []
for item in items:
if not isinstance(item, Mapping):
continue
label = str(item.get("label") or "").strip()
value = item.get("value")
slug = str(item.get("slug") or "").strip()
normalized_items.append({"label": {"en": label}, "value": {"en": value}, "slug": slug})
if label:
flat_details[label] = {"en": {"label": label, "value": value}}
if normalized_items:
detail_sections[section_name] = normalized_items
category_v2 = None
category_make = None
category_model = None
if categories:
names_en = [str(item.get("name") or "").strip() for item in categories if isinstance(item, Mapping)]
slug_paths = [str(item.get("full_slug") or item.get("slug") or "").strip() for item in categories if isinstance(item, Mapping)]
ids = [item.get("legacy_id") for item in categories if isinstance(item, Mapping)]
category_v2 = {
"names_en": names_en,
"slug_paths": slug_paths,
"ids": ids,
}
if len(names_en) >= 4:
category_make = names_en[2] or None
category_model = names_en[3] or None
photos: list[str] = []
for photo in listing.get("photos_combined") or []:
picked = pick_photo_url(photo)
if picked and picked not in photos:
photos.append(picked)
for photo in listing.get("photos") or []:
picked = pick_photo_url(photo)
if picked and picked not in photos:
photos.append(picked)
posted_timestamp = listing.get("posted_timestamp")
posted_at_iso = None
if isinstance(posted_timestamp, (int, float)) and posted_timestamp > 0:
posted_at_iso = datetime.fromtimestamp(posted_timestamp, tz=timezone.utc).isoformat()
price = listing.get("price") if isinstance(listing.get("price"), Mapping) else {}
price_raw = price.get("raw") or price.get("formatted")
neighborhood_name = tracking.get("neighbourhood", {}).get("name") if isinstance(tracking.get("neighbourhood"), Mapping) else None
year = detail_value_by_slug("year")
body_type = detail_value_by_slug("body_type")
kilometers = detail_value_by_slug("kilometers")
engine_capacity = detail_value_by_slug("engine_capacity_cc")
transmission_type = detail_value_by_slug("transmission_type")
steering_side = detail_value_by_slug("steering_side")
exterior_color = detail_value_by_slug("exterior_color")
seller_type = detail_value_by_slug("seller_type")
trim = detail_value_by_slug("motors_trim")
summary: dict[str, Any] = {
"source_url": vehicle_url,
"name": {"en": listing.get("name")},
"title": listing.get("name"),
"description": listing.get("description") or listing.get("long_description"),
"make": category_make,
"model": category_model,
"trim": trim,
"year": year,
"body_type": body_type,
"odometer": kilometers,
"kilometers": kilometers,
"engine": engine_capacity,
"engine_volume": engine_capacity,
"transmission_type": transmission_type,
"gearbox": transmission_type,
"steering_side": steering_side,
"steering_wheel": steering_side,
"exterior_color": exterior_color,
"color": exterior_color,
"seller": seller_type,
"price": price_raw,
"buy_now": price_raw,
"currency": price.get("currency") or "AED",
"details_v2": detail_sections,
"details": flat_details,
"image_urls": photos,
"photo_mains": photos,
"category_v2": category_v2,
"location": location.get("name"),
"location_name": location.get("name"),
"site": {"en": "UAE"},
"posted_at": posted_at_iso,
"permalink": listing.get("short_url") or vehicle_url,
"absolute_url": absolute_url,
"id": tracking.get("legacy_id") or listing.get("listing_id") or listing.get("object_id"),
"objectID": listing.get("encoded_object_id") or listing.get("object_id"),
"uuid": listing.get("listing_uuid") or listing.get("uuid"),
}
if neighborhood_name:
summary["neighbourhood"] = {"en": neighborhood_name}
return summary
brnch = str(js_data.get("BranchNumber", "") or "").strip()
img_keys = js_data.get("imageKeys") or []
image_urls = [
@@ -1421,16 +1628,18 @@ class DUBIZZLEScraper:
@staticmethod
def _build_payload_insights(vehicle_summary: dict) -> dict:
currency = vehicle_summary.get("currency") or "USD"
effective_price = vehicle_summary.get("buy_now") or vehicle_summary.get("price")
return {
"vehicle_core": vehicle_summary,
"pricing": {
"buy_now": vehicle_summary.get("buy_now"),
"buy_now": effective_price,
"current_bid": vehicle_summary.get("current_bid"),
"actual_cash_value": vehicle_summary.get("actual_cash_value"),
"estimated_repair_cost": vehicle_summary.get("estimated_repair_cost"),
"currency": "USD",
"currency": currency,
},
"bids": {"amount": vehicle_summary.get("current_bid"), "currency": "USD"},
"bids": {"amount": vehicle_summary.get("current_bid"), "currency": currency},
"damage": {"primary": vehicle_summary.get("primary_damage"), "secondary": vehicle_summary.get("secondary_damage")},
"auction": {"auction_date": vehicle_summary.get("auction_date"), "branch": vehicle_summary.get("location")},
"images": {"count": len(vehicle_summary.get("image_urls") or []), "urls": vehicle_summary.get("image_urls") or []},
@@ -1530,6 +1739,27 @@ class DUBIZZLEScraper:
Использует json.JSONDecoder.raw_decode для быстрого поиска JSON
вместо посимвольного сканирования скобок.
"""
next_match = NEXT_DATA_RE.search(html)
if next_match:
try:
next_data = json.loads(html_module.unescape(next_match.group(1)))
actions = (
next_data.get("props", {})
.get("pageProps", {})
.get("reduxWrapperActionsGIPP", [])
)
for entry in actions:
payload = entry.get("payload") if isinstance(entry, dict) else None
listing = payload.get("listing") if isinstance(payload, dict) else None
if isinstance(listing, dict) and isinstance(listing.get("details"), dict):
return {
"ok": True,
"source": "next_data",
"listing": listing,
}
except Exception:
pass
search = "inventoryView"
pos = html.find(search)
if pos < 0:
@@ -2259,6 +2489,7 @@ class DUBIZZLEScraper:
protection_events = int(stream_result.get("protection_events", 0))
failures.extend(stream_result["failures"])
all_listing_origin_urls = stream_result["all_listing_origin_urls"]
all_listing_origin_ids = stream_result.get("all_listing_origin_ids", set())
logger.info("Streaming sync processed %d vehicles", total)
@@ -2271,6 +2502,13 @@ class DUBIZZLEScraper:
)
if skip_mark_sold:
logger.debug("Skipping mark_sold: caller requested")
elif all_listing_origin_ids and not is_partial_scan:
try:
sold_count = self.persistence.mark_sold_not_in_listing(all_listing_origin_ids)
if sold_count:
logger.info("Marked %d cars as sold by origin_id", sold_count)
except Exception as exc:
logger.warning("Failed to mark sold cars by origin_id: %s", exc)
elif all_listing_origin_urls and not is_partial_scan:
try:
sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls)
@@ -2363,6 +2601,8 @@ class DUBIZZLEScraper:
segment_results: list[dict[str, Any]] = []
completed_all = True
skipped_segments = 0
aggregated_active_urls: set[str] = set()
aggregated_active_ids: set[str] = set()
# В segmented-режиме полный прогон должен проходить ВСЕ сегменты.
# Runtime-фильтры применяются позже (на уровне конкретных карточек),
@@ -2425,6 +2665,7 @@ class DUBIZZLEScraper:
listing_url=seg_url,
year_min=seg_year_min,
year_max=seg_year_max,
skip_mark_sold=True,
)
total_cars_upserted += result.get("cars_upserted", 0)
total_cars_failed += result.get("cars_failed", 0)
@@ -2432,6 +2673,8 @@ class DUBIZZLEScraper:
total_skipped += result.get("skipped_existing", 0)
total_discovered += result.get("listing", {}).get("vehicles_collected", 0)
all_failures.extend(result.get("failures", []))
aggregated_active_urls.update(result.get("all_listing_origin_urls", set()) or set())
aggregated_active_ids.update(result.get("all_listing_origin_ids", set()) or set())
segment_results.append({
"segment": seg,
"segment_index": seg_idx,
@@ -2514,6 +2757,15 @@ class DUBIZZLEScraper:
finally:
self.set_progress_callback(outer_progress_callback)
if completed_all:
try:
if aggregated_active_ids:
self.persistence.mark_sold_not_in_listing(aggregated_active_ids)
elif aggregated_active_urls:
self.persistence.mark_sold_not_in_listing_by_urls(aggregated_active_urls)
except Exception:
logger.warning("Segmented sync final sold-mark failed", exc_info=True)
elapsed = round(time.perf_counter() - started_at, 3)
status = "success" if not all_failures else "partial_success" if total_cars_upserted else "failed"
@@ -2537,6 +2789,8 @@ class DUBIZZLEScraper:
"images_upserted": total_images_upserted,
"skipped_existing": total_skipped,
"total_discovered": total_discovered,
"all_listing_origin_urls": aggregated_active_urls,
"all_listing_origin_ids": aggregated_active_ids,
"elapsed_seconds": elapsed,
"failures": all_failures,
"segment_results": segment_results,