fix docker scraping
improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
This commit is contained in:
qananasikq
@@ -4,16 +4,22 @@ import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from ..core.utils import LOT_RE, PRICE_RE, VIN_RE, deep_find_key, first_non_empty
|
||||
from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
|
||||
|
||||
logger = logging.getLogger("iaai_scraper.parsers")
|
||||
|
||||
|
||||
class VehicleParser:
|
||||
# DOM + JSON парсер страницы авто.
|
||||
# Парсер данных страницы автомобиля.
|
||||
|
||||
# Регулярные выражения для парсинга и детекции защиты.
|
||||
_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
|
||||
_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
|
||||
_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
|
||||
_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
|
||||
_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
|
||||
|
||||
SUMMARY_KEY_MAP = {
|
||||
"vin": {"vin", "vehicleidentificationnumber"},
|
||||
"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
|
||||
"year": {"year"},
|
||||
"make": {"make", "manufacturer", "brand"},
|
||||
@@ -32,10 +38,10 @@ class VehicleParser:
|
||||
"seller": {"seller", "sellername"},
|
||||
"location": {"location", "branchname", "auctionlocation", "branch"},
|
||||
"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
|
||||
"body_type": {"bodytype", "bodystyle"},
|
||||
"drive": {"driveline", "drive"},
|
||||
"gearbox": {"transmission", "gearbox"},
|
||||
"engine": {"engine", "enginevolume"},
|
||||
"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
|
||||
"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
|
||||
"gearbox": {"transmission", "gearbox", "transmissiontype"},
|
||||
"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
|
||||
"fuel_type": {"fueltype", "fuel"},
|
||||
"cylinders": {"cylinders", "cylindercount"},
|
||||
"color": {"color", "primarycolor", "exteriorcolor"},
|
||||
@@ -43,24 +49,35 @@ class VehicleParser:
|
||||
|
||||
DOM_LABEL_MAP: dict[str, str] = {
|
||||
"stock #": "lot_number",
|
||||
"vin (status)": "vin",
|
||||
"vin": "vin",
|
||||
"stock": "lot_number",
|
||||
"primary damage": "primary_damage",
|
||||
"secondary damage": "secondary_damage",
|
||||
"odometer": "odometer",
|
||||
"odometer (miles)": "odometer",
|
||||
"mileage": "odometer",
|
||||
"body style": "body_type",
|
||||
"body type": "body_type",
|
||||
"vehicle type": "body_type",
|
||||
"engine": "engine",
|
||||
"engine type": "engine",
|
||||
"transmission": "gearbox",
|
||||
"drive line type": "drive",
|
||||
"driveline type": "drive",
|
||||
"drive line": "drive",
|
||||
"driveline": "drive",
|
||||
"drive type": "drive",
|
||||
"fuel type": "fuel_type",
|
||||
"fuel": "fuel_type",
|
||||
"cylinders": "cylinders",
|
||||
"exterior/interior": "color",
|
||||
"exterior color": "color",
|
||||
"color": "color",
|
||||
"model": "model",
|
||||
"series": "trim",
|
||||
"selling branch": "location",
|
||||
"vehicle location": "vehicle_location",
|
||||
"auction date and time": "auction_date",
|
||||
"sale date": "auction_date",
|
||||
"lane/run #": "lane",
|
||||
"actual cash value": "actual_cash_value",
|
||||
"estimated repair cost": "estimated_repair_cost",
|
||||
@@ -69,6 +86,7 @@ class VehicleParser:
|
||||
"title/sale doc brand": "title_brand",
|
||||
"start code": "run_and_drive",
|
||||
"key": "keys",
|
||||
"keys": "keys",
|
||||
"manufactured in": "manufactured_in",
|
||||
"vehicle class": "vehicle_class",
|
||||
}
|
||||
@@ -79,15 +97,41 @@ class VehicleParser:
|
||||
return result
|
||||
lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
|
||||
known_labels = set(self.DOM_LABEL_MAP.keys())
|
||||
skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
|
||||
max_fields = len(set(self.DOM_LABEL_MAP.values()))
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
clean = line.rstrip(":").lower().strip()
|
||||
if clean in known_labels and i + 1 < len(lines):
|
||||
# Ранний выход, когда уже нашли все поля.
|
||||
if len(result) >= max_fields:
|
||||
break
|
||||
|
||||
# Сценарий 1: "метка: значение" в одной строке.
|
||||
colon_pos = line.find(":")
|
||||
if colon_pos > 0:
|
||||
label_part = line[:colon_pos].strip().lower()
|
||||
value_part = line[colon_pos + 1:].strip()
|
||||
if label_part in known_labels and value_part and value_part.lower() not in skip_values:
|
||||
field_name = self.DOM_LABEL_MAP[label_part]
|
||||
if field_name not in result or not result[field_name]:
|
||||
result[field_name] = value_part
|
||||
continue
|
||||
|
||||
# Сценарий 2: метка и значение на соседних строках.
|
||||
clean = line.rstrip(":").strip().lower()
|
||||
clean_alt = clean.rstrip("#").strip()
|
||||
matched_label = None
|
||||
if clean in known_labels:
|
||||
matched_label = clean
|
||||
elif clean_alt in known_labels:
|
||||
matched_label = clean_alt
|
||||
|
||||
if matched_label and i + 1 < len(lines):
|
||||
value = lines[i + 1].strip()
|
||||
if value.rstrip(":").lower().strip() in known_labels:
|
||||
continue
|
||||
if value.lower() in ("more actions", "view", "print", "share", "back to results"):
|
||||
if value.lower() in skip_values:
|
||||
continue
|
||||
field_name = self.DOM_LABEL_MAP[clean]
|
||||
field_name = self.DOM_LABEL_MAP[matched_label]
|
||||
if field_name not in result or not result[field_name]:
|
||||
result[field_name] = value
|
||||
return result
|
||||
@@ -120,11 +164,11 @@ class VehicleParser:
|
||||
page_title = title_match.group(1).strip()
|
||||
title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
|
||||
|
||||
# Один проход по payload для всех полей.
|
||||
all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
|
||||
|
||||
summary: dict[str, Any] = {"source_url": vehicle_url}
|
||||
for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
|
||||
values: list[Any] = []
|
||||
for payload in payloads:
|
||||
values.extend(deep_find_key(payload, candidate_keys))
|
||||
for field, values in all_found.items():
|
||||
if field in dom_kv:
|
||||
values.append(dom_kv[field])
|
||||
summary[field] = first_non_empty(values)
|
||||
@@ -133,11 +177,6 @@ class VehicleParser:
|
||||
summary["make"] = summary.get("make") or title_parsed.get("make")
|
||||
summary["model"] = summary.get("model") or title_parsed.get("model")
|
||||
summary["trim"] = summary.get("trim") or dom_kv.get("trim")
|
||||
summary["vin"] = summary.get("vin") or self._extract_vin(dom_text) or self._extract_vin(page_html)
|
||||
if summary.get("vin") and isinstance(summary["vin"], str):
|
||||
vin_clean = re.sub(r"\s*\(.*?\)\s*$", "", summary["vin"]).strip()
|
||||
vin_match = VIN_RE.search(vin_clean)
|
||||
summary["vin"] = vin_match.group(1) if vin_match else vin_clean
|
||||
summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
|
||||
summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
|
||||
for dom_field, dom_value in dom_kv.items():
|
||||
@@ -147,7 +186,7 @@ class VehicleParser:
|
||||
if not summary.get("actual_cash_value") and prices:
|
||||
summary["actual_cash_value"] = prices[0]
|
||||
if not summary.get("buy_now"):
|
||||
buy_now_match = re.search(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", dom_text or "", re.IGNORECASE)
|
||||
buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
|
||||
if buy_now_match:
|
||||
summary["buy_now"] = buy_now_match.group(1)
|
||||
elif prices:
|
||||
@@ -160,25 +199,30 @@ class VehicleParser:
|
||||
p = item.get("payload")
|
||||
if isinstance(p, (dict, list)):
|
||||
payloads.append(p)
|
||||
for field, candidate_keys in self.SUMMARY_KEY_MAP.items():
|
||||
# Дополнительный проход по встроенному JSON.
|
||||
extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
|
||||
for field, vals in extra.items():
|
||||
if not summary.get(field):
|
||||
val = first_non_empty(deep_find_key(p, candidate_keys))
|
||||
if val:
|
||||
summary[field] = val
|
||||
v = first_non_empty(vals)
|
||||
if v:
|
||||
summary[field] = v
|
||||
|
||||
# Передаём image_urls без повторного извлечения.
|
||||
image_urls = summary.get("image_urls") or []
|
||||
return {
|
||||
"vehicle_summary": summary,
|
||||
"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url),
|
||||
"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
|
||||
"embedded_json": embedded,
|
||||
"dom_hints": self._dom_hints(dom_text),
|
||||
"access_notes": self._build_access_notes(summary, responses),
|
||||
}
|
||||
|
||||
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "") -> dict[str, Any]:
|
||||
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
|
||||
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
|
||||
if image_urls is None:
|
||||
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
|
||||
return {
|
||||
"vehicle_core": {
|
||||
"vin": summary.get("vin"), "lot_number": summary.get("lot_number"), "year": summary.get("year"),
|
||||
"lot_number": summary.get("lot_number"), "year": summary.get("year"),
|
||||
"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
|
||||
"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
|
||||
"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
|
||||
@@ -238,7 +282,6 @@ class VehicleParser:
|
||||
endpoints = [item.get("url", "") for item in responses]
|
||||
dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
|
||||
return {
|
||||
"vin_visible": bool(summary.get("vin")),
|
||||
"images_visible": bool(summary.get("image_urls")),
|
||||
"network_json_count": len(responses),
|
||||
"possible_captcha": bool(dom_hints.get("has_captcha_text")),
|
||||
@@ -266,11 +309,6 @@ class VehicleParser:
|
||||
return "JPY"
|
||||
return "USD"
|
||||
|
||||
@staticmethod
|
||||
def _extract_vin(text: str) -> str | None:
|
||||
match = VIN_RE.search(text or "")
|
||||
return match.group(1) if match else None
|
||||
|
||||
@staticmethod
|
||||
def _extract_lot_number(text: str) -> str | None:
|
||||
match = LOT_RE.search(text or "")
|
||||
@@ -287,6 +325,9 @@ class VehicleParser:
|
||||
for script_text in scripts:
|
||||
if "{" not in script_text and "[" not in script_text:
|
||||
continue
|
||||
# Пропускаем слишком большие минифицированные блоки.
|
||||
if len(script_text) > 51_200:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(script_text.strip())
|
||||
except Exception:
|
||||
@@ -361,8 +402,7 @@ class VehicleParser:
|
||||
"has_buy_now_text": "buy now" in lowered,
|
||||
"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
|
||||
"has_damage_text": "damage" in lowered,
|
||||
"has_vin_text": "vin" in lowered,
|
||||
"has_title_text": "title" in lowered,
|
||||
"has_captcha_text": any(token in lowered for token in ["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"]),
|
||||
"has_antibot_text": any(token in lowered for token in ["incapsula", "access denied", "request unsuccessful", "bot detection"]),
|
||||
"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
|
||||
"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user