refactor mobile.de parser, fix country mapping, update README
This commit is contained in:
qananasikq
@@ -1,224 +1,7 @@
|
||||
# Роуты запуска задач синхронизации и просмотра истории sync-runs.
|
||||
"""API route package.
|
||||
|
||||
import json
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel
|
||||
from redis import Redis
|
||||
from sqlalchemy import select, func
|
||||
|
||||
from ...core.config import Settings
|
||||
from ..deps import get_persistence
|
||||
from ...storage.db import PersistenceService
|
||||
from ...storage.models import SyncRun
|
||||
from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
|
||||
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class SyncVehicleRequest(BaseModel):
|
||||
vehicle_url: str
|
||||
lane: str = "MOBILEDE"
|
||||
|
||||
|
||||
class SyncListingRequest(BaseModel):
|
||||
make: str | None = None
|
||||
model: str | None = None
|
||||
lane: str = "MOBILEDE_cars"
|
||||
limit: int | None = None
|
||||
only_new: bool | None = None
|
||||
|
||||
|
||||
class MobileDeSyncSearchRequest(BaseModel):
|
||||
start_page: int = 1
|
||||
max_pages: int = 5
|
||||
lane: str = "mobile_de_cars"
|
||||
search_url: str | None = None
|
||||
make_id: str | None = None
|
||||
model_id: str | None = None
|
||||
price_min: str | None = None
|
||||
price_max: str | None = None
|
||||
year_min: str | None = None
|
||||
year_max: str | None = None
|
||||
delay_seconds: float = 0.7
|
||||
use_cursor: bool = False
|
||||
continuous: bool = False
|
||||
|
||||
|
||||
class MobileDeSyncDetailRequest(BaseModel):
|
||||
listing_id: str
|
||||
lane: str = "mobile_de_cars"
|
||||
|
||||
|
||||
class MobileDeRuntimeSegmentsRequest(BaseModel):
|
||||
lane: str = "mobile_de_cars"
|
||||
delay_seconds: float = 0.7
|
||||
use_cursor: bool = True
|
||||
continuous: bool = False
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-search")
|
||||
def start_mobilede_sync_search(body: MobileDeSyncSearchRequest):
|
||||
result = mobilede_sync_search_task.apply_async(
|
||||
kwargs=body.model_dump(),
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-detail")
|
||||
def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
|
||||
result = mobilede_sync_detail_task.apply_async(
|
||||
kwargs=body.model_dump(),
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
"listing_id": body.listing_id,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-runtime-segments")
|
||||
def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
|
||||
result = mobilede_sync_runtime_segments_task.apply_async(
|
||||
kwargs=body.model_dump(),
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"queue": MOBILEDE_SYNC_QUEUE,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tasks/sync-vehicle")
|
||||
def start_sync_vehicle(
|
||||
body: SyncVehicleRequest,
|
||||
):
|
||||
# Запустить задачу скрапинга одного автомобиля через Celery
|
||||
result = sync_vehicle_task.apply_async(
|
||||
kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"vehicle_url": body.vehicle_url,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tasks/sync-listing")
|
||||
def start_sync_listing(
|
||||
body: SyncListingRequest,
|
||||
):
|
||||
# Запустить задачу полного цикла листинга через Celery
|
||||
result = sync_listing_task.apply_async(
|
||||
kwargs={
|
||||
"make": body.make,
|
||||
"model": body.model,
|
||||
"lane": body.lane,
|
||||
"limit": body.limit,
|
||||
"only_new": body.only_new,
|
||||
},
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/tasks/{task_id}")
|
||||
def get_task_status(task_id: str):
|
||||
result = celery_app.AsyncResult(task_id)
|
||||
|
||||
payload: dict = {
|
||||
"task_id": task_id,
|
||||
"state": result.state,
|
||||
}
|
||||
|
||||
if result.successful():
|
||||
payload["result"] = result.result
|
||||
elif result.failed():
|
||||
payload["error"] = str(result.result)
|
||||
elif result.info is not None:
|
||||
payload["meta"] = result.info
|
||||
|
||||
progress = _read_task_progress(task_id)
|
||||
if progress is not None:
|
||||
payload["progress"] = progress
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
def _read_task_progress(task_id: str) -> dict | None:
|
||||
redis_client = None
|
||||
try:
|
||||
settings = Settings()
|
||||
redis_client = Redis.from_url(
|
||||
settings.redis.url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
|
||||
socket_timeout=settings.redis.socket_timeout_seconds,
|
||||
health_check_interval=settings.redis.health_check_interval_seconds,
|
||||
retry_on_timeout=True,
|
||||
)
|
||||
raw = redis_client.get(f"mobilede:state:task_progress:{task_id}")
|
||||
if not raw:
|
||||
return None
|
||||
data = json.loads(raw)
|
||||
return data if isinstance(data, dict) else None
|
||||
except Exception:
|
||||
return None
|
||||
finally:
|
||||
if redis_client is not None:
|
||||
try:
|
||||
redis_client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@router.get("/sync-runs")
|
||||
def list_sync_runs(
|
||||
page: int = Query(1, ge=1),
|
||||
per_page: int = Query(20, ge=1, le=100),
|
||||
persistence: PersistenceService = Depends(get_persistence),
|
||||
):
|
||||
# Рстория запусков синхронизации
|
||||
with persistence.session_scope() as session:
|
||||
total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
|
||||
offset = (page - 1) * per_page
|
||||
runs = session.execute(
|
||||
select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
|
||||
).scalars().all()
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"items": [
|
||||
{
|
||||
"id": r.id,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||
"status": r.status,
|
||||
"lane": r.lane,
|
||||
"ids_fetched": r.ids_fetched,
|
||||
"cars_upserted": r.cars_upserted,
|
||||
"cars_failed": r.cars_failed,
|
||||
"images_upserted": r.images_upserted,
|
||||
"error_summary": r.error_summary,
|
||||
}
|
||||
for r in runs
|
||||
],
|
||||
}
|
||||
Routers are defined in dedicated modules (`cars`, `health`, `tasks`). This
|
||||
package intentionally does not create routes to avoid duplicate task endpoints
|
||||
and accidental imports of legacy parser channels during FastAPI startup.
|
||||
"""
|
||||
|
||||
|
||||
@@ -11,25 +11,12 @@ from ...core.config import Settings
|
||||
from ..deps import get_persistence
|
||||
from ...storage.db import PersistenceService
|
||||
from ...storage.models import SyncRun
|
||||
from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, MOBILEDE_SYNC_QUEUE, celery_app
|
||||
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task, sync_vehicle_task, sync_listing_task
|
||||
from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, celery_app
|
||||
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class SyncVehicleRequest(BaseModel):
|
||||
vehicle_url: str
|
||||
lane: str = "MOBILEDE"
|
||||
|
||||
|
||||
class SyncListingRequest(BaseModel):
|
||||
make: str | None = None
|
||||
model: str | None = None
|
||||
lane: str = "MOBILEDE_cars"
|
||||
limit: int | None = None
|
||||
only_new: bool | None = None
|
||||
|
||||
|
||||
class MobileDeSyncSearchRequest(BaseModel):
|
||||
start_page: int = 1
|
||||
max_pages: int = 5
|
||||
@@ -43,7 +30,7 @@ class MobileDeSyncSearchRequest(BaseModel):
|
||||
year_max: str | None = None
|
||||
delay_seconds: float = 0.7
|
||||
use_cursor: bool = False
|
||||
continuous: bool | None = None
|
||||
continuous: bool | None = False
|
||||
|
||||
|
||||
class MobileDeSyncDetailRequest(BaseModel):
|
||||
@@ -55,7 +42,7 @@ class MobileDeRuntimeSegmentsRequest(BaseModel):
|
||||
lane: str = "mobile_de_cars"
|
||||
delay_seconds: float = 0.7
|
||||
use_cursor: bool = True
|
||||
continuous: bool | None = None
|
||||
continuous: bool | None = False
|
||||
|
||||
|
||||
@router.post("/mobilede/tasks/sync-search")
|
||||
@@ -98,45 +85,6 @@ def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tasks/sync-vehicle")
|
||||
def start_sync_vehicle(
|
||||
body: SyncVehicleRequest,
|
||||
):
|
||||
# Запустить задачу скрапинга одного автомобиля через Celery
|
||||
result = sync_vehicle_task.apply_async(
|
||||
kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"vehicle_url": body.vehicle_url,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tasks/sync-listing")
|
||||
def start_sync_listing(
|
||||
body: SyncListingRequest,
|
||||
):
|
||||
# Запустить задачу полного цикла листинга через Celery
|
||||
result = sync_listing_task.apply_async(
|
||||
kwargs={
|
||||
"make": body.make,
|
||||
"model": body.model,
|
||||
"lane": body.lane,
|
||||
"limit": body.limit,
|
||||
"only_new": body.only_new,
|
||||
},
|
||||
queue=MOBILEDE_SYNC_QUEUE,
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/tasks/{task_id}")
|
||||
def get_task_status(task_id: str):
|
||||
result = celery_app.AsyncResult(task_id)
|
||||
|
||||
@@ -1,6 +0,0 @@
|
||||
from .factory import BrowserFactory
|
||||
from .listing import ListingCollector
|
||||
from .network import NetworkCapture
|
||||
from .pace import HumanPacer
|
||||
|
||||
__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]
|
||||
@@ -1,214 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
|
||||
from playwright.sync_api import Browser, BrowserContext, Playwright
|
||||
|
||||
try:
|
||||
from playwright_stealth import stealth_sync
|
||||
except ImportError:
|
||||
stealth_sync = None
|
||||
|
||||
from ..core.config import Settings
|
||||
|
||||
logger = logging.getLogger("MOBILEDE_scraper.browser")
|
||||
|
||||
|
||||
def _build_init_script() -> str:
|
||||
# Маскировка браузера.
|
||||
hardware_concurrency = random.choice([4, 8, 12, 16])
|
||||
device_memory = random.choice([4, 8, 16])
|
||||
languages = ["en-US", "en"]
|
||||
|
||||
return f"""
|
||||
(() => {{
|
||||
const define = (obj, prop, value) => {{
|
||||
try {{
|
||||
Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
|
||||
}} catch (e) {{}}
|
||||
}};
|
||||
|
||||
define(navigator, 'webdriver', undefined);
|
||||
define(navigator, 'platform', 'Win32');
|
||||
define(navigator, 'vendor', 'Google Inc.');
|
||||
define(navigator, 'language', '{languages[0]}');
|
||||
define(navigator, 'languages', {json.dumps(languages)});
|
||||
define(navigator, 'hardwareConcurrency', {hardware_concurrency});
|
||||
define(navigator, 'deviceMemory', {device_memory});
|
||||
define(navigator, 'maxTouchPoints', 0);
|
||||
|
||||
if (!window.chrome) {{
|
||||
Object.defineProperty(window, 'chrome', {{
|
||||
value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
|
||||
configurable: true
|
||||
}});
|
||||
}}
|
||||
|
||||
const originalQuery = navigator.permissions && navigator.permissions.query;
|
||||
if (originalQuery) {{
|
||||
navigator.permissions.query = (params) => (
|
||||
params && params.name === 'notifications'
|
||||
? Promise.resolve({{ state: Notification.permission }})
|
||||
: originalQuery(params)
|
||||
);
|
||||
}}
|
||||
|
||||
const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
|
||||
WebGLRenderingContext.prototype.getParameter = function(parameter) {{
|
||||
if (parameter === 37445) return 'Intel Inc.';
|
||||
if (parameter === 37446) return 'Intel Iris OpenGL Engine';
|
||||
return originalGetParameter.call(this, parameter);
|
||||
}};
|
||||
}})();
|
||||
"""
|
||||
|
||||
|
||||
class BrowserFactory:
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.settings = settings
|
||||
|
||||
def _resolve_engine(self) -> str:
|
||||
# Выбор движка.
|
||||
engine = self.settings.browser_engine.strip().lower()
|
||||
if engine == "auto":
|
||||
# Для MOBILEDE стабильнее Chromium.
|
||||
return "chromium"
|
||||
if engine in ("firefox", "chromium"):
|
||||
return engine
|
||||
logger.warning("Unknown MOBILEDE_BROWSER_ENGINE=%r, falling back to auto", engine)
|
||||
return "chromium"
|
||||
|
||||
def create_browser(self, playwright: Playwright) -> Browser:
|
||||
engine = self._resolve_engine()
|
||||
proxy_dict = self.settings.proxy.to_playwright_dict()
|
||||
logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
|
||||
|
||||
if engine == "firefox":
|
||||
launch_kwargs: dict = {"headless": self.settings.headless}
|
||||
if proxy_dict:
|
||||
launch_kwargs["proxy"] = proxy_dict
|
||||
logger.info("Using proxy: %s", self.settings.proxy.server)
|
||||
# Настройки Firefox.
|
||||
launch_kwargs["firefox_user_prefs"] = {
|
||||
"dom.webdriver.enabled": False,
|
||||
"useAutomationExtension": False,
|
||||
# Базовые оптимизации.
|
||||
"media.autoplay.default": 5,
|
||||
"media.volume_scale": "0.0",
|
||||
"media.audio.playback.standalone": False,
|
||||
"dom.ipc.processCount": 1,
|
||||
"dom.ipc.plugins.enabled": False,
|
||||
"browser.cache.disk.enable": False,
|
||||
"browser.cache.memory.enable": True,
|
||||
"browser.cache.memory.max_entry_size": 8192,
|
||||
"network.prefetch-next": False,
|
||||
"network.dns.disablePrefetch": True,
|
||||
"permissions.default.image": 2,
|
||||
"javascript.options.mem.gc_incremental_mark_slice_ms": 20,
|
||||
}
|
||||
logger.info("Launching Firefox (headless=%s)", self.settings.headless)
|
||||
return playwright.firefox.launch(**launch_kwargs)
|
||||
|
||||
# Запуск Chromium.
|
||||
args = [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--no-default-browser-check",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-features=IsolateOrigins,site-per-process",
|
||||
]
|
||||
if self.settings.headless:
|
||||
args.append("--headless=new")
|
||||
pw_headless = False
|
||||
logger.info("Using Chromium new-headless mode (--headless=new)")
|
||||
else:
|
||||
pw_headless = False
|
||||
|
||||
launch_kwargs = {"headless": pw_headless, "args": args}
|
||||
if proxy_dict:
|
||||
launch_kwargs["proxy"] = proxy_dict
|
||||
logger.info("Using proxy: %s", self.settings.proxy.server)
|
||||
try:
|
||||
logger.info("Trying to launch real Chrome channel")
|
||||
return playwright.chromium.launch(channel="chrome", **launch_kwargs)
|
||||
except Exception:
|
||||
logger.warning("Chrome channel launch failed, falling back to Chromium")
|
||||
return playwright.chromium.launch(**launch_kwargs)
|
||||
|
||||
def create_context(self, browser: Browser) -> BrowserContext:
|
||||
viewport = random.choice(self.settings.fingerprint.viewport_presets)
|
||||
timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
|
||||
color_scheme = random.choice(["light", "dark"])
|
||||
|
||||
is_firefox = browser.browser_type.name == "firefox"
|
||||
|
||||
ctx_kwargs: dict = {
|
||||
"viewport": viewport,
|
||||
"screen": viewport,
|
||||
"locale": self.settings.fingerprint.locale,
|
||||
"timezone_id": timezone_id,
|
||||
"color_scheme": color_scheme,
|
||||
"java_script_enabled": True,
|
||||
"ignore_https_errors": False,
|
||||
}
|
||||
|
||||
if is_firefox:
|
||||
# Заголовки Firefox.
|
||||
ctx_kwargs["user_agent"] = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
|
||||
"Gecko/20100101 Firefox/128.0"
|
||||
)
|
||||
ctx_kwargs["extra_http_headers"] = {
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"DNT": "1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
else:
|
||||
ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
|
||||
ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
|
||||
ctx_kwargs["is_mobile"] = False
|
||||
ctx_kwargs["has_touch"] = False
|
||||
ctx_kwargs["extra_http_headers"] = {
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"DNT": "1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
|
||||
"Sec-CH-UA-Mobile": "?0",
|
||||
"Sec-CH-UA-Platform": '"Windows"',
|
||||
}
|
||||
|
||||
context = browser.new_context(**ctx_kwargs)
|
||||
context.set_default_timeout(self.settings.default_timeout_ms)
|
||||
context.set_default_navigation_timeout(self.settings.default_timeout_ms)
|
||||
|
||||
if not is_firefox:
|
||||
# Маскировка Chromium.
|
||||
context.add_init_script(_build_init_script())
|
||||
if stealth_sync:
|
||||
context.on("page", lambda page: stealth_sync(page))
|
||||
logger.debug("playwright-stealth attached to context")
|
||||
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def enable_resource_blocking(page) -> None:
|
||||
# Блокируем тяжёлые ресурсы.
|
||||
BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
|
||||
BLOCKED_URL_PATTERNS = (
|
||||
"google-analytics", "googletagmanager", "facebook.net",
|
||||
"doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
|
||||
"analytics", "tracking", "adservice",
|
||||
)
|
||||
|
||||
def _handle_route(route):
|
||||
req = route.request
|
||||
if req.resource_type in BLOCKED_TYPES:
|
||||
route.abort()
|
||||
return
|
||||
url = req.url.lower()
|
||||
if any(pat in url for pat in BLOCKED_URL_PATTERNS):
|
||||
route.abort()
|
||||
return
|
||||
route.continue_()
|
||||
|
||||
page.route("**/*", _handle_route)
|
||||
@@ -1,863 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import html
|
||||
import json
|
||||
import logging
|
||||
import math
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Iterator
|
||||
from urllib.parse import quote, urljoin
|
||||
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
|
||||
from ..core.config import Settings
|
||||
|
||||
logger = logging.getLogger("MOBILEDE_scraper.fast_client")
|
||||
|
||||
TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
|
||||
CHALLENGE_MARKERS = (
|
||||
"_incapsula_resource",
|
||||
"incapsula",
|
||||
"incident id",
|
||||
"request unsuccessful",
|
||||
"access denied",
|
||||
)
|
||||
COOKIE_ACCEPT_SELECTORS = (
|
||||
"button:has-text('Accept All')",
|
||||
"button:has-text('Accept all')",
|
||||
"button:has-text('I Agree')",
|
||||
"button:has-text('Agree')",
|
||||
"button:has-text('Only necessary')",
|
||||
"button:has-text('Только необходимые')",
|
||||
"button:has-text('Принять все')",
|
||||
"[id*='accept']",
|
||||
"[class*='accept']",
|
||||
)
|
||||
LISTING_MARKER = 'id="GBPSearchQuery"'
|
||||
DETAIL_MARKER = 'id="ProductDetailsVM"'
|
||||
RESIZER_URL = "https://vis.MOBILEDE.com/resizer"
|
||||
BRAND_SCOPE_OVERRIDES = {
|
||||
# MOBILEDE does not resolve every rare make through /Vehiclelisting/Cars/{make}.
|
||||
# CUPRA is available through a saved Search scope URL from the site UI.
|
||||
"CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d",
|
||||
}
|
||||
DEFAULT_USER_AGENT = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
)
|
||||
PLAYWRIGHT_REFRESH_POLLS = 8
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FastListingVehicle:
|
||||
inventory_id: str
|
||||
tenant: str | None
|
||||
auction_id: str | None
|
||||
auction_date: str | None
|
||||
inventory_status: str | None
|
||||
currency: str | None
|
||||
timed_auction_closed: bool
|
||||
timed_auction_indicator: bool
|
||||
prebid_indicator: bool
|
||||
buynow_indicator: bool
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class FastListingPage:
|
||||
vehicles: list[FastListingVehicle]
|
||||
result_count: int
|
||||
page_size: int
|
||||
current_page: int
|
||||
gbp_search_query: dict[str, Any]
|
||||
|
||||
|
||||
class HybridSessionAuth:
|
||||
"""Requests session with Playwright cookie refresh fallback.
|
||||
|
||||
Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot
|
||||
cookies when MOBILEDE returns a challenge or an expected hidden payload is absent.
|
||||
"""
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._settings = settings
|
||||
self._thread_local = threading.local()
|
||||
self._lock = threading.Lock()
|
||||
self._refresh_lock = threading.Lock()
|
||||
self._bootstrap_cookies_loaded = False
|
||||
self._anonymous_bootstrap_attempted = False
|
||||
self._refresh_generation = 0
|
||||
self._latest_refresh_cookies: list[dict[str, Any]] = []
|
||||
|
||||
def request(
|
||||
self,
|
||||
method: str,
|
||||
url: str,
|
||||
*,
|
||||
timeout: int,
|
||||
retries: int,
|
||||
retry_backoff_ms: int,
|
||||
headers: dict[str, str] | None = None,
|
||||
data: Any | None = None,
|
||||
json_body: Any | None = None,
|
||||
expected_marker: str | None = None,
|
||||
) -> requests.Response:
|
||||
session = self._get_session()
|
||||
self._ensure_anonymous_session_bootstrap(session=session)
|
||||
self._sync_session_with_latest_refresh(session)
|
||||
last_error: Exception | None = None
|
||||
refresh_attempts = 0
|
||||
attempt = 0
|
||||
max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts))
|
||||
|
||||
while attempt <= retries:
|
||||
try:
|
||||
request_started_at = time.perf_counter()
|
||||
if self._settings.scraping_profile.verbose_http_logs:
|
||||
logger.debug(
|
||||
"HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s",
|
||||
method,
|
||||
url,
|
||||
attempt + 1,
|
||||
retries + 1,
|
||||
timeout,
|
||||
expected_marker,
|
||||
)
|
||||
response = session.request(
|
||||
method=method,
|
||||
url=url,
|
||||
headers=headers,
|
||||
data=data,
|
||||
json=json_body,
|
||||
timeout=(min(10, max(1, timeout)), max(1, timeout)),
|
||||
)
|
||||
if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400:
|
||||
logger.debug(
|
||||
"HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s",
|
||||
method,
|
||||
url,
|
||||
response.status_code,
|
||||
time.perf_counter() - request_started_at,
|
||||
expected_marker,
|
||||
)
|
||||
except requests.RequestException as exc:
|
||||
last_error = exc
|
||||
if attempt >= retries:
|
||||
break
|
||||
self._sleep_backoff(retry_backoff_ms, attempt)
|
||||
attempt += 1
|
||||
continue
|
||||
|
||||
if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries:
|
||||
response.close()
|
||||
self._sleep_backoff(retry_backoff_ms, attempt)
|
||||
attempt += 1
|
||||
continue
|
||||
|
||||
if is_challenge_response(
|
||||
status_code=response.status_code,
|
||||
body_text=response.text,
|
||||
expected_marker=expected_marker,
|
||||
):
|
||||
response.close()
|
||||
if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts:
|
||||
raise RuntimeError(
|
||||
"MOBILEDE challenge persisted after "
|
||||
f"{refresh_attempts} Playwright refresh attempts for url={url}"
|
||||
)
|
||||
refresh_attempts += 1
|
||||
logger.info(
|
||||
"Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s",
|
||||
url,
|
||||
response.status_code,
|
||||
expected_marker,
|
||||
refresh_attempts,
|
||||
max_refresh_attempts,
|
||||
)
|
||||
self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
|
||||
logger.info("Retrying HTTP request after Playwright refresh url=%s", url)
|
||||
if refresh_attempts > 1:
|
||||
self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
|
||||
continue
|
||||
|
||||
return response
|
||||
|
||||
if last_error is not None:
|
||||
raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error
|
||||
raise RuntimeError(f"Request failed url={url} after retries")
|
||||
|
||||
def persist_storage_state(self) -> None:
|
||||
session = self._get_session()
|
||||
with self._lock:
|
||||
self._save_storage_state(session)
|
||||
|
||||
def _get_session(self) -> requests.Session:
|
||||
session = getattr(self._thread_local, "session", None)
|
||||
if session is None:
|
||||
session = requests.Session()
|
||||
pool_size = max(20, int(self._settings.fetch_concurrency) * 2)
|
||||
adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size)
|
||||
session.mount("http://", adapter)
|
||||
session.mount("https://", adapter)
|
||||
session.headers.update(
|
||||
{
|
||||
"user-agent": DEFAULT_USER_AGENT,
|
||||
"accept-language": "en-US,en;q=0.9",
|
||||
"cache-control": "no-cache",
|
||||
"pragma": "no-cache",
|
||||
}
|
||||
)
|
||||
if self._settings.proxy.enabled:
|
||||
proxies = self._settings.proxy.to_requests_proxies()
|
||||
if proxies:
|
||||
session.proxies.update(proxies)
|
||||
with self._lock:
|
||||
self._bootstrap_session_cookies(session)
|
||||
self._thread_local.session = session
|
||||
self._thread_local.session_generation = 0
|
||||
self._sync_session_with_latest_refresh(session)
|
||||
return session
|
||||
|
||||
def _sync_session_with_latest_refresh(self, session: requests.Session) -> None:
|
||||
with self._lock:
|
||||
latest_generation = self._refresh_generation
|
||||
session_generation = getattr(self._thread_local, "session_generation", 0)
|
||||
if latest_generation <= session_generation or not self._latest_refresh_cookies:
|
||||
return
|
||||
cookies = list(self._latest_refresh_cookies)
|
||||
self._apply_cookies_to_session(session, cookies)
|
||||
self._thread_local.session_generation = latest_generation
|
||||
|
||||
def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None:
|
||||
if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled:
|
||||
return
|
||||
if self._session_has_MOBILEDE_cookies(session):
|
||||
self._anonymous_bootstrap_attempted = True
|
||||
return
|
||||
with self._lock:
|
||||
if self._anonymous_bootstrap_attempted:
|
||||
return
|
||||
self._anonymous_bootstrap_attempted = True
|
||||
logger.info("No MOBILEDE cookies preloaded. Attempting anonymous session bootstrap via Playwright.")
|
||||
try:
|
||||
self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
|
||||
except Exception as exc:
|
||||
logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc)
|
||||
|
||||
@staticmethod
|
||||
def _session_has_MOBILEDE_cookies(session: requests.Session) -> bool:
|
||||
for item in session.cookies:
|
||||
domain = str(getattr(item, "domain", "") or "")
|
||||
if not domain or "MOBILEDE.com" in domain.lower():
|
||||
return True
|
||||
return False
|
||||
|
||||
def _bootstrap_session_cookies(self, session: requests.Session) -> None:
|
||||
if self._bootstrap_cookies_loaded:
|
||||
return
|
||||
self._load_storage_state_cookies(session)
|
||||
self._bootstrap_cookies_loaded = True
|
||||
|
||||
def _load_storage_state_cookies(self, session: requests.Session) -> None:
|
||||
tokens_file = self._settings.tokens_file
|
||||
if not tokens_file:
|
||||
return
|
||||
path = __import__("pathlib").Path(tokens_file)
|
||||
if not path.exists():
|
||||
return
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to read storage state file '%s': %s", path, exc)
|
||||
return
|
||||
cookies = payload.get("cookies") if isinstance(payload, dict) else None
|
||||
if not isinstance(cookies, list):
|
||||
return
|
||||
applied = 0
|
||||
for item in cookies:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
name = parse_text(item.get("name"))
|
||||
value = parse_text(item.get("value"))
|
||||
if not name or value is None:
|
||||
continue
|
||||
domain = parse_text(item.get("domain")) or ".MOBILEDE.com"
|
||||
cookie_path = parse_text(item.get("path")) or "/"
|
||||
expires = parse_int(item.get("expires"))
|
||||
session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
|
||||
applied += 1
|
||||
if applied:
|
||||
logger.info("Loaded %s cookies from storage state", applied)
|
||||
|
||||
def _refresh_session_via_playwright(
|
||||
self,
|
||||
*,
|
||||
expected_marker: str | None = None,
|
||||
session: requests.Session | None = None,
|
||||
) -> None:
|
||||
target_session = session or self._get_session()
|
||||
with self._lock:
|
||||
baseline_generation = self._refresh_generation
|
||||
|
||||
with self._refresh_lock:
|
||||
with self._lock:
|
||||
if self._refresh_generation > baseline_generation and self._latest_refresh_cookies:
|
||||
self._apply_cookies_to_session(target_session, self._latest_refresh_cookies)
|
||||
self._thread_local.session_generation = self._refresh_generation
|
||||
return
|
||||
|
||||
logger.info("MOBILEDE session challenge detected. Refreshing session via Playwright.")
|
||||
cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
|
||||
logger.info("Playwright refresh returned %d cookies", len(cookies))
|
||||
self._apply_cookies_to_session(target_session, cookies)
|
||||
logger.info("Playwright cookies applied to requests session")
|
||||
|
||||
with self._lock:
|
||||
self._refresh_generation += 1
|
||||
self._latest_refresh_cookies = list(cookies)
|
||||
self._anonymous_bootstrap_attempted = True
|
||||
refreshed_generation = self._refresh_generation
|
||||
self._thread_local.session_generation = refreshed_generation
|
||||
logger.info("Playwright refresh completed generation=%s", refreshed_generation)
|
||||
|
||||
def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
|
||||
from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
|
||||
from playwright.sync_api import sync_playwright
|
||||
|
||||
with sync_playwright() as playwright:
|
||||
browser = playwright.chromium.launch(headless=self._settings.headless)
|
||||
try:
|
||||
context = browser.new_context(
|
||||
locale=self._settings.fingerprint.locale,
|
||||
viewport={"width": 1366, "height": 768},
|
||||
user_agent=DEFAULT_USER_AGENT,
|
||||
proxy=self._settings.proxy.to_playwright_dict(),
|
||||
)
|
||||
page = context.new_page()
|
||||
home_target = self._settings.home_url
|
||||
filtered_urls = self._settings.listing.filtered_search_urls
|
||||
target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars")
|
||||
timeout_ms = max(30_000, self._settings.default_timeout_ms)
|
||||
logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER)
|
||||
page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
|
||||
self._accept_cookie_banner(page)
|
||||
page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
|
||||
self._accept_cookie_banner(page)
|
||||
self._wait_until_non_challenge(
|
||||
page=page,
|
||||
target=target,
|
||||
timeout_ms=timeout_ms,
|
||||
expected_marker=expected_marker or LISTING_MARKER,
|
||||
)
|
||||
cookies = context.cookies()
|
||||
logger.info("Playwright context returned %d cookies", len(cookies))
|
||||
except PlaywrightTimeoutError as exc:
|
||||
raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
|
||||
finally:
|
||||
try:
|
||||
browser.close()
|
||||
except Exception as exc:
|
||||
logger.info("Playwright browser close failed after cookie refresh: %s", exc)
|
||||
|
||||
if not isinstance(cookies, list) or not cookies:
|
||||
raise RuntimeError("Playwright refresh did not return cookies")
|
||||
return [cookie for cookie in cookies if isinstance(cookie, dict)]
|
||||
|
||||
@staticmethod
|
||||
def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None:
|
||||
session.cookies.clear()
|
||||
for cookie in cookies:
|
||||
name = parse_text(cookie.get("name"))
|
||||
value = parse_text(cookie.get("value"))
|
||||
if not name or value is None:
|
||||
continue
|
||||
domain = parse_text(cookie.get("domain")) or ".MOBILEDE.com"
|
||||
cookie_path = parse_text(cookie.get("path")) or "/"
|
||||
expires = parse_int(cookie.get("expires"))
|
||||
session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
|
||||
|
||||
@staticmethod
|
||||
def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None:
|
||||
poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
|
||||
navigation_error_count = 0
|
||||
for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
|
||||
except Exception:
|
||||
pass
|
||||
page.wait_for_timeout(poll_ms)
|
||||
body: str | None = None
|
||||
for _ in range(3):
|
||||
try:
|
||||
body = page.content()
|
||||
break
|
||||
except Exception as exc:
|
||||
message = str(exc).lower()
|
||||
if "page.content" not in message or "navigating and changing the content" not in message:
|
||||
raise
|
||||
navigation_error_count += 1
|
||||
page.wait_for_timeout(max(200, poll_ms // 4))
|
||||
if body is not None and not is_challenge_response(
|
||||
status_code=200,
|
||||
body_text=body,
|
||||
expected_marker=expected_marker,
|
||||
):
|
||||
logger.info(
|
||||
"Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s",
|
||||
target,
|
||||
poll_index + 1,
|
||||
PLAYWRIGHT_REFRESH_POLLS,
|
||||
expected_marker,
|
||||
)
|
||||
return
|
||||
try:
|
||||
logger.info(
|
||||
"Playwright session refresh still waiting target=%s poll=%s/%s marker=%s",
|
||||
target,
|
||||
poll_index + 1,
|
||||
PLAYWRIGHT_REFRESH_POLLS,
|
||||
expected_marker,
|
||||
)
|
||||
page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
|
||||
except Exception:
|
||||
pass
|
||||
raise RuntimeError(
|
||||
"Playwright refresh completed but challenge page is still active "
|
||||
f"(navigation_content_errors={navigation_error_count})"
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _accept_cookie_banner(page: Any) -> None:
|
||||
for selector in COOKIE_ACCEPT_SELECTORS:
|
||||
try:
|
||||
locator = page.locator(selector).first
|
||||
if locator.count() == 0 or not locator.is_visible(timeout=500):
|
||||
continue
|
||||
locator.click(timeout=2_000)
|
||||
page.wait_for_timeout(250)
|
||||
return
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
def _save_storage_state(self, session: requests.Session) -> None:
|
||||
tokens_file = self._settings.tokens_file
|
||||
if not tokens_file:
|
||||
return
|
||||
path = __import__("pathlib").Path(tokens_file)
|
||||
cookies: list[dict[str, Any]] = []
|
||||
for cookie in session.cookies:
|
||||
payload: dict[str, Any] = {
|
||||
"name": cookie.name,
|
||||
"value": cookie.value,
|
||||
"domain": cookie.domain or ".MOBILEDE.com",
|
||||
"path": cookie.path or "/",
|
||||
"httpOnly": False,
|
||||
"secure": bool(cookie.secure),
|
||||
"sameSite": "Lax",
|
||||
}
|
||||
if cookie.expires is not None:
|
||||
payload["expires"] = int(cookie.expires)
|
||||
cookies.append(payload)
|
||||
try:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
except PermissionError as exc:
|
||||
logger.info("Cannot persist MOBILEDE storage state to '%s': %s", path, exc)
|
||||
except OSError as exc:
|
||||
logger.info("Failed to persist MOBILEDE storage state to '%s': %s", path, exc)
|
||||
|
||||
@staticmethod
|
||||
def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None:
|
||||
if retry_backoff_ms <= 0:
|
||||
return
|
||||
time.sleep(retry_backoff_ms * (2**attempt) / 1000)
|
||||
|
||||
|
||||
class MobiledeFastClient:
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self._settings = settings
|
||||
self._auth = HybridSessionAuth(settings)
|
||||
|
||||
def persist_session_state(self) -> None:
|
||||
self._auth.persist_storage_state()
|
||||
|
||||
def iter_listing_vehicles(
|
||||
self,
|
||||
*,
|
||||
listing_start_url: str | None = None,
|
||||
make: str | None = None,
|
||||
max_pages: int | None = None,
|
||||
) -> Iterator[FastListingVehicle]:
|
||||
seen_inventory_ids: set[str] = set()
|
||||
scope_paths = resolve_listing_scope_paths(
|
||||
listing_start_url=listing_start_url or "",
|
||||
brands={make} if make else set(),
|
||||
)
|
||||
for scope_path in scope_paths:
|
||||
first_page_html = self._fetch_listing_first_page(scope_path)
|
||||
search_scope_path = build_search_scope_path_from_html(first_page_html)
|
||||
if search_scope_path and search_scope_path != scope_path:
|
||||
logger.info(
|
||||
"Resolved listing scope to fast Search URL: scope=%s search_scope=%s",
|
||||
scope_path,
|
||||
search_scope_path,
|
||||
)
|
||||
scope_path = search_scope_path
|
||||
first_page = parse_listing_page(first_page_html)
|
||||
for vehicle in first_page.vehicles:
|
||||
if vehicle.inventory_id in seen_inventory_ids:
|
||||
continue
|
||||
seen_inventory_ids.add(vehicle.inventory_id)
|
||||
yield vehicle
|
||||
|
||||
page_size = max(1, first_page.page_size)
|
||||
total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size))
|
||||
if max_pages is not None and max_pages > 0:
|
||||
total_pages = min(total_pages, max_pages)
|
||||
gbp_search_query = first_page.gbp_search_query
|
||||
for page_number in range(2, total_pages + 1):
|
||||
page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size)
|
||||
parsed_page = parse_listing_page(page_html)
|
||||
gbp_search_query = parsed_page.gbp_search_query
|
||||
for vehicle in parsed_page.vehicles:
|
||||
if vehicle.inventory_id in seen_inventory_ids:
|
||||
continue
|
||||
seen_inventory_ids.add(vehicle.inventory_id)
|
||||
yield vehicle
|
||||
|
||||
def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]:
|
||||
escaped_id = quote(inventory_id, safe="~")
|
||||
url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
|
||||
response = self._auth.request(
|
||||
"GET",
|
||||
url,
|
||||
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
|
||||
retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
|
||||
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
|
||||
headers={"accept": "text/html,application/xhtml+xml"},
|
||||
expected_marker=DETAIL_MARKER,
|
||||
)
|
||||
with response:
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
|
||||
return parse_product_details_vm(response.text)
|
||||
|
||||
def fetch_vehicle_detail_html(self, inventory_id: str) -> str:
|
||||
escaped_id = quote(inventory_id, safe="~")
|
||||
url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
|
||||
response = self._auth.request(
|
||||
"GET",
|
||||
url,
|
||||
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
|
||||
retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
|
||||
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
|
||||
headers={"accept": "text/html,application/xhtml+xml"},
|
||||
expected_marker=DETAIL_MARKER,
|
||||
)
|
||||
with response:
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
|
||||
return response.text
|
||||
|
||||
def _fetch_listing_first_page(self, scope_path: str) -> str:
|
||||
url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/"))
|
||||
response = self._auth.request(
|
||||
"GET",
|
||||
url,
|
||||
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
|
||||
retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
|
||||
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
|
||||
headers={"accept": "text/html,application/xhtml+xml"},
|
||||
expected_marker=LISTING_MARKER,
|
||||
)
|
||||
with response:
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}")
|
||||
return response.text
|
||||
|
||||
def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str:
|
||||
query_payload = dict(gbp_search_query)
|
||||
query_payload["CurrentPage"] = page_number
|
||||
query_payload["PageSize"] = page_size
|
||||
search_url = urljoin(self._settings.home_url, "Search")
|
||||
common_headers = {
|
||||
"accept": "text/html,application/xhtml+xml,*/*",
|
||||
"x-requested-with": "XMLHttpRequest",
|
||||
}
|
||||
attempts: list[tuple[dict[str, str], Any, Any]] = [
|
||||
({**common_headers, "content-type": "application/json"}, None, query_payload),
|
||||
({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}),
|
||||
({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None),
|
||||
({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None),
|
||||
]
|
||||
attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))]
|
||||
last_error: Exception | None = None
|
||||
for headers, data, json_body in attempts:
|
||||
try:
|
||||
response = self._auth.request(
|
||||
"POST",
|
||||
search_url,
|
||||
timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
|
||||
retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
|
||||
retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
|
||||
headers=headers,
|
||||
data=data,
|
||||
json_body=json_body,
|
||||
expected_marker=LISTING_MARKER,
|
||||
)
|
||||
with response:
|
||||
if response.status_code >= 400:
|
||||
raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}")
|
||||
body = response.text
|
||||
if LISTING_MARKER not in body:
|
||||
raise RuntimeError("Listing page response does not include GBPSearchQuery")
|
||||
return body
|
||||
except Exception as exc:
|
||||
last_error = exc
|
||||
continue
|
||||
if last_error is not None:
|
||||
raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error
|
||||
raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}")
|
||||
|
||||
|
||||
def build_brand_scope_paths(brands: set[str]) -> list[str]:
|
||||
if not brands:
|
||||
return ["/Vehiclelisting/Cars"]
|
||||
paths: list[str] = []
|
||||
for brand in sorted(brands):
|
||||
raw = brand.strip()
|
||||
if not raw:
|
||||
continue
|
||||
override = BRAND_SCOPE_OVERRIDES.get(raw.upper())
|
||||
if override:
|
||||
if override not in paths:
|
||||
paths.append(override)
|
||||
continue
|
||||
slug_hyphen = quote(raw.replace(" ", "-"), safe="-")
|
||||
slug_raw = quote(raw, safe="")
|
||||
for slug in (slug_hyphen, slug_raw):
|
||||
path = f"/Vehiclelisting/Cars/{slug}"
|
||||
if path not in paths:
|
||||
paths.append(path)
|
||||
return paths or ["/Vehiclelisting/Cars"]
|
||||
|
||||
|
||||
def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
|
||||
explicit_scope = listing_start_url.strip()
|
||||
if explicit_scope:
|
||||
if explicit_scope.lower().startswith(("http://", "https://", "/")):
|
||||
return [explicit_scope]
|
||||
return [f"/Search?url={explicit_scope}"]
|
||||
return build_brand_scope_paths(brands)
|
||||
|
||||
|
||||
def build_search_scope_path_from_html(html_text: str) -> str | None:
|
||||
tiny_url = parse_attribute_value(html_text, "data-tinyurl")
|
||||
if tiny_url:
|
||||
return f"/Search?url={tiny_url}"
|
||||
|
||||
data_query_raw = parse_attribute_value(html_text, "data-query")
|
||||
if data_query_raw:
|
||||
try:
|
||||
data_query = json.loads(data_query_raw)
|
||||
except (json.JSONDecodeError, ValueError, TypeError):
|
||||
data_query = None
|
||||
if isinstance(data_query, dict):
|
||||
url_value = parse_text(data_query.get("Url"))
|
||||
if url_value:
|
||||
return f"/Search?url={url_value}"
|
||||
return None
|
||||
|
||||
|
||||
def parse_listing_page(html_text: str) -> FastListingPage:
|
||||
gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
|
||||
vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
|
||||
result_count_raw = parse_hidden_input_value(html_text, "ResultCount")
|
||||
page_size_raw = parse_hidden_input_value(html_text, "PageSize")
|
||||
current_page_raw = parse_hidden_input_value(html_text, "CurrentPage")
|
||||
if not gbp_raw:
|
||||
raise RuntimeError("Listing page missing GBPSearchQuery")
|
||||
if vehicle_raw is None:
|
||||
raise RuntimeError("Listing page missing VehicleDetails")
|
||||
gbp_payload = json.loads(gbp_raw)
|
||||
if not isinstance(gbp_payload, dict):
|
||||
raise RuntimeError("GBPSearchQuery payload is not object")
|
||||
vehicle_payload = json.loads(vehicle_raw)
|
||||
if not isinstance(vehicle_payload, list):
|
||||
raise RuntimeError("VehicleDetails payload is not array")
|
||||
|
||||
vehicles: list[FastListingVehicle] = []
|
||||
for item in vehicle_payload:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
inventory_id = parse_text(item.get("Id"))
|
||||
if not inventory_id:
|
||||
continue
|
||||
vehicles.append(
|
||||
FastListingVehicle(
|
||||
inventory_id=inventory_id,
|
||||
tenant=parse_text(item.get("Tenant")),
|
||||
auction_id=parse_text(item.get("ActnLnId")),
|
||||
auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")),
|
||||
inventory_status=parse_text(item.get("InventoryStatus")),
|
||||
currency=parse_text(item.get("Currency")),
|
||||
timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")),
|
||||
timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")),
|
||||
prebid_indicator=parse_bool(item.get("PreBidIndicator")),
|
||||
buynow_indicator=parse_bool(item.get("BuyNowIndicator")),
|
||||
)
|
||||
)
|
||||
return FastListingPage(
|
||||
vehicles=vehicles,
|
||||
result_count=parse_int(result_count_raw) or len(vehicles),
|
||||
page_size=parse_int(page_size_raw) or max(1, len(vehicles)),
|
||||
current_page=parse_int(current_page_raw) or 1,
|
||||
gbp_search_query=gbp_payload,
|
||||
)
|
||||
|
||||
|
||||
def parse_product_details_vm(html_text: str) -> dict[str, Any]:
|
||||
match = re.search(
|
||||
r"<script[^>]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*</script>",
|
||||
html_text,
|
||||
flags=re.DOTALL | re.IGNORECASE,
|
||||
)
|
||||
if match is None:
|
||||
raise RuntimeError("ProductDetailsVM script not found")
|
||||
payload = json.loads(match.group(1))
|
||||
if not isinstance(payload, dict):
|
||||
raise RuntimeError("ProductDetailsVM root is not object")
|
||||
return payload
|
||||
|
||||
|
||||
def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
|
||||
escaped_id = re.escape(input_id)
|
||||
patterns = (
|
||||
rf"<input[^>]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"",
|
||||
rf"<input[^>]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'",
|
||||
)
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, html_text, flags=re.IGNORECASE)
|
||||
if match is not None:
|
||||
return html.unescape(match.group(1))
|
||||
return None
|
||||
|
||||
|
||||
def parse_attribute_value(html_text: str, attribute_name: str) -> str | None:
|
||||
escaped_name = re.escape(attribute_name)
|
||||
patterns = (
|
||||
rf"\b{escaped_name}=\"([^\"]*)\"",
|
||||
rf"\b{escaped_name}='([^']*)'",
|
||||
)
|
||||
for pattern in patterns:
|
||||
match = re.search(pattern, html_text, flags=re.IGNORECASE)
|
||||
if match is not None:
|
||||
value = html.unescape(match.group(1)).strip()
|
||||
return value or None
|
||||
return None
|
||||
|
||||
|
||||
def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
|
||||
seen_fullres: set[str] = set()
|
||||
images: list[dict[str, str | int]] = []
|
||||
for index, item in enumerate(image_keys):
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
key = parse_text(item.get("k"))
|
||||
if key is None:
|
||||
continue
|
||||
width = parse_int(item.get("w")) or 1600
|
||||
height = parse_int(item.get("h")) or 1200
|
||||
if width <= 0:
|
||||
width = 1600
|
||||
if height <= 0:
|
||||
height = 1200
|
||||
order_index = parse_int(item.get("i"))
|
||||
if order_index is None:
|
||||
order_index = parse_int(item.get("in"))
|
||||
if order_index is None:
|
||||
order_index = index
|
||||
preview_width = min(640, width)
|
||||
preview_height = max(1, int(round(height * (preview_width / width))))
|
||||
escaped_key = quote(key, safe="~")
|
||||
fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}"
|
||||
preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}"
|
||||
if fullres in seen_fullres:
|
||||
continue
|
||||
seen_fullres.add(fullres)
|
||||
images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview})
|
||||
images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image"))))
|
||||
return images
|
||||
|
||||
|
||||
def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool:
|
||||
if status_code in {401, 403}:
|
||||
return True
|
||||
if _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
|
||||
return False
|
||||
lowered = (body_text or "").lower()
|
||||
if any(marker in lowered for marker in CHALLENGE_MARKERS):
|
||||
return True
|
||||
if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
|
||||
if "<html" in lowered or "<body" in lowered:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def _expected_marker_present(*, body_text: str, expected_marker: str | None) -> bool:
|
||||
if not expected_marker:
|
||||
return False
|
||||
if expected_marker in body_text:
|
||||
return True
|
||||
if '"' in expected_marker and expected_marker.replace('"', "'") in body_text:
|
||||
return True
|
||||
if "'" in expected_marker and expected_marker.replace("'", '"') in body_text:
|
||||
return True
|
||||
marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker)
|
||||
if marker_match is None:
|
||||
return False
|
||||
marker_id = re.escape(marker_match.group(1))
|
||||
return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE))
|
||||
|
||||
|
||||
def parse_text(value: Any) -> str | None:
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
return text if text else None
|
||||
return None
|
||||
|
||||
|
||||
def parse_bool(value: Any) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
return value.strip().lower() in {"true", "1", "yes", "on"}
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
return value != 0
|
||||
return False
|
||||
|
||||
|
||||
def parse_int(value: Any) -> int | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, int):
|
||||
return value
|
||||
if isinstance(value, float):
|
||||
return int(round(value))
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if not text:
|
||||
return None
|
||||
normalized = text.replace(",", "").replace(" ", "").replace("$", "")
|
||||
match = re.search(r"-?\d+(?:\.\d+)?", normalized)
|
||||
if match is None:
|
||||
return None
|
||||
try:
|
||||
return int(round(float(match.group(0))))
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
@@ -1,714 +0,0 @@
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from playwright.sync_api import Page
|
||||
|
||||
from .pace import HumanPacer
|
||||
from ..core.config import Settings
|
||||
from ..core.utils import first_non_empty
|
||||
|
||||
logger = logging.getLogger("MOBILEDE_scraper.listing")
|
||||
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
|
||||
VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
|
||||
COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
|
||||
"button:has-text('Accept All')",
|
||||
"button:has-text('Accept all')",
|
||||
"button:has-text('I Agree')",
|
||||
"button:has-text('Agree')",
|
||||
"button:has-text('Only necessary')",
|
||||
"button:has-text('Только необходимые')",
|
||||
"button:has-text('Принять все')",
|
||||
"[id*='accept']",
|
||||
"[class*='accept']",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ListingVehicleLink:
|
||||
href: str
|
||||
title: str = ""
|
||||
lot_number: str | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ListingPageResult:
|
||||
source_url: str
|
||||
page_number: int
|
||||
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
|
||||
pagination_available: bool = False
|
||||
next_page_detected: bool = False
|
||||
|
||||
|
||||
class ListingCollector:
|
||||
_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
|
||||
"a[aria-label*='Next']",
|
||||
"button[aria-label*='Next']",
|
||||
"a[aria-label*='next']",
|
||||
"button[aria-label*='next']",
|
||||
"a[title*='Next']",
|
||||
"button[title*='Next']",
|
||||
"a[title*='next']",
|
||||
"button[title*='next']",
|
||||
"a[rel='next']",
|
||||
"link[rel='next']",
|
||||
"a.pagination-next",
|
||||
"button.pagination-next",
|
||||
"a.next",
|
||||
"button.next",
|
||||
"a:has-text('Next')",
|
||||
"button:has-text('Next')",
|
||||
"a:has-text('NEXT')",
|
||||
"button:has-text('NEXT')",
|
||||
"a:has-text('›')",
|
||||
"button:has-text('›')",
|
||||
"a:has-text('»')",
|
||||
"button:has-text('»')",
|
||||
"a:has(img[src*='icon-arrow-right'])",
|
||||
"button:has(img[src*='icon-arrow-right'])",
|
||||
"a:has(img[src*='arrow-right'])",
|
||||
"button:has(img[src*='arrow-right'])",
|
||||
)
|
||||
|
||||
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
|
||||
self.settings = settings
|
||||
self.pacer = pacer
|
||||
|
||||
@staticmethod
|
||||
def _get_current_page_number(page: Page) -> int | None:
|
||||
try:
|
||||
value = page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
|
||||
const current = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
||||
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
|
||||
});
|
||||
if (current) {
|
||||
return parseInt((current.textContent || '').trim(), 10);
|
||||
}
|
||||
const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
|
||||
return match ? parseInt(match[1], 10) : null;
|
||||
}
|
||||
"""
|
||||
)
|
||||
return int(value) if value is not None else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
|
||||
if old_first_href:
|
||||
try:
|
||||
page.wait_for_function(
|
||||
f"""() => {{
|
||||
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
|
||||
return a && a.getAttribute('href') !== '{old_first_href}';
|
||||
}}""",
|
||||
timeout=12000,
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if expected_page_number is not None:
|
||||
current_page = ListingCollector._get_current_page_number(page)
|
||||
if current_page == expected_page_number:
|
||||
return True
|
||||
|
||||
try:
|
||||
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
current_page = ListingCollector._get_current_page_number(page)
|
||||
if expected_page_number is not None and current_page == expected_page_number:
|
||||
return True
|
||||
|
||||
return not old_first_href
|
||||
|
||||
@staticmethod
|
||||
def has_page_number(page: Page, target_page_number: int) -> bool:
|
||||
try:
|
||||
return bool(page.evaluate(
|
||||
"""
|
||||
(targetPageNumber) => {
|
||||
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
|
||||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
|
||||
return controls.some((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
|
||||
const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
|
||||
return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
|
||||
});
|
||||
}
|
||||
""",
|
||||
target_page_number,
|
||||
))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
|
||||
url = url_override or self.settings.listing.cars_url
|
||||
logger.info("Opening cars listing page: %s", url)
|
||||
last_err = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
page.goto(url, wait_until="commit", timeout=60_000)
|
||||
last_err = None
|
||||
break
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
|
||||
# Небольшой backoff при ошибках открытия листинга.
|
||||
time.sleep(5 * (attempt + 1))
|
||||
if last_err:
|
||||
logger.warning("All goto attempts failed, trying JS navigation")
|
||||
try:
|
||||
page.evaluate(f"window.location.href = '{url}'")
|
||||
except Exception:
|
||||
pass
|
||||
# Быстрая проверка готовности страницы.
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=12_000)
|
||||
except Exception:
|
||||
pass
|
||||
self._accept_cookie_banner(page)
|
||||
self._wait_for_listing_content(page)
|
||||
logger.info("Listing page URL: %s", page.url)
|
||||
self.pacer.after_listing_open()
|
||||
|
||||
def _accept_cookie_banner(self, page: Page) -> None:
|
||||
for selector in COOKIE_ACCEPT_SELECTORS:
|
||||
locator = page.locator(selector).first
|
||||
try:
|
||||
if locator.count() == 0:
|
||||
continue
|
||||
if not locator.is_visible(timeout=500):
|
||||
continue
|
||||
locator.click(timeout=2_000)
|
||||
logger.info("Accepted cookie banner using selector: %s", selector)
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=3_000)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
def _wait_for_listing_content(self, page: Page) -> None:
|
||||
try:
|
||||
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
|
||||
try:
|
||||
page.wait_for_function(
|
||||
"""() => {
|
||||
const html = document.documentElement?.innerHTML || '';
|
||||
const text = document.body?.innerText || '';
|
||||
return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
|
||||
}""",
|
||||
timeout=12_000,
|
||||
)
|
||||
except Exception:
|
||||
# Короткая пауза вместо длинного sleep.
|
||||
time.sleep(1.0)
|
||||
|
||||
def apply_filters(
|
||||
self,
|
||||
page: Page,
|
||||
make: str | None = None,
|
||||
model: str | None = None,
|
||||
year_min: int | None = None,
|
||||
year_max: int | None = None,
|
||||
) -> dict[str, str | int | None]:
|
||||
applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
|
||||
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
|
||||
applied["make"] = make
|
||||
self.pacer.after_filter_action()
|
||||
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
|
||||
applied["model"] = model
|
||||
self.pacer.after_filter_action()
|
||||
if year_min is not None or year_max is not None:
|
||||
if self._apply_year_range(page, year_min, year_max):
|
||||
applied["year_min"] = year_min
|
||||
applied["year_max"] = year_max
|
||||
self.pacer.after_filter_action()
|
||||
return applied
|
||||
|
||||
def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
|
||||
"""Заполняет поля фильтра Year и нажимает Apply Year."""
|
||||
if year_min is None and year_max is None:
|
||||
return False
|
||||
try:
|
||||
success = page.evaluate(
|
||||
"""([yearMin, yearMax]) => {
|
||||
const inputs = Array.from(document.querySelectorAll('input'));
|
||||
const yearInputs = inputs.filter(inp => {
|
||||
const v = parseInt(inp.value, 10);
|
||||
return !isNaN(v) && v >= 1900 && v <= 2100;
|
||||
});
|
||||
if (yearInputs.length < 2) return false;
|
||||
yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
|
||||
const setVal = (el, val) => {
|
||||
const setter = Object.getOwnPropertyDescriptor(
|
||||
HTMLInputElement.prototype, 'value'
|
||||
).set;
|
||||
setter.call(el, String(val));
|
||||
el.dispatchEvent(new Event('input', {bubbles: true}));
|
||||
el.dispatchEvent(new Event('change', {bubbles: true}));
|
||||
};
|
||||
if (yearMin !== null) setVal(yearInputs[0], yearMin);
|
||||
if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
|
||||
const container = yearInputs[0].closest(
|
||||
'[class*="filter"], [class*="year"], section, fieldset'
|
||||
) || yearInputs[0].parentElement.parentElement;
|
||||
if (container) {
|
||||
const btn = Array.from(container.querySelectorAll(
|
||||
'button, a, [role="button"], span[class*="apply"]'
|
||||
)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
|
||||
if (btn) { btn.click(); return true; }
|
||||
}
|
||||
yearInputs[yearInputs.length - 1].dispatchEvent(
|
||||
new KeyboardEvent('keydown', {
|
||||
key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
|
||||
})
|
||||
);
|
||||
return true;
|
||||
}""",
|
||||
[year_min, year_max],
|
||||
)
|
||||
if success:
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=15_000)
|
||||
except Exception:
|
||||
pass
|
||||
self._wait_for_listing_content(page)
|
||||
logger.info("Applied year range filter: %s — %s", year_min, year_max)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to apply year range filter: %s", exc)
|
||||
return False
|
||||
|
||||
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
|
||||
# Считываем ссылки одним проходом по DOM.
|
||||
self._accept_cookie_banner(page)
|
||||
self._wait_for_listing_content(page)
|
||||
try:
|
||||
raw_items = page.eval_on_selector_all(
|
||||
"a[href], [data-href], [href]",
|
||||
"""
|
||||
(nodes) => nodes.map((node) => ({
|
||||
href:
|
||||
node.getAttribute('href') ||
|
||||
node.getAttribute('data-href') ||
|
||||
node.getAttribute('data-url') ||
|
||||
'',
|
||||
title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
|
||||
text: (node.textContent || '').trim(),
|
||||
}))
|
||||
""",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
|
||||
raw_items = []
|
||||
total = min(len(raw_items), self.settings.listing.page_link_limit)
|
||||
links: list[ListingVehicleLink] = []
|
||||
seen: set[str] = set()
|
||||
for idx in range(total):
|
||||
item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
|
||||
href = str(item.get("href") or "")
|
||||
match = VEHICLE_HREF_RE.search(href)
|
||||
if not match:
|
||||
continue
|
||||
lot_number = match.group(1)
|
||||
absolute = urljoin(self.settings.home_url, match.group(0))
|
||||
if absolute in seen:
|
||||
continue
|
||||
seen.add(absolute)
|
||||
title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
|
||||
links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
|
||||
if len(links) >= self.settings.listing.max_vehicles_per_run:
|
||||
break
|
||||
|
||||
# Fallback: на MOBILEDE ссылки иногда не рендерятся как <a>,
|
||||
# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
|
||||
if not links:
|
||||
try:
|
||||
page.wait_for_timeout(1_500)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
html = page.content()
|
||||
except Exception as exc:
|
||||
logger.debug("page.content() failed on page %d: %s", page_number, exc)
|
||||
html = ""
|
||||
|
||||
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
|
||||
if absolute in seen:
|
||||
continue
|
||||
seen.add(absolute)
|
||||
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
|
||||
if len(links) >= self.settings.listing.max_vehicles_per_run:
|
||||
break
|
||||
|
||||
if links:
|
||||
logger.info(
|
||||
"Page %d: recovered %d vehicle links from HTML fallback",
|
||||
page_number,
|
||||
len(links),
|
||||
)
|
||||
next_page_detected = self._has_next_page(page)
|
||||
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
|
||||
|
||||
def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
|
||||
if not html:
|
||||
return []
|
||||
|
||||
# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
|
||||
normalized = html.replace("\\/", "/")
|
||||
found: list[tuple[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for match in VEHICLE_HREF_RE.finditer(normalized):
|
||||
lot_number = match.group(1)
|
||||
absolute = urljoin(self.settings.home_url, match.group(0))
|
||||
if absolute in seen:
|
||||
continue
|
||||
seen.add(absolute)
|
||||
found.append((absolute, lot_number))
|
||||
if len(found) >= self.settings.listing.page_link_limit:
|
||||
break
|
||||
|
||||
return found
|
||||
|
||||
def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
|
||||
# Запоминаем первую ссылку текущей страницы для определения смены контента.
|
||||
old_first_href = ""
|
||||
try:
|
||||
first_link = page.locator(VEHICLE_LINK_SELECTOR).first
|
||||
if first_link.count() > 0:
|
||||
old_first_href = first_link.get_attribute("href") or ""
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for selector in self._NEXT_PAGE_SELECTORS:
|
||||
locator = page.locator(selector).first
|
||||
if locator.count() == 0:
|
||||
continue
|
||||
try:
|
||||
disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
|
||||
aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
|
||||
classes = (locator.get_attribute("class", timeout=1500) or "").lower()
|
||||
except Exception:
|
||||
continue
|
||||
if disabled or aria_disabled == "true" or "disabled" in classes:
|
||||
continue
|
||||
try:
|
||||
self.pacer.move_mouse_to(page, locator)
|
||||
locator.click(timeout=8000)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
|
||||
self.pacer.after_page_change()
|
||||
return True
|
||||
|
||||
# Fallback для mobilede: пагинация часто рендерится как набор номеров страниц
|
||||
# + стрелка с иконкой, без явного текста Next.
|
||||
try:
|
||||
clicked = bool(page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
|
||||
const disabled = (el) => {
|
||||
if (!el) return true;
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
|
||||
return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
|
||||
};
|
||||
|
||||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
|
||||
.filter((el) => visible(el) && !disabled(el));
|
||||
|
||||
const current = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
||||
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
|
||||
});
|
||||
|
||||
if (current) {
|
||||
const currentPage = parseInt((current.textContent || '').trim(), 10);
|
||||
const nextNumber = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
|
||||
});
|
||||
if (nextNumber) {
|
||||
nextNumber.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
const iconNext = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim().toLowerCase();
|
||||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||||
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
|
||||
return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
|
||||
});
|
||||
|
||||
if (iconNext) {
|
||||
iconNext.click();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
"""
|
||||
))
|
||||
if clicked:
|
||||
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
|
||||
self.pacer.after_page_change()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug("Numeric/icon pagination fallback failed: %s", exc)
|
||||
|
||||
# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
|
||||
try:
|
||||
clicked = bool(page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
|
||||
for (const el of candidates) {
|
||||
const text = (el.textContent || '').trim().toLowerCase();
|
||||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||||
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
|
||||
const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
|
||||
const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
|
||||
if (!disabled && visible && looksNext) {
|
||||
el.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
"""
|
||||
))
|
||||
if clicked:
|
||||
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
|
||||
self.pacer.after_page_change()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug("JS next-page fallback failed: %s", exc)
|
||||
|
||||
logger.warning("Could not navigate to next page from %s", page.url)
|
||||
return False
|
||||
|
||||
def collect_listing_links(
|
||||
self,
|
||||
page: Page,
|
||||
*,
|
||||
make: str | None = None,
|
||||
model: str | None = None,
|
||||
known_origin_ids: set[str] | None = None,
|
||||
max_duration_seconds: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
self.open_cars_listing(page)
|
||||
applied_filters = self.apply_filters(page, make=make, model=model)
|
||||
started_at = time.perf_counter()
|
||||
truncated_by_time_budget = False
|
||||
pages: list[dict[str, object]] = []
|
||||
all_links: list[str] = []
|
||||
early_stopped = False
|
||||
threshold = self.settings.listing.early_stop_threshold
|
||||
|
||||
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
|
||||
if max_duration_seconds is not None and max_duration_seconds > 0:
|
||||
elapsed = time.perf_counter() - started_at
|
||||
if elapsed >= max_duration_seconds:
|
||||
truncated_by_time_budget = True
|
||||
logger.warning(
|
||||
"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
|
||||
page_number,
|
||||
elapsed,
|
||||
max_duration_seconds,
|
||||
)
|
||||
break
|
||||
page_result = self.collect_current_page(page, page_number=page_number)
|
||||
pages.append({
|
||||
"page_number": page_result.page_number,
|
||||
"source_url": page_result.source_url,
|
||||
"links_found": len(page_result.vehicle_links),
|
||||
"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
|
||||
"next_page_detected": page_result.next_page_detected,
|
||||
})
|
||||
for item in page_result.vehicle_links:
|
||||
if item.href not in all_links:
|
||||
all_links.append(item.href)
|
||||
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
|
||||
break
|
||||
|
||||
# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
|
||||
# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
|
||||
if (
|
||||
known_origin_ids is not None
|
||||
and threshold > 0.0
|
||||
and page_result.vehicle_links
|
||||
):
|
||||
page_known = sum(
|
||||
1 for item in page_result.vehicle_links
|
||||
if item.lot_number and f"mobilede:{item.lot_number}" in known_origin_ids
|
||||
)
|
||||
page_new = len(page_result.vehicle_links) - page_known
|
||||
ratio = page_known / len(page_result.vehicle_links)
|
||||
# Останавливаемся только если нет новых И порог превышен
|
||||
if ratio >= threshold and page_new == 0:
|
||||
logger.info(
|
||||
"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
|
||||
page_number, ratio * 100, page_known,
|
||||
len(page_result.vehicle_links), threshold * 100,
|
||||
)
|
||||
early_stopped = True
|
||||
break
|
||||
elif page_new > 0 and ratio >= threshold:
|
||||
logger.info(
|
||||
"Page %d: %.0f%% known but %d new found — продолжаем",
|
||||
page_number, ratio * 100, page_new,
|
||||
)
|
||||
|
||||
if (
|
||||
len(all_links) >= self.settings.listing.max_vehicles_per_run
|
||||
or self.settings.listing.collect_current_page_only
|
||||
or not self.settings.listing.include_pagination
|
||||
or not page_result.next_page_detected
|
||||
):
|
||||
break
|
||||
if not self.go_to_next_page(page):
|
||||
break
|
||||
|
||||
return {
|
||||
"listing_url": self.settings.listing.cars_url,
|
||||
"applied_filters": applied_filters,
|
||||
"pages_collected": len(pages),
|
||||
"vehicles_collected": len(all_links),
|
||||
"vehicle_urls": all_links,
|
||||
"early_stopped": early_stopped,
|
||||
"truncated_by_time_budget": truncated_by_time_budget,
|
||||
"pages": pages,
|
||||
"strategy": {
|
||||
"sequential": True,
|
||||
"collect_current_page_only": self.settings.listing.collect_current_page_only,
|
||||
"include_pagination": self.settings.listing.include_pagination,
|
||||
"max_pages_per_run": self.settings.listing.max_pages_per_run,
|
||||
"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
|
||||
"early_stop_threshold": threshold,
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
|
||||
for selector in selectors:
|
||||
locator = page.locator(selector).first
|
||||
if locator.count() == 0:
|
||||
continue
|
||||
try:
|
||||
locator.click()
|
||||
locator.fill(value)
|
||||
page.keyboard.press("Enter")
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=15000)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _has_next_page(page: Page) -> bool:
|
||||
for selector in ListingCollector._NEXT_PAGE_SELECTORS:
|
||||
locator = page.locator(selector)
|
||||
count = locator.count()
|
||||
if count == 0:
|
||||
continue
|
||||
if not hasattr(locator, "nth"):
|
||||
return True
|
||||
# Проверяем, что хотя бы один элемент не disabled.
|
||||
# Disabled "Next" на последней странице не означает наличия следующей.
|
||||
for i in range(min(count, 3)):
|
||||
try:
|
||||
el = locator.nth(i)
|
||||
disabled_attr = el.get_attribute("disabled", timeout=300)
|
||||
aria_disabled = el.get_attribute("aria-disabled", timeout=300)
|
||||
cls = (el.get_attribute("class", timeout=300) or "").lower()
|
||||
if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
try:
|
||||
return bool(page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
|
||||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
|
||||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||||
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
|
||||
return !disabled && visible(el);
|
||||
});
|
||||
|
||||
const hasExplicitNext = controls.some((el) => {
|
||||
const text = (el.textContent || '').trim().toLowerCase();
|
||||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||||
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
|
||||
return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
|
||||
});
|
||||
|
||||
if (hasExplicitNext) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const current = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
||||
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
|
||||
});
|
||||
|
||||
if (!current) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const currentPage = parseInt((current.textContent || '').trim(), 10);
|
||||
return controls.some((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
|
||||
});
|
||||
}
|
||||
"""
|
||||
))
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
@@ -1,130 +0,0 @@
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from playwright.sync_api import Page, Request, Response
|
||||
|
||||
from ..core.config import Settings
|
||||
|
||||
logger = logging.getLogger("MOBILEDE_scraper.network")
|
||||
|
||||
|
||||
@dataclass
|
||||
class NetworkCapture:
|
||||
# Перехватчик сетевых запросов.
|
||||
|
||||
settings: Settings
|
||||
requests: list[dict[str, Any]] = field(default_factory=list)
|
||||
json_responses: list[dict[str, Any]] = field(default_factory=list)
|
||||
_seen_req: set[str] = field(default_factory=set)
|
||||
_seen_resp: set[str] = field(default_factory=set)
|
||||
_origin: str | None = None
|
||||
_page: Any = field(default=None)
|
||||
|
||||
def attach(self, page: Page, origin_url: str | None = None) -> None:
|
||||
# Снимаем старые подписки перед повторным attach.
|
||||
try:
|
||||
page.remove_listener("request", self._on_request)
|
||||
page.remove_listener("response", self._on_response)
|
||||
except Exception:
|
||||
pass
|
||||
# Подписка на сетевые события
|
||||
try:
|
||||
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
|
||||
except Exception:
|
||||
self._origin = None
|
||||
self._page = page
|
||||
page.on("request", self._on_request)
|
||||
page.on("response", self._on_response)
|
||||
|
||||
def _is_same_origin(self, url: str) -> bool:
|
||||
# Фильтр по домену
|
||||
if not self.settings.capture.capture_same_origin_only or not self._origin:
|
||||
return True
|
||||
netloc = urlparse(url).netloc.lower()
|
||||
return netloc == self._origin or netloc.endswith(".MOBILEDE.com")
|
||||
|
||||
def _on_request(self, request: Request) -> None:
|
||||
# Берём только xhr/fetch
|
||||
if request.resource_type not in {"xhr", "fetch"}:
|
||||
return
|
||||
if not self._is_same_origin(request.url):
|
||||
return
|
||||
if len(self.requests) >= self.settings.capture.max_requests:
|
||||
return
|
||||
key = f"{request.method}:{request.url}:{request.post_data or ''}"
|
||||
# Убираем дубли
|
||||
if key in self._seen_req:
|
||||
return
|
||||
self._seen_req.add(key)
|
||||
self.requests.append({
|
||||
"url": request.url, "method": request.method,
|
||||
"resource_type": request.resource_type, "post_data": request.post_data,
|
||||
})
|
||||
|
||||
def _on_response(self, response: Response) -> None:
|
||||
# Сохраняем JSON
|
||||
request = response.request
|
||||
if request.resource_type not in {"xhr", "fetch"}:
|
||||
return
|
||||
if not self._is_same_origin(response.url):
|
||||
return
|
||||
if len(self.json_responses) >= self.settings.capture.max_json_responses:
|
||||
return
|
||||
if not self._is_json(response):
|
||||
return
|
||||
key = f"{request.method}:{response.url}:{response.status}"
|
||||
if key in self._seen_resp:
|
||||
return
|
||||
self._seen_resp.add(key)
|
||||
try:
|
||||
payload = response.json()
|
||||
except Exception:
|
||||
try:
|
||||
payload = json.loads(response.text())
|
||||
except Exception:
|
||||
return
|
||||
self.json_responses.append({
|
||||
"url": response.url, "status": response.status,
|
||||
"request_method": request.method, "post_data": request.post_data,
|
||||
"resource_type": request.resource_type, "payload": payload,
|
||||
"category": self._categorize(response.url),
|
||||
})
|
||||
|
||||
@staticmethod
|
||||
def _is_json(response: Response) -> bool:
|
||||
ct = (response.headers.get("content-type") or "").lower()
|
||||
if "application/json" in ct or "+json" in ct:
|
||||
return True
|
||||
url = response.url.lower()
|
||||
return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
|
||||
|
||||
@staticmethod
|
||||
def _categorize(url: str) -> str:
|
||||
# Простая категория URL
|
||||
low = url.lower()
|
||||
mapping = {
|
||||
"images": ["image", "media", "photos", "gallery"],
|
||||
"bids": ["bid", "offer", "buy-now", "buynow"],
|
||||
"auction": ["auction", "sale", "lane", "branch"],
|
||||
"vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
|
||||
"documents": ["title", "document", "report"],
|
||||
}
|
||||
for cat, markers in mapping.items():
|
||||
if any(m in low for m in markers):
|
||||
return cat
|
||||
return "other"
|
||||
|
||||
def export(self) -> dict[str, Any]:
|
||||
responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
|
||||
return {
|
||||
"requests": self.requests,
|
||||
"json_responses": responses,
|
||||
"capture_limits": {
|
||||
"same_origin_only": self.settings.capture.capture_same_origin_only,
|
||||
"max_requests": self.settings.capture.max_requests,
|
||||
"max_json_responses": self.settings.capture.max_json_responses,
|
||||
},
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
import random
|
||||
import time
|
||||
|
||||
from playwright.sync_api import Locator, Page
|
||||
|
||||
from ..core.config import Settings
|
||||
|
||||
|
||||
class HumanPacer:
|
||||
# Случайные паузы между действиями.
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.settings = settings
|
||||
|
||||
def pause(self, min_s: float, max_s: float) -> None:
|
||||
if self.settings.pace.enabled:
|
||||
time.sleep(random.uniform(min_s, max_s))
|
||||
|
||||
def after_listing_open(self) -> None:
|
||||
self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
|
||||
|
||||
def after_filter_action(self) -> None:
|
||||
self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
|
||||
|
||||
def before_vehicle_open(self) -> None:
|
||||
self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
|
||||
|
||||
def after_vehicle_open(self) -> None:
|
||||
self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
|
||||
|
||||
def between_vehicles(self) -> None:
|
||||
self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
|
||||
|
||||
def after_page_change(self) -> None:
|
||||
self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
|
||||
|
||||
def move_mouse_to(self, page: Page, locator: Locator) -> None:
|
||||
try:
|
||||
box = locator.bounding_box()
|
||||
except Exception:
|
||||
box = None
|
||||
if not box:
|
||||
return
|
||||
x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
|
||||
y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
|
||||
page.mouse.move(x, y, steps=random.randint(8, 18))
|
||||
@@ -1,4 +1,4 @@
|
||||
import json
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
@@ -12,7 +12,7 @@ load_dotenv()
|
||||
TRUE_VALUES = {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
# Хелперы для чтения env-переменных с приведением типов
|
||||
# Хелперы env.
|
||||
|
||||
def _env_str(name: str, default: str) -> str:
|
||||
value = os.getenv(name)
|
||||
@@ -47,80 +47,13 @@ def _env_float(name: str, default: float) -> float:
|
||||
return float(_env_str(name, str(default)).strip())
|
||||
|
||||
|
||||
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FingerprintConfig:
|
||||
user_agent: str = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/135.0.0.0 Safari/537.36"
|
||||
)
|
||||
viewport_presets: tuple[dict[str, int], ...] = (
|
||||
{"width": 1920, "height": 1080},
|
||||
{"width": 1600, "height": 900},
|
||||
{"width": 1536, "height": 864},
|
||||
{"width": 1440, "height": 900},
|
||||
{"width": 1366, "height": 768},
|
||||
)
|
||||
timezone_candidates: tuple[str, ...] = (
|
||||
"America/New_York",
|
||||
"America/Chicago",
|
||||
"America/Los_Angeles",
|
||||
)
|
||||
locale: str = "en-US"
|
||||
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
|
||||
|
||||
|
||||
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CaptureConfig:
|
||||
capture_same_origin_only: bool = _env_bool("MOBILEDE_CAPTURE_SAME_ORIGIN_ONLY", True)
|
||||
max_requests: int = _env_int("MOBILEDE_MAX_CAPTURED_REQUESTS", 40)
|
||||
max_json_responses: int = _env_int("MOBILEDE_MAX_CAPTURED_JSON_RESPONSES", 30)
|
||||
|
||||
|
||||
# Конфиг пауз между действиями (имитация человека)
|
||||
|
||||
@dataclass(slots=True)
|
||||
class HumanPaceConfig:
|
||||
enabled: bool = _env_bool("MOBILEDE_HUMAN_PACE_ENABLED", True)
|
||||
after_listing_open_min_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MIN_S", 0.5)
|
||||
after_listing_open_max_s: float = _env_float("MOBILEDE_AFTER_LISTING_OPEN_MAX_S", 1.2)
|
||||
after_filter_action_min_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MIN_S", 0.5)
|
||||
after_filter_action_max_s: float = _env_float("MOBILEDE_AFTER_FILTER_ACTION_MAX_S", 1.2)
|
||||
before_vehicle_open_min_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
|
||||
before_vehicle_open_max_s: float = _env_float("MOBILEDE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
|
||||
after_vehicle_open_min_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
|
||||
after_vehicle_open_max_s: float = _env_float("MOBILEDE_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
|
||||
between_vehicles_min_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MIN_S", 0.05)
|
||||
between_vehicles_max_s: float = _env_float("MOBILEDE_BETWEEN_VEHICLES_MAX_S", 0.15)
|
||||
after_page_change_min_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MIN_S", 0.8)
|
||||
after_page_change_max_s: float = _env_float("MOBILEDE_AFTER_PAGE_CHANGE_MAX_S", 1.8)
|
||||
|
||||
|
||||
# Конфиг сбора листинга (URL, лимиты страниц и машин)
|
||||
# Конфиг ссылок.
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ListingConfig:
|
||||
cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.MOBILEDE.com/Vehiclelisting/Cars")
|
||||
cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.mobile.de/Vehiclelisting/Cars")
|
||||
filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
|
||||
filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
|
||||
max_pages_per_run: int = _env_int("MOBILEDE_MAX_PAGES_PER_RUN", 9999)
|
||||
max_vehicles_per_run: int = _env_int("MOBILEDE_MAX_VEHICLES_PER_RUN", 50000)
|
||||
page_link_limit: int = _env_int("MOBILEDE_PAGE_LINK_LIMIT", 500)
|
||||
include_pagination: bool = _env_bool("MOBILEDE_INCLUDE_PAGINATION", True)
|
||||
collect_current_page_only: bool = _env_bool("MOBILEDE_COLLECT_CURRENT_PAGE_ONLY", False)
|
||||
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
|
||||
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
|
||||
early_stop_threshold: float = _env_float("MOBILEDE_EARLY_STOP_THRESHOLD", 0.8)
|
||||
# Сегментация листинга по брендам для обхода лимита пагинации MOBILEDE (~22 600 машин).
|
||||
# JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка
|
||||
# или "runtime" для построения по runtime_config.filters.brands.
|
||||
# Пустая строка = без сегментации (backward compatible).
|
||||
listing_segments_json: str = _env_str("MOBILEDE_LISTING_SEGMENTS", "")
|
||||
fast_segment_year_splits: bool = _env_bool("MOBILEDE_FAST_SEGMENT_YEAR_SPLITS", True)
|
||||
|
||||
@property
|
||||
def filtered_search_urls(self) -> list[str]:
|
||||
@@ -143,106 +76,7 @@ class ListingConfig:
|
||||
return urls
|
||||
|
||||
|
||||
# Список брендов MOBILEDE для автоматической сегментации.
|
||||
# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
|
||||
MOBILEDE_DEFAULT_MAKES: tuple[str, ...] = (
|
||||
"TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
|
||||
"KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
|
||||
"VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
|
||||
"RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
|
||||
"MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
|
||||
"MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
|
||||
"SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
|
||||
"OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
|
||||
"RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
|
||||
"BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
|
||||
"MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
|
||||
)
|
||||
|
||||
# Пагинационный потолок mobilede: ~226 страниц × 100 = 22 600 результатов.
|
||||
MOBILEDE_PAGINATION_CEILING = 22_600
|
||||
|
||||
# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
|
||||
_LARGE_MAKES: frozenset[str] = frozenset({
|
||||
"TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
|
||||
"KIA", "DODGE", "JEEP",
|
||||
})
|
||||
_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
|
||||
(1900, 2012),
|
||||
(2013, 2019),
|
||||
(2020, 2027),
|
||||
)
|
||||
|
||||
|
||||
def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
|
||||
"""Строит сегменты по переданному списку брендов.
|
||||
|
||||
Крупные бренды режутся по годовым диапазонам так же, как в ``auto``.
|
||||
"""
|
||||
segments: list[dict[str, str | int | None]] = []
|
||||
seen: set[str] = set()
|
||||
for raw_make in makes:
|
||||
make = str(raw_make or "").strip().upper()
|
||||
if not make or make in seen:
|
||||
continue
|
||||
seen.add(make)
|
||||
if make in _LARGE_MAKES:
|
||||
for yr_min, yr_max in _YEAR_SPLITS:
|
||||
segments.append({"make": make, "year_min": yr_min, "year_max": yr_max})
|
||||
else:
|
||||
segments.append({"make": make, "year_min": None, "year_max": None})
|
||||
return segments
|
||||
|
||||
|
||||
def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
|
||||
"""Строит HTTP-first сегменты без UI-фильтров годов.
|
||||
|
||||
Это ближе к MOBILEDE-fast: один бренд = один hidden-payload HTTP обход.
|
||||
Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля.
|
||||
"""
|
||||
segments: list[dict[str, str | int | None]] = []
|
||||
seen: set[str] = set()
|
||||
for raw_make in makes:
|
||||
make = str(raw_make or "").strip().upper()
|
||||
if not make or make in seen:
|
||||
continue
|
||||
seen.add(make)
|
||||
segments.append({"make": make, "year_min": None, "year_max": None})
|
||||
return segments
|
||||
|
||||
|
||||
def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
|
||||
"""Парсит MOBILEDE_LISTING_SEGMENTS в список сегментов.
|
||||
|
||||
Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
|
||||
Специальное значение ``"auto"`` генерирует сегменты из MOBILEDE_DEFAULT_MAKES.
|
||||
Крупные бренды автоматически разбиваются по диапазонам годов.
|
||||
"""
|
||||
raw = raw.strip()
|
||||
if not raw:
|
||||
return []
|
||||
if raw.lower() == "auto":
|
||||
return build_listing_segments_for_makes(list(MOBILEDE_DEFAULT_MAKES))
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return []
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
segments = []
|
||||
for item in data:
|
||||
if isinstance(item, str):
|
||||
segments.append({"make": item.upper(), "year_min": None, "year_max": None})
|
||||
elif isinstance(item, dict):
|
||||
segments.append({
|
||||
"make": str(item.get("make") or "").upper() or None,
|
||||
"year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
|
||||
"year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
|
||||
})
|
||||
return segments
|
||||
|
||||
|
||||
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
|
||||
# Конфиг PostgreSQL.
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DatabaseConfig:
|
||||
@@ -254,7 +88,7 @@ class DatabaseConfig:
|
||||
auto_create_tables: bool = _env_bool("MOBILEDE_DATABASE_AUTO_CREATE_TABLES", False)
|
||||
|
||||
|
||||
# --- Конфиг Redis (URL для Celery broker) ---
|
||||
# Конфиг Redis.
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RedisConfig:
|
||||
@@ -264,17 +98,7 @@ class RedisConfig:
|
||||
health_check_interval_seconds: int = _env_int("MOBILEDE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
|
||||
|
||||
|
||||
# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DiscoveryConfig:
|
||||
mode: str = _env_str("MOBILEDE_DISCOVERY_MODE", "sitemap")
|
||||
hourly_mode: str = _env_str("MOBILEDE_HOURLY_MODE", "rolling_refresh")
|
||||
hourly_refresh_batch_size: int = _env_int("MOBILEDE_HOURLY_REFRESH_BATCH_SIZE", 500)
|
||||
always_full_scan: bool = _env_bool("MOBILEDE_ALWAYS_FULL_SCAN", False)
|
||||
|
||||
|
||||
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
|
||||
# Конфиг Celery.
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CeleryConfig:
|
||||
@@ -334,7 +158,7 @@ class ScrapingProfileConfig:
|
||||
self.listing_retries = None
|
||||
|
||||
|
||||
# --- Конфиг прокси (server, username, password) ---
|
||||
# Конфиг прокси.
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ProxyConfig:
|
||||
@@ -383,11 +207,11 @@ class ProxyConfig:
|
||||
return {"http": proxy_url, "https": proxy_url}
|
||||
|
||||
|
||||
# Главный объект настроек: собирает все блоки конфигурации
|
||||
# Общие настройки.
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Settings:
|
||||
home_url: str = "https://www.MOBILEDE.com/"
|
||||
home_url: str = "https://www.mobile.de/"
|
||||
default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000)
|
||||
network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400)
|
||||
fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000)
|
||||
@@ -407,15 +231,11 @@ class Settings:
|
||||
tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE")
|
||||
runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
|
||||
scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60)
|
||||
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
|
||||
capture: CaptureConfig = field(default_factory=CaptureConfig)
|
||||
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
|
||||
listing: ListingConfig = field(default_factory=ListingConfig)
|
||||
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
||||
redis: RedisConfig = field(default_factory=RedisConfig)
|
||||
celery: CeleryConfig = field(default_factory=CeleryConfig)
|
||||
proxy: ProxyConfig = field(default_factory=ProxyConfig)
|
||||
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
|
||||
scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
|
||||
|
||||
@property
|
||||
@@ -430,5 +250,5 @@ class Settings:
|
||||
def block_resources(self) -> bool:
|
||||
return self.celery.block_resources
|
||||
|
||||
# Глобальный синглтон — используется по умолчанию во всех модулях.
|
||||
# Глобальные настройки.
|
||||
settings = Settings()
|
||||
|
||||
@@ -39,7 +39,7 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
|
||||
root.addHandler(handler)
|
||||
root.propagate = False
|
||||
fmt = logging.Formatter(
|
||||
"%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
|
||||
"%(asctime)s | %(levelname)s | %(name)s | %(message)s"
|
||||
)
|
||||
for handler in root.handlers:
|
||||
handler.setFormatter(fmt)
|
||||
|
||||
@@ -1,15 +1 @@
|
||||
from .sitemap import (
|
||||
SitemapDiscoveryError,
|
||||
SitemapDiscoveryResult,
|
||||
SitemapDiscoveryStats,
|
||||
discover_vehicle_urls_from_sitemap,
|
||||
discover_vehicle_urls_from_sitemap_with_stats,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SitemapDiscoveryError",
|
||||
"SitemapDiscoveryResult",
|
||||
"SitemapDiscoveryStats",
|
||||
"discover_vehicle_urls_from_sitemap",
|
||||
"discover_vehicle_urls_from_sitemap_with_stats",
|
||||
]
|
||||
__all__: list[str] = []
|
||||
|
||||
@@ -1,210 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Iterable
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
from urllib.request import Request, urlopen, ProxyHandler, build_opener
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
logger = logging.getLogger("MOBILEDE_scraper.discovery.sitemap")
|
||||
|
||||
DEFAULT_SITEMAP_INDEX_URL = "https://www.MOBILEDE.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
|
||||
_SITEMAP_TIMEOUT_SECONDS = 30
|
||||
_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
class SitemapDiscoveryError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _is_vehicle_sitemap_url(url: str) -> bool:
|
||||
lowered = url.strip().lower()
|
||||
# Берём только sitemap с авто.
|
||||
if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
|
||||
return False
|
||||
return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
|
||||
|
||||
|
||||
def _normalize_vehicle_url(url: str) -> str:
|
||||
parts = urlsplit(url.strip())
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
||||
|
||||
|
||||
def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
|
||||
request = Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
|
||||
"Accept-Encoding": "gzip",
|
||||
},
|
||||
)
|
||||
if proxy_url:
|
||||
handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
|
||||
opener = build_opener(handler)
|
||||
response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
|
||||
else:
|
||||
response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
|
||||
with response:
|
||||
payload = response.read()
|
||||
encoding = str(response.headers.get("Content-Encoding") or "").lower()
|
||||
if encoding == "gzip" or url.lower().endswith(".gz"):
|
||||
return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
|
||||
return payload
|
||||
|
||||
|
||||
def _local_name(tag: str) -> str:
|
||||
if "}" in tag:
|
||||
return tag.rsplit("}", 1)[1]
|
||||
return tag
|
||||
|
||||
|
||||
def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
|
||||
for match in _LOC_TAG_RE.finditer(xml_bytes):
|
||||
try:
|
||||
value = match.group(1).decode("utf-8", errors="ignore").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if value:
|
||||
yield value
|
||||
|
||||
|
||||
def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
|
||||
try:
|
||||
root = ET.fromstring(xml_bytes)
|
||||
except ET.ParseError as exc:
|
||||
fallback_values = list(_iter_loc_values_fallback(xml_bytes))
|
||||
if fallback_values:
|
||||
logger.warning(
|
||||
"Falling back to regex sitemap loc extraction after XML parse error: %s",
|
||||
exc,
|
||||
)
|
||||
yield from fallback_values
|
||||
return
|
||||
raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
|
||||
|
||||
for element in root.iter():
|
||||
if _local_name(element.tag) != "loc":
|
||||
continue
|
||||
if not element.text:
|
||||
continue
|
||||
value = element.text.strip()
|
||||
if value:
|
||||
yield value
|
||||
|
||||
|
||||
def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for url in urls:
|
||||
normalized = _normalize_vehicle_url(url)
|
||||
if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
|
||||
continue
|
||||
if normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
result.append(normalized)
|
||||
return result
|
||||
|
||||
|
||||
def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
|
||||
return any("/vehicledetail/" in url.lower() for url in urls)
|
||||
|
||||
|
||||
def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
|
||||
logger.info("Downloading sitemap index: %s", index_url)
|
||||
index_xml = _download_bytes(index_url, proxy_url=proxy_url)
|
||||
sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
|
||||
if not sitemap_urls:
|
||||
raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
|
||||
|
||||
all_vehicle_urls: list[str] = []
|
||||
for sitemap_url in sitemap_urls:
|
||||
logger.info("Downloading sitemap: %s", sitemap_url)
|
||||
try:
|
||||
sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
|
||||
raw_urls = list(_iter_loc_values(sitemap_xml))
|
||||
except SitemapDiscoveryError as exc:
|
||||
logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
|
||||
continue
|
||||
|
||||
vehicle_urls = _filter_vehicle_urls(raw_urls)
|
||||
if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
|
||||
logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
|
||||
continue
|
||||
logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
|
||||
all_vehicle_urls.extend(vehicle_urls)
|
||||
|
||||
deduped = _filter_vehicle_urls(all_vehicle_urls)
|
||||
if not deduped:
|
||||
raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
|
||||
logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
|
||||
return deduped
|
||||
|
||||
|
||||
@dataclass
|
||||
class SitemapDiscoveryStats:
|
||||
transport: str = "http"
|
||||
fetched_sitemaps: int = 0
|
||||
blocked_sitemaps: int = 0
|
||||
malformed_sitemaps: int = 0
|
||||
direct_probe_hits: int = 0
|
||||
direct_probe_misses: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SitemapDiscoveryResult:
|
||||
vehicle_urls: list[str] = field(default_factory=list)
|
||||
stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
|
||||
|
||||
|
||||
def discover_vehicle_urls_from_sitemap_with_stats(
|
||||
settings=None,
|
||||
index_url: str = DEFAULT_SITEMAP_INDEX_URL,
|
||||
) -> SitemapDiscoveryResult:
|
||||
"""Возвращает URL и статистику."""
|
||||
proxy_url = None
|
||||
if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
|
||||
proxy_url = settings.proxy.server
|
||||
stats = SitemapDiscoveryStats(transport="http")
|
||||
logger.info("Downloading sitemap index: %s", index_url)
|
||||
index_xml = _download_bytes(index_url, proxy_url=proxy_url)
|
||||
sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
|
||||
if not sitemap_urls:
|
||||
raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
|
||||
|
||||
all_vehicle_urls: list[str] = []
|
||||
for sitemap_url in sitemap_urls:
|
||||
logger.info("Downloading sitemap: %s", sitemap_url)
|
||||
try:
|
||||
sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
|
||||
raw_urls = list(_iter_loc_values(sitemap_xml))
|
||||
stats.fetched_sitemaps += 1
|
||||
except SitemapDiscoveryError as exc:
|
||||
logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
|
||||
stats.malformed_sitemaps += 1
|
||||
continue
|
||||
except Exception as exc:
|
||||
logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
|
||||
stats.blocked_sitemaps += 1
|
||||
continue
|
||||
|
||||
vehicle_urls = _filter_vehicle_urls(raw_urls)
|
||||
if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
|
||||
logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
|
||||
continue
|
||||
logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
|
||||
all_vehicle_urls.extend(vehicle_urls)
|
||||
|
||||
deduped = _filter_vehicle_urls(all_vehicle_urls)
|
||||
if not deduped:
|
||||
raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
|
||||
logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
|
||||
return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)
|
||||
@@ -1,472 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import concurrent.futures
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from typing import Any, Callable
|
||||
|
||||
from .browser.fast_client import FastListingVehicle, MobiledeFastClient
|
||||
from .core.runtime_config import RuntimeConfig
|
||||
from .parsing.fast_mapper import INACTIVE_STATUS_VALUES, FastCarMapper
|
||||
from .storage.db import PersistenceService
|
||||
from .storage.schemas import CarRecord
|
||||
|
||||
logger = logging.getLogger("MOBILEDE_scraper.fast_sync")
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FastSyncStats:
|
||||
ids_fetched: int = 0
|
||||
cars_upserted: int = 0
|
||||
cars_failed: int = 0
|
||||
cars_filtered: int = 0
|
||||
images_upserted: int = 0
|
||||
skipped_existing: int = 0
|
||||
protection_events: int = 0
|
||||
|
||||
|
||||
def passes_condition_check(row: FastListingVehicle) -> bool:
|
||||
if row.timed_auction_closed:
|
||||
return False
|
||||
status = (row.inventory_status or "").strip().upper()
|
||||
return status not in INACTIVE_STATUS_VALUES
|
||||
|
||||
|
||||
class FastSyncEngine:
|
||||
"""Full MOBILEDE-fast style sync pipeline adapted to this project's storage.
|
||||
|
||||
Flow: hidden listing payloads -> concurrent ProductDetailsVM HTTP fetch ->
|
||||
CarRecord preparation in memory -> single batch DB upsert. Browser is used
|
||||
only inside MobiledeFastClient to refresh cookies when MOBILEDE challenge appears.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
*,
|
||||
client: MobiledeFastClient,
|
||||
mapper: FastCarMapper,
|
||||
persistence: PersistenceService,
|
||||
batch_size: int,
|
||||
fetch_concurrency: int,
|
||||
report_progress: Callable[[str, Any], None] | None = None,
|
||||
) -> None:
|
||||
self.client = client
|
||||
self.mapper = mapper
|
||||
self.persistence = persistence
|
||||
self.batch_size = max(1, int(batch_size))
|
||||
self.fetch_concurrency = max(1, int(fetch_concurrency))
|
||||
self.report_progress = report_progress
|
||||
|
||||
@staticmethod
|
||||
def _is_transient_detail_error(exc: Exception) -> bool:
|
||||
text = str(exc).lower()
|
||||
return any(
|
||||
marker in text
|
||||
for marker in (
|
||||
"sslerror",
|
||||
"ssleoferror",
|
||||
"unexpected_eof_while_reading",
|
||||
"eof occurred in violation of protocol",
|
||||
"max retries exceeded",
|
||||
"read timed out",
|
||||
"readtimeout",
|
||||
"connection reset",
|
||||
"connection aborted",
|
||||
"connection closed",
|
||||
"temporarily unavailable",
|
||||
"too many requests",
|
||||
"status=429",
|
||||
"status=500",
|
||||
"status=502",
|
||||
"status=503",
|
||||
"status=504",
|
||||
)
|
||||
)
|
||||
|
||||
def sync_listing(
|
||||
self,
|
||||
*,
|
||||
runtime_config: RuntimeConfig,
|
||||
make: str | None = None,
|
||||
model: str | None = None,
|
||||
lane: str = "MOBILEDE_cars",
|
||||
limit: int | None = None,
|
||||
only_new: bool = False,
|
||||
listing_url: str | None = None,
|
||||
max_pages: int | None = None,
|
||||
skip_mark_sold: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
del lane
|
||||
started_at = time.perf_counter()
|
||||
stats = FastSyncStats()
|
||||
errors: list[dict[str, str]] = []
|
||||
selected: dict[str, FastListingVehicle] = {}
|
||||
rows_seen = 0
|
||||
rows_skipped_condition = 0
|
||||
|
||||
filters = runtime_config.filters
|
||||
logger.info(
|
||||
"Fast HTTP-first listing started: make=%s listing_url=%s max_pages=%s concurrency=%s batch_size=%s",
|
||||
make or "ALL",
|
||||
listing_url or "default",
|
||||
max_pages,
|
||||
self.fetch_concurrency,
|
||||
self.batch_size,
|
||||
)
|
||||
for vehicle in self.client.iter_listing_vehicles(
|
||||
listing_start_url=listing_url,
|
||||
make=make,
|
||||
max_pages=max_pages,
|
||||
):
|
||||
rows_seen += 1
|
||||
if runtime_config.sync.condition_check_enabled and not passes_condition_check(vehicle):
|
||||
rows_skipped_condition += 1
|
||||
continue
|
||||
if vehicle.inventory_id in selected:
|
||||
continue
|
||||
selected[vehicle.inventory_id] = vehicle
|
||||
if limit is not None and limit > 0 and len(selected) >= limit:
|
||||
break
|
||||
|
||||
candidates = list(selected.values())
|
||||
all_listing_origin_urls = {f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates}
|
||||
if only_new and candidates:
|
||||
origin_urls = [f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates]
|
||||
origin_ids = [f"mobilede:{v.inventory_id}" for v in candidates]
|
||||
existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(origin_urls, origin_ids)
|
||||
fresh: list[FastListingVehicle] = []
|
||||
for vehicle in candidates:
|
||||
if f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}" in existing_urls or f"mobilede:{vehicle.inventory_id}" in existing_ids:
|
||||
stats.skipped_existing += 1
|
||||
continue
|
||||
fresh.append(vehicle)
|
||||
candidates = fresh
|
||||
|
||||
self._progress(
|
||||
"fast_listing_collected",
|
||||
rows_seen=rows_seen,
|
||||
rows_filtered_condition=rows_skipped_condition,
|
||||
rows_selected=len(candidates),
|
||||
skipped_existing=stats.skipped_existing,
|
||||
)
|
||||
logger.info(
|
||||
"Fast HTTP-first listing collected: rows_seen=%d selected=%d skipped_existing=%d filtered_condition=%d only_new=%s",
|
||||
rows_seen,
|
||||
len(candidates),
|
||||
stats.skipped_existing,
|
||||
rows_skipped_condition,
|
||||
only_new,
|
||||
)
|
||||
|
||||
scan_completed = not (limit is not None and limit > 0) and not errors
|
||||
|
||||
if not candidates:
|
||||
return self._result(
|
||||
started_at=started_at,
|
||||
stats=stats,
|
||||
failures=errors,
|
||||
listing={
|
||||
"mode": "fast_hidden_payload",
|
||||
"vehicles_collected": 0,
|
||||
"vehicle_urls": [],
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
"rows_seen": rows_seen,
|
||||
"rows_filtered_condition": rows_skipped_condition,
|
||||
},
|
||||
all_listing_origin_urls=all_listing_origin_urls,
|
||||
full_scan_completed=scan_completed,
|
||||
)
|
||||
|
||||
prepared_rows: list[CarRecord] = []
|
||||
db_processed = 0
|
||||
retry_candidates: list[FastListingVehicle] = []
|
||||
started_details = time.perf_counter()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor:
|
||||
future_to_vehicle = {
|
||||
executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
|
||||
for vehicle in candidates
|
||||
}
|
||||
for index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
|
||||
vehicle = future_to_vehicle[future]
|
||||
try:
|
||||
payload = future.result()
|
||||
record = self.mapper.map_payload_to_record(
|
||||
detail_payload=payload,
|
||||
vehicle_url=f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}",
|
||||
listing_vehicle=vehicle,
|
||||
)
|
||||
if model and model.casefold() not in record.model.casefold():
|
||||
stats.cars_filtered += 1
|
||||
continue
|
||||
if not filters.matches({
|
||||
"brand": record.brand,
|
||||
"model": record.model,
|
||||
"year": record.year,
|
||||
"body_type": record.body_type,
|
||||
"color": record.color,
|
||||
"drive": record.drive,
|
||||
"gearbox": record.gearbox,
|
||||
"price": record.price,
|
||||
"mileage": record.mileage,
|
||||
}):
|
||||
stats.cars_filtered += 1
|
||||
continue
|
||||
prepared_rows.append(record)
|
||||
stats.ids_fetched += 1
|
||||
if len(prepared_rows) >= self.batch_size:
|
||||
db_processed += self._flush_db_records(
|
||||
rows=prepared_rows,
|
||||
stats=stats,
|
||||
errors=errors,
|
||||
processed=db_processed + len(prepared_rows),
|
||||
total=len(candidates),
|
||||
)
|
||||
prepared_rows.clear()
|
||||
except Exception as exc:
|
||||
stats.cars_failed += 1
|
||||
errors.append({"vehicle_url": f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}", "error": str(exc)})
|
||||
if _looks_like_protection(exc):
|
||||
stats.protection_events += 1
|
||||
if self._is_transient_detail_error(exc):
|
||||
retry_candidates.append(vehicle)
|
||||
logger.info("Fast detail transient failure queued for retry inventory_id=%s: %s", vehicle.inventory_id, exc)
|
||||
else:
|
||||
logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc)
|
||||
|
||||
if index % 100 == 0 or index == len(candidates):
|
||||
elapsed = max(0.001, time.perf_counter() - started_details)
|
||||
if self.client._settings.scraping_profile.verbose_progress_logs:
|
||||
logger.info(
|
||||
"Fast HTTP-first details progress: processed=%d/%d ok=%d failed=%d queued_db=%d rate=%.2f/s",
|
||||
index,
|
||||
len(candidates),
|
||||
stats.ids_fetched,
|
||||
stats.cars_failed,
|
||||
len(prepared_rows),
|
||||
index / elapsed,
|
||||
)
|
||||
self._progress(
|
||||
"fast_detail_progress",
|
||||
processed=index,
|
||||
total=len(candidates),
|
||||
ids_fetched=stats.ids_fetched,
|
||||
cars_failed=stats.cars_failed,
|
||||
queued_for_db=len(prepared_rows),
|
||||
throughput=round(index / elapsed, 2),
|
||||
)
|
||||
|
||||
if retry_candidates:
|
||||
retry_started = time.perf_counter()
|
||||
retry_workers = max(1, min(8, self.fetch_concurrency // 2, len(retry_candidates)))
|
||||
retry_errors: list[dict[str, str]] = []
|
||||
logger.info(
|
||||
"Fast HTTP-first retrying transient detail failures: total=%d workers=%d",
|
||||
len(retry_candidates),
|
||||
retry_workers,
|
||||
)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=retry_workers) as executor:
|
||||
future_to_vehicle = {
|
||||
executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
|
||||
for vehicle in retry_candidates
|
||||
}
|
||||
for retry_index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
|
||||
vehicle = future_to_vehicle[future]
|
||||
try:
|
||||
payload = future.result()
|
||||
record = self.mapper.map_payload_to_record(
|
||||
detail_payload=payload,
|
||||
vehicle_url=f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}",
|
||||
listing_vehicle=vehicle,
|
||||
)
|
||||
if model and model.casefold() not in record.model.casefold():
|
||||
stats.cars_filtered += 1
|
||||
continue
|
||||
if not filters.matches({
|
||||
"brand": record.brand,
|
||||
"model": record.model,
|
||||
"year": record.year,
|
||||
"body_type": record.body_type,
|
||||
"color": record.color,
|
||||
"drive": record.drive,
|
||||
"gearbox": record.gearbox,
|
||||
"price": record.price,
|
||||
"mileage": record.mileage,
|
||||
}):
|
||||
stats.cars_filtered += 1
|
||||
continue
|
||||
prepared_rows.append(record)
|
||||
stats.ids_fetched += 1
|
||||
stats.cars_failed = max(0, stats.cars_failed - 1)
|
||||
if len(prepared_rows) >= self.batch_size:
|
||||
db_processed += self._flush_db_records(
|
||||
rows=prepared_rows,
|
||||
stats=stats,
|
||||
errors=errors,
|
||||
processed=db_processed + len(prepared_rows),
|
||||
total=len(candidates),
|
||||
)
|
||||
prepared_rows.clear()
|
||||
except Exception as exc:
|
||||
retry_errors.append({
|
||||
"vehicle_url": f"https://www.MOBILEDE.com/VehicleDetail/{vehicle.inventory_id}",
|
||||
"error": str(exc),
|
||||
})
|
||||
if _looks_like_protection(exc):
|
||||
stats.protection_events += 1
|
||||
if retry_index % 100 == 0 or retry_index == len(retry_candidates):
|
||||
elapsed = max(0.001, time.perf_counter() - retry_started)
|
||||
logger.info(
|
||||
"Fast HTTP-first retry progress: processed=%d/%d recovered=%d remaining_failed=%d rate=%.2f/s",
|
||||
retry_index,
|
||||
len(retry_candidates),
|
||||
len(retry_candidates) - len(retry_errors),
|
||||
len(retry_errors),
|
||||
retry_index / elapsed,
|
||||
)
|
||||
transient_urls = {f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in retry_candidates}
|
||||
errors = [error for error in errors if error.get("vehicle_url") not in transient_urls]
|
||||
errors.extend(retry_errors)
|
||||
|
||||
if prepared_rows:
|
||||
db_processed += self._flush_db_records(
|
||||
rows=prepared_rows,
|
||||
stats=stats,
|
||||
errors=errors,
|
||||
processed=db_processed + len(prepared_rows),
|
||||
total=len(candidates),
|
||||
)
|
||||
prepared_rows.clear()
|
||||
|
||||
self.client.persist_session_state()
|
||||
|
||||
scan_completed = not (limit is not None and limit > 0) and not errors
|
||||
mark_sold_scope_partial = bool(
|
||||
(limit is not None and limit > 0)
|
||||
or make
|
||||
or model
|
||||
or listing_url
|
||||
or only_new
|
||||
)
|
||||
if all_listing_origin_urls and not mark_sold_scope_partial and not skip_mark_sold and scan_completed:
|
||||
try:
|
||||
sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls, lane="MOBILEDE")
|
||||
except Exception as exc:
|
||||
sold_count = 0
|
||||
logger.warning("Fast sold reconcile failed: %s", exc)
|
||||
else:
|
||||
sold_count = 0
|
||||
|
||||
listing = {
|
||||
"mode": "fast_hidden_payload",
|
||||
"vehicles_collected": len(candidates),
|
||||
"vehicle_urls": [f"https://www.MOBILEDE.com/VehicleDetail/{v.inventory_id}" for v in candidates],
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
"rows_seen": rows_seen,
|
||||
"rows_filtered_condition": rows_skipped_condition,
|
||||
"sold_marked": sold_count,
|
||||
}
|
||||
return self._result(
|
||||
started_at=started_at,
|
||||
stats=stats,
|
||||
failures=errors,
|
||||
listing=listing,
|
||||
all_listing_origin_urls=all_listing_origin_urls,
|
||||
full_scan_completed=scan_completed,
|
||||
)
|
||||
|
||||
def _result(
|
||||
self,
|
||||
*,
|
||||
started_at: float,
|
||||
stats: FastSyncStats,
|
||||
failures: list[dict[str, str]],
|
||||
listing: dict[str, Any],
|
||||
all_listing_origin_urls: set[str],
|
||||
full_scan_completed: bool,
|
||||
) -> dict[str, Any]:
|
||||
status = "success" if not failures else ("partial_success" if stats.cars_upserted else "failed")
|
||||
total = int(listing.get("vehicles_collected") or 0)
|
||||
fail_ratio = (stats.cars_failed / total) if total > 0 else 0.0
|
||||
protection_ratio = (stats.protection_events / total) if total > 0 else 0.0
|
||||
anti_bot_detected = total > 0 and ((stats.protection_events >= 30 and protection_ratio >= 0.10) or fail_ratio >= 0.30)
|
||||
return {
|
||||
"status": status,
|
||||
"listing": listing,
|
||||
"total": total,
|
||||
"total_discovered": total,
|
||||
"skipped_existing": stats.skipped_existing,
|
||||
"cars_upserted": stats.cars_upserted,
|
||||
"cars_failed": stats.cars_failed,
|
||||
"cars_filtered": stats.cars_filtered,
|
||||
"images_upserted": stats.images_upserted,
|
||||
"protection_events": stats.protection_events,
|
||||
"failures": failures,
|
||||
"all_listing_origin_urls": all_listing_origin_urls,
|
||||
"full_scan_completed": full_scan_completed and not anti_bot_detected,
|
||||
"anti_bot_detected": anti_bot_detected,
|
||||
"fail_ratio": round(fail_ratio, 4),
|
||||
"protection_ratio": round(protection_ratio, 4),
|
||||
"elapsed_seconds": round(time.perf_counter() - started_at, 3),
|
||||
}
|
||||
|
||||
def _progress(self, stage: str, **meta: Any) -> None:
|
||||
if self.report_progress is None:
|
||||
return
|
||||
try:
|
||||
self.report_progress(stage, **meta)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _fetch_detail_payload(self, inventory_id: str) -> dict[str, Any]:
|
||||
jitter = float(self.client._settings.scraping_profile.request_jitter_max_s)
|
||||
if jitter > 0:
|
||||
time.sleep(random.uniform(0.0, jitter))
|
||||
return self.client.fetch_vehicle_detail_payload(inventory_id)
|
||||
|
||||
def _flush_db_records(
|
||||
self,
|
||||
*,
|
||||
rows: list[CarRecord],
|
||||
stats: FastSyncStats,
|
||||
errors: list[dict[str, str]],
|
||||
processed: int,
|
||||
total: int,
|
||||
) -> int:
|
||||
if not rows:
|
||||
return 0
|
||||
batch = list(rows)
|
||||
try:
|
||||
upsert = self.persistence.upsert_cars_batch(batch)
|
||||
stats.cars_upserted += int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0))
|
||||
stats.images_upserted += int(upsert.get("images_upserted", 0))
|
||||
except Exception as exc:
|
||||
stats.cars_failed += len(batch)
|
||||
errors.append({"vehicle_url": f"db_batch_{processed - len(batch)}", "error": str(exc)})
|
||||
logger.exception("Fast DB apply failed processed=%s size=%s: %s", processed, len(batch), exc)
|
||||
self._progress(
|
||||
"fast_db_progress",
|
||||
processed=processed,
|
||||
total=total,
|
||||
cars_upserted=stats.cars_upserted,
|
||||
cars_failed=stats.cars_failed,
|
||||
images_upserted=stats.images_upserted,
|
||||
)
|
||||
logger.info(
|
||||
"Fast HTTP-first DB batch: processed=%d/%d batch=%d upserted=%d failed=%d images=%d",
|
||||
processed,
|
||||
total,
|
||||
len(batch),
|
||||
stats.cars_upserted,
|
||||
stats.cars_failed,
|
||||
stats.images_upserted,
|
||||
)
|
||||
return len(batch)
|
||||
|
||||
|
||||
def _looks_like_protection(exc: Exception) -> bool:
|
||||
message = str(exc).lower()
|
||||
return any(token in message for token in ("captcha", "antibot", "challenge", "blocked", "403", "429", "incapsula"))
|
||||
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from collections.abc import Callable, Iterable
|
||||
@@ -34,6 +33,16 @@ MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BAC
|
||||
MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
|
||||
MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
|
||||
MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
|
||||
MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip()
|
||||
MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120")))
|
||||
MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS = max(1000, int(os.getenv("MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS", "60000")))
|
||||
MOBILEDE_FLARESOLVERR_SESSION = os.getenv("MOBILEDE_FLARESOLVERR_SESSION", "").strip()
|
||||
MOBILEDE_FLARESOLVERR_STATUSES = {
|
||||
int(item.strip())
|
||||
for item in os.getenv("MOBILEDE_FLARESOLVERR_STATUSES", "403,429,503").split(",")
|
||||
if item.strip().isdigit()
|
||||
}
|
||||
|
||||
|
||||
class MobileDeClient:
|
||||
@@ -43,11 +52,24 @@ class MobileDeClient:
|
||||
self.session = session or requests.Session()
|
||||
self.session.headers.update(DEFAULT_HEADERS)
|
||||
self.delay_seconds = max(0.0, delay_seconds)
|
||||
self.proxy_config = ProxyConfig()
|
||||
proxies = self.proxy_config.to_requests_proxies()
|
||||
if proxies:
|
||||
self.session.proxies.update(proxies)
|
||||
|
||||
@staticmethod
|
||||
def _is_retryable_status(status_code: int) -> bool:
|
||||
return int(status_code) in MOBILEDE_HTTP_RETRY_STATUSES
|
||||
|
||||
@staticmethod
|
||||
def _should_use_flaresolverr(status_code: int | None) -> bool:
|
||||
return (
|
||||
MOBILEDE_FLARESOLVERR_ENABLED
|
||||
and bool(MOBILEDE_FLARESOLVERR_URL)
|
||||
and status_code is not None
|
||||
and int(status_code) in MOBILEDE_FLARESOLVERR_STATUSES
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _compute_backoff(attempt: int) -> float:
|
||||
base = MOBILEDE_HTTP_BACKOFF_BASE_SECONDS * (2 ** max(0, attempt - 1))
|
||||
@@ -61,12 +83,10 @@ class MobileDeClient:
|
||||
adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
proxy_cfg = ProxyConfig()
|
||||
proxies = proxy_cfg.to_requests_proxies()
|
||||
if proxies:
|
||||
session.proxies.update(proxies)
|
||||
logger.info("mobile.de worker HTTP client using proxy: %s", proxy_cfg.server)
|
||||
return cls(session=session, delay_seconds=delay_seconds)
|
||||
client = cls(session=session, delay_seconds=delay_seconds)
|
||||
if client.proxy_config.enabled:
|
||||
logger.info("mobile.de worker HTTP client using proxy: %s", client.proxy_config.server)
|
||||
return client
|
||||
|
||||
@staticmethod
|
||||
def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
|
||||
@@ -118,19 +138,25 @@ class MobileDeClient:
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
response = self.session.get(url, timeout=timeout)
|
||||
if self._is_retryable_status(response.status_code) and attempt < attempts:
|
||||
sleep_seconds = self._compute_backoff(attempt)
|
||||
logger.warning(
|
||||
"mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
|
||||
response.status_code,
|
||||
attempt,
|
||||
attempts,
|
||||
sleep_seconds,
|
||||
url,
|
||||
)
|
||||
if sleep_seconds:
|
||||
time.sleep(sleep_seconds)
|
||||
continue
|
||||
if self._is_retryable_status(response.status_code):
|
||||
if attempt < attempts:
|
||||
sleep_seconds = self._compute_backoff(attempt)
|
||||
logger.warning(
|
||||
"mobile.de retryable status=%s attempt=%s/%s sleep=%.2fs url=%s",
|
||||
response.status_code,
|
||||
attempt,
|
||||
attempts,
|
||||
sleep_seconds,
|
||||
url,
|
||||
)
|
||||
if sleep_seconds:
|
||||
time.sleep(sleep_seconds)
|
||||
continue
|
||||
if self._should_use_flaresolverr(response.status_code):
|
||||
try:
|
||||
return self._fetch_html_with_flaresolverr(url)
|
||||
except Exception as exc:
|
||||
logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, exc)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
except requests.RequestException as exc:
|
||||
@@ -138,6 +164,11 @@ class MobileDeClient:
|
||||
status_code = getattr(getattr(exc, "response", None), "status_code", None)
|
||||
retryable = bool(status_code is not None and self._is_retryable_status(int(status_code)))
|
||||
if attempt >= attempts or not retryable:
|
||||
if self._should_use_flaresolverr(status_code):
|
||||
try:
|
||||
return self._fetch_html_with_flaresolverr(url)
|
||||
except Exception as flaresolverr_exc:
|
||||
logger.warning("mobile.de FlareSolverr fallback failed url=%s error=%s", url, flaresolverr_exc)
|
||||
raise
|
||||
sleep_seconds = self._compute_backoff(attempt)
|
||||
logger.warning(
|
||||
@@ -155,6 +186,56 @@ class MobileDeClient:
|
||||
raise last_error
|
||||
raise RuntimeError("mobile.de fetch_html failed without a captured exception")
|
||||
|
||||
def _fetch_html_with_flaresolverr(self, url: str) -> str:
|
||||
payload: dict[str, object] = {
|
||||
"cmd": "request.get",
|
||||
"url": url,
|
||||
"maxTimeout": MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS,
|
||||
}
|
||||
if MOBILEDE_FLARESOLVERR_SESSION:
|
||||
payload["session"] = MOBILEDE_FLARESOLVERR_SESSION
|
||||
|
||||
response = requests.post(
|
||||
MOBILEDE_FLARESOLVERR_URL,
|
||||
json=payload,
|
||||
timeout=MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS,
|
||||
)
|
||||
response.raise_for_status()
|
||||
data = response.json()
|
||||
if data.get("status") != "ok":
|
||||
raise RuntimeError(str(data.get("message") or data))
|
||||
|
||||
solution = data.get("solution")
|
||||
if not isinstance(solution, dict):
|
||||
raise RuntimeError("FlareSolverr response does not contain solution")
|
||||
|
||||
html = solution.get("response")
|
||||
if not isinstance(html, str) or not html:
|
||||
raise RuntimeError("FlareSolverr response does not contain HTML")
|
||||
|
||||
user_agent = solution.get("userAgent")
|
||||
if isinstance(user_agent, str) and user_agent:
|
||||
self.session.headers.update({"user-agent": user_agent})
|
||||
|
||||
cookies = solution.get("cookies")
|
||||
if isinstance(cookies, list):
|
||||
for cookie in cookies:
|
||||
if not isinstance(cookie, dict):
|
||||
continue
|
||||
name = cookie.get("name")
|
||||
value = cookie.get("value")
|
||||
if not isinstance(name, str) or not isinstance(value, str):
|
||||
continue
|
||||
self.session.cookies.set(
|
||||
name,
|
||||
value,
|
||||
domain=cookie.get("domain") if isinstance(cookie.get("domain"), str) else None,
|
||||
path=cookie.get("path") if isinstance(cookie.get("path"), str) else "/",
|
||||
)
|
||||
|
||||
logger.info("mobile.de fetched via FlareSolverr url=%s", url)
|
||||
return html
|
||||
|
||||
def fetch_search_page(
|
||||
self,
|
||||
page_number: int = 1,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
from __future__ import annotations
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
@@ -11,23 +11,27 @@ from .models import MobileDeListing
|
||||
|
||||
_BODY_MAP = {
|
||||
"cabrio": "OPEN",
|
||||
"кабриолет": "OPEN",
|
||||
"roadster": "OPEN",
|
||||
"кабрио": "OPEN",
|
||||
"limousine": "SEDAN",
|
||||
"седан": "SEDAN",
|
||||
"sedan": "SEDAN",
|
||||
"сeдан": "SEDAN",
|
||||
"suv": "SUV",
|
||||
"внедорожник": "SUV",
|
||||
"внедорож": "SUV",
|
||||
"kombi": "STATION_WAGON",
|
||||
"estate": "STATION_WAGON",
|
||||
"универсал": "STATION_WAGON",
|
||||
"van": "MINIVAN",
|
||||
"фургон": "MINIVAN",
|
||||
"минивэн": "MINIVAN",
|
||||
"coupe": "COUPE",
|
||||
"купе": "COUPE",
|
||||
"hatchback": "HATCHBACK",
|
||||
"kleinwagen": "HATCHBACK",
|
||||
"маленький": "HATCHBACK",
|
||||
}
|
||||
|
||||
_GEARBOX_MAP = {
|
||||
"автомат": "AT",
|
||||
"automatik": "AT",
|
||||
"automatic": "AT",
|
||||
"механ": "MT",
|
||||
"manual": "MT",
|
||||
@@ -36,26 +40,33 @@ _GEARBOX_MAP = {
|
||||
|
||||
_COLOR_MAP = {
|
||||
"schwarz": "black",
|
||||
"черный": "black",
|
||||
"weiß": "white",
|
||||
"черн": "black",
|
||||
"black": "black",
|
||||
"weiss": "white",
|
||||
"белый": "white",
|
||||
"серый": "gray",
|
||||
"weiß": "white",
|
||||
"бел": "white",
|
||||
"white": "white",
|
||||
"grau": "gray",
|
||||
"сер": "gray",
|
||||
"gray": "gray",
|
||||
"silber": "silver",
|
||||
"сереб": "silver",
|
||||
"silver": "silver",
|
||||
"rot": "red",
|
||||
"красный": "red",
|
||||
"красн": "red",
|
||||
"red": "red",
|
||||
"blau": "blue",
|
||||
"синий": "blue",
|
||||
"син": "blue",
|
||||
"blue": "blue",
|
||||
"grün": "green",
|
||||
"gruen": "green",
|
||||
"зеленый": "green",
|
||||
"зелен": "green",
|
||||
"green": "green",
|
||||
}
|
||||
|
||||
|
||||
class MobileDeMapper:
|
||||
"""Map mobile.de search/detail payloads into the existing CarRecord schema."""
|
||||
"""Map mobile.de payloads into CarRecord."""
|
||||
|
||||
def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
|
||||
raw = listing.raw or {}
|
||||
@@ -76,7 +87,7 @@ class MobileDeMapper:
|
||||
price=self._money_to_int(listing.price or raw.get("p")),
|
||||
currency="EUR",
|
||||
mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
|
||||
country="NA",
|
||||
country=self._normalize_country("DE"),
|
||||
is_sold=False,
|
||||
color=self._normalize_color(attr.get("ecol")),
|
||||
drive=None,
|
||||
@@ -102,17 +113,64 @@ class MobileDeMapper:
|
||||
)
|
||||
|
||||
def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
|
||||
title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN")
|
||||
attrs = self._detail_attrs_by_tag(detail.get("attributes"))
|
||||
make = detail.get("make") if isinstance(detail.get("make"), dict) else {}
|
||||
model_payload = detail.get("model") if isinstance(detail.get("model"), dict) else {}
|
||||
contact = detail.get("contact") if isinstance(detail.get("contact"), dict) else {}
|
||||
|
||||
short_title = self._text(detail.get("shortTitle") or make.get("localized") or "UNKNOWN")
|
||||
subtitle = self._text(detail.get("subTitle"))
|
||||
fake_listing = MobileDeListing(
|
||||
id=str(listing_id),
|
||||
url=MobileDeClient.build_detail_url(listing_id),
|
||||
title=title,
|
||||
subtitle=subtitle,
|
||||
price=self._text(detail.get("price") or detail.get("p")),
|
||||
raw=detail,
|
||||
title = " ".join(part for part in [short_title, subtitle] if part)
|
||||
|
||||
brand = self._text(make.get("localized") or self._brand_from_title(short_title) or "UNKNOWN")
|
||||
model = self._text(model_payload.get("localized") or self._model_from_title(short_title, brand) or subtitle or "UNKNOWN")
|
||||
origin_id = self.origin_id(str(listing_id))
|
||||
|
||||
price_amount, price_currency = self._detail_price(detail.get("price"))
|
||||
mileage = self._int_from_text(attrs.get("mileage")) or 0
|
||||
year = self._year_from_first_registration(attrs.get("firstRegistration"))
|
||||
gearbox = self._normalize_gearbox(attrs.get("transmission"))
|
||||
body_type = self._normalize_body(attrs.get("category") or detail.get("category"))
|
||||
color = self._normalize_color(attrs.get("color") or attrs.get("manufacturerColorName"))
|
||||
|
||||
damage_text = self._text(attrs.get("damageCondition")).lower()
|
||||
is_damaged = ("дтп" in damage_text and "без дтп" not in damage_text) or bool(detail.get("hasDamage"))
|
||||
|
||||
owners_text = self._text(attrs.get("numPreviousOwners"))
|
||||
one_owner = owners_text in {"1", "01", "1.0"}
|
||||
|
||||
return CarRecord(
|
||||
parser_id=self._parser_id(origin_id),
|
||||
brand=brand[:50] or "UNKNOWN",
|
||||
model=model[:50] or "UNKNOWN",
|
||||
year=year,
|
||||
price=price_amount,
|
||||
currency=price_currency or "EUR",
|
||||
mileage=mileage,
|
||||
country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
|
||||
is_sold=False,
|
||||
color=color,
|
||||
drive=self._normalize_drive(attrs.get("wheelDrive") or attrs.get("drivetrain")),
|
||||
gearbox=gearbox,
|
||||
steering_wheel="LEFT",
|
||||
body_type=body_type,
|
||||
engine_volume=self._int_from_text(attrs.get("cubicCapacity") or attrs.get("cc")),
|
||||
selling_type="CLASSIFIED",
|
||||
one_owner=one_owner,
|
||||
new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
|
||||
is_hidden=False,
|
||||
origin="MOBILE_DE",
|
||||
origin_url=MobileDeClient.build_detail_url(listing_id),
|
||||
origin_id=origin_id,
|
||||
is_damaged=is_damaged,
|
||||
evaluation=self._text(detail.get("priceRating") or detail.get("rating")) or None,
|
||||
non_smoking=True,
|
||||
rental=False,
|
||||
repair_history=is_damaged,
|
||||
slug=self._slugify(title or f"{brand} {model}"),
|
||||
last_seen_at=datetime.now(timezone.utc),
|
||||
images=self._images_from_listing(detail),
|
||||
)
|
||||
return self.listing_to_car_record(fake_listing)
|
||||
|
||||
@staticmethod
|
||||
def origin_id(listing_id: str) -> str:
|
||||
@@ -129,6 +187,9 @@ class MobileDeMapper:
|
||||
|
||||
@classmethod
|
||||
def _money_to_int(cls, value: Any) -> int | None:
|
||||
if isinstance(value, dict):
|
||||
amount = ((value.get("grs") or {}).get("amount") if isinstance(value.get("grs"), dict) else None) or value.get("amount")
|
||||
return cls._int_from_text(amount)
|
||||
return cls._int_from_text(value)
|
||||
|
||||
@staticmethod
|
||||
@@ -167,6 +228,32 @@ class MobileDeMapper:
|
||||
return mapped
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_drive(value: Any) -> str | None:
|
||||
text = "" if value is None else str(value).lower()
|
||||
if any(marker in text for marker in ("front", "fwd", "перед")):
|
||||
return "FWD"
|
||||
if any(marker in text for marker in ("rear", "rwd", "зад")):
|
||||
return "RWD"
|
||||
if any(marker in text for marker in ("all", "awd", "4x4", "quattro", "полный")):
|
||||
return "4WD"
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_country(value: Any) -> str:
|
||||
text = "" if value is None else str(value).strip().upper()
|
||||
if text in {"DE", "GERMANY", "DEUTSCHLAND"}:
|
||||
return "DE"
|
||||
if text in {"US", "USA", "UNITED STATES"}:
|
||||
return "US"
|
||||
if text in {"CA", "CANADA"}:
|
||||
return "CA"
|
||||
if text in {"JP", "JAPAN"}:
|
||||
return "JP"
|
||||
if text in {"KR", "KOREA", "SOUTH KOREA"}:
|
||||
return "KR"
|
||||
return "NA"
|
||||
|
||||
@staticmethod
|
||||
def _normalize_body(value: Any) -> str:
|
||||
text = "" if value is None else str(value).lower()
|
||||
@@ -188,6 +275,33 @@ class MobileDeMapper:
|
||||
slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
|
||||
return slug[:180] or "mobilede-car"
|
||||
|
||||
@staticmethod
|
||||
def _detail_attrs_by_tag(value: Any) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
if not isinstance(value, list):
|
||||
return result
|
||||
for item in value:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
tag = str(item.get("tag") or "").strip()
|
||||
val = str(item.get("value") or "").strip()
|
||||
if tag and val and tag not in result:
|
||||
result[tag] = val
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def _detail_price(value: Any) -> tuple[int | None, str | None]:
|
||||
if not isinstance(value, dict):
|
||||
return None, None
|
||||
grs = value.get("grs") if isinstance(value.get("grs"), dict) else {}
|
||||
amount = grs.get("amount") if isinstance(grs, dict) else None
|
||||
currency = grs.get("currency") if isinstance(grs, dict) else None
|
||||
if amount is None:
|
||||
amount = value.get("amount")
|
||||
if currency is None:
|
||||
currency = value.get("currency")
|
||||
return MobileDeMapper._int_from_text(amount), (str(currency).strip() if currency else None)
|
||||
|
||||
@staticmethod
|
||||
def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
|
||||
urls: list[str] = []
|
||||
@@ -203,6 +317,13 @@ class MobileDeMapper:
|
||||
src = item.get("src") or item.get("url") or item.get("uri")
|
||||
if src:
|
||||
urls.append(MobileDeMapper._normalize_image_url(str(src)))
|
||||
media_gallery = raw.get("mediaGallery")
|
||||
if isinstance(media_gallery, list):
|
||||
for item in media_gallery:
|
||||
if isinstance(item, dict):
|
||||
src = item.get("uri") or item.get("url")
|
||||
if src:
|
||||
urls.append(MobileDeMapper._normalize_image_url(str(src)))
|
||||
return [
|
||||
ImageRecord(fullres_image=url, preview_image=url, order_index=index)
|
||||
for index, url in enumerate(dict.fromkeys(url for url in urls if url))
|
||||
|
||||
@@ -314,9 +314,7 @@ class MobileDeScraper:
|
||||
max_pages,
|
||||
)
|
||||
try:
|
||||
data = self.collect_search(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
params = self._build_search_params(
|
||||
search_url=search_url,
|
||||
make_id=make_id,
|
||||
model_id=model_id,
|
||||
@@ -328,96 +326,121 @@ class MobileDeScraper:
|
||||
mileage_max=mileage_max,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
pages_payload = list(data.get("pages", []))
|
||||
listing_count = int(data.get("listing_count", 0) or 0)
|
||||
unique_ids = set(data.get("unique_listing_ids", []))
|
||||
pages_collected = len(pages_payload)
|
||||
def _on_page(page, meta: dict[str, int | None]) -> None:
|
||||
if progress_callback is None:
|
||||
return
|
||||
payload = {
|
||||
**meta,
|
||||
"page_url": page.url,
|
||||
"unique_ids_seen": len(unique_ids) + len({listing.id for listing in page.listings if listing.id}),
|
||||
}
|
||||
progress_callback("page_collected", payload)
|
||||
|
||||
records: list[CarRecord] = []
|
||||
for page in pages_payload:
|
||||
page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])]
|
||||
records.extend(self._dedupe_page_records(page_records, seen_record_keys))
|
||||
pages_collected = 0
|
||||
early_stopped = False
|
||||
for page in self.client.iter_search_pages(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
):
|
||||
pages_collected += 1
|
||||
pages_payload.append(asdict(page))
|
||||
listing_count += len(page.listings)
|
||||
unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
|
||||
|
||||
if only_new and records:
|
||||
existing_origin_ids = self.persistence.get_existing_origin_ids(
|
||||
[record.origin_id for record in records if record.origin_id]
|
||||
)
|
||||
should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
|
||||
if should_cut_tail:
|
||||
filtered_records: list[CarRecord] = []
|
||||
for record_index, record in enumerate(records):
|
||||
is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
|
||||
filtered_records.append(record)
|
||||
if is_existing:
|
||||
existing_streak += 1
|
||||
if (
|
||||
existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
|
||||
and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
|
||||
):
|
||||
head_cut_triggered = True
|
||||
skipped_existing += max(0, len(records) - record_index - 1)
|
||||
break
|
||||
else:
|
||||
existing_streak = 0
|
||||
new_records_kept += 1
|
||||
records = filtered_records
|
||||
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"records_mapped",
|
||||
{
|
||||
"record_count": len(records),
|
||||
"skipped_existing": skipped_existing,
|
||||
"only_new": bool(only_new),
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
},
|
||||
page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
|
||||
page_records = self._dedupe_page_records(page_records, seen_record_keys)
|
||||
page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
|
||||
self._apply_only_new_page_policy(
|
||||
page_records=page_records,
|
||||
only_new=only_new,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
skipped_existing=skipped_existing,
|
||||
existing_streak=existing_streak,
|
||||
new_records_kept=new_records_kept,
|
||||
)
|
||||
)
|
||||
|
||||
upsert = self.persistence.upsert_cars_batch(records) if records else {
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"images_upserted": 0,
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"records_mapped",
|
||||
{
|
||||
"record_count": len(page_records),
|
||||
"skipped_existing": skipped_existing,
|
||||
"only_new": bool(only_new),
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
"page_number": page.page_number,
|
||||
},
|
||||
)
|
||||
|
||||
upsert = self.persistence.upsert_cars_batch(page_records) if page_records else {
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"images_upserted": 0,
|
||||
}
|
||||
page_inserted = int(upsert.get("inserted", 0))
|
||||
page_updated = int(upsert.get("updated", 0))
|
||||
page_images = int(upsert.get("images_upserted", 0))
|
||||
|
||||
ids_fetched += len(page_records)
|
||||
inserted_total += page_inserted
|
||||
updated_total += page_updated
|
||||
images_upserted += page_images
|
||||
cars_upserted = inserted_total + updated_total
|
||||
|
||||
logger.debug(
|
||||
"mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
|
||||
run_id,
|
||||
page.page_number,
|
||||
page_inserted,
|
||||
page_updated,
|
||||
page_images,
|
||||
)
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"db_upsert_done",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
"pages_in_batch": 1,
|
||||
"page_number": page.page_number,
|
||||
"inserted": page_inserted,
|
||||
"updated": page_updated,
|
||||
"images_upserted": page_images,
|
||||
},
|
||||
)
|
||||
|
||||
if head_cut_triggered:
|
||||
early_stopped = True
|
||||
break
|
||||
|
||||
data = {
|
||||
"source": "mobile.de",
|
||||
"strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
|
||||
"search_url": search_url,
|
||||
"pages": pages_payload,
|
||||
"listing_count": listing_count,
|
||||
"unique_listing_count": len(unique_ids),
|
||||
"unique_listing_ids": sorted(unique_ids),
|
||||
"early_stopped": early_stopped,
|
||||
}
|
||||
ids_fetched = len(records)
|
||||
inserted_total = int(upsert.get("inserted", 0))
|
||||
updated_total = int(upsert.get("updated", 0))
|
||||
images_upserted = int(upsert.get("images_upserted", 0))
|
||||
cars_upserted = inserted_total + updated_total
|
||||
|
||||
logger.debug(
|
||||
"mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s",
|
||||
run_id,
|
||||
pages_collected,
|
||||
inserted_total,
|
||||
updated_total,
|
||||
images_upserted,
|
||||
)
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"db_upsert_done",
|
||||
"search_collection_done",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
"pages_in_batch": pages_collected,
|
||||
"inserted": inserted_total,
|
||||
"updated": updated_total,
|
||||
"images_upserted": images_upserted,
|
||||
"listing_count": listing_count,
|
||||
"unique_listing_count": len(unique_ids),
|
||||
},
|
||||
)
|
||||
|
||||
if head_cut_triggered:
|
||||
logger.info(
|
||||
"mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s",
|
||||
len(records),
|
||||
skipped_existing,
|
||||
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
|
||||
)
|
||||
|
||||
data["early_stopped"] = head_cut_triggered
|
||||
self.persistence.finish_sync_run(
|
||||
run_id,
|
||||
status="success",
|
||||
@@ -432,7 +455,16 @@ class MobileDeScraper:
|
||||
data.get("listing_count", 0),
|
||||
data.get("unique_listing_count", 0),
|
||||
)
|
||||
return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data}
|
||||
return {
|
||||
"run_id": run_id,
|
||||
"upsert": {
|
||||
"inserted": inserted_total,
|
||||
"updated": updated_total,
|
||||
"images_upserted": images_upserted,
|
||||
},
|
||||
"skipped_existing": skipped_existing,
|
||||
**data,
|
||||
}
|
||||
except Exception as exc:
|
||||
self.persistence.finish_sync_run(
|
||||
run_id,
|
||||
|
||||
@@ -1 +0,0 @@
|
||||
__all__: list[str] = []
|
||||
@@ -1,515 +0,0 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text
|
||||
from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES
|
||||
from ..storage.schemas import CarRecord, ImageRecord
|
||||
|
||||
ORIGIN_PREFIX = "mobilede:"
|
||||
ORIGIN_URL_BASE = "https://www.MOBILEDE.com/VehicleDetail"
|
||||
COLOR_ALIASES = {
|
||||
"grau": "gray",
|
||||
"grau metallic": "gray",
|
||||
"gray": "gray",
|
||||
"grey": "gray",
|
||||
"silber": "silver",
|
||||
"silver": "silver",
|
||||
"schwarz": "black",
|
||||
"black": "black",
|
||||
"weiss": "white",
|
||||
"weiß": "white",
|
||||
"white": "white",
|
||||
"blau": "blue",
|
||||
"blue": "blue",
|
||||
"rot": "red",
|
||||
"red": "red",
|
||||
"gruen": "green",
|
||||
"grün": "green",
|
||||
"green": "green",
|
||||
"braun": "brown",
|
||||
"brown": "brown",
|
||||
"beige": "beige",
|
||||
"orange": "orange",
|
||||
"gelb": "yellow",
|
||||
"yellow": "yellow",
|
||||
"violett": "purple",
|
||||
"lila": "purple",
|
||||
"purple": "purple",
|
||||
"gold": "gold",
|
||||
}
|
||||
DAMAGE_NEUTRAL_VALUES = {
|
||||
"NORMAL WEAR & TEAR",
|
||||
"NORMAL WEAR",
|
||||
"NORMALWEAR&TEAR",
|
||||
"NONE",
|
||||
"NO DAMAGE",
|
||||
"NO VISIBLE DAMAGE",
|
||||
"MINOR DENT/SCRATCHES",
|
||||
}
|
||||
INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"}
|
||||
|
||||
|
||||
class FastCarMapper:
|
||||
"""Maps MOBILEDE ProductDetailsVM payloads directly to project CarRecord."""
|
||||
|
||||
def map_payload_to_record(
|
||||
self,
|
||||
*,
|
||||
detail_payload: dict[str, Any],
|
||||
vehicle_url: str | None = None,
|
||||
listing_vehicle: FastListingVehicle | None = None,
|
||||
) -> CarRecord:
|
||||
inventory_view = detail_payload.get("inventoryView")
|
||||
if not isinstance(inventory_view, dict):
|
||||
raise RuntimeError("detail payload missing inventoryView")
|
||||
attributes = inventory_view.get("attributes")
|
||||
if not isinstance(attributes, dict):
|
||||
raise RuntimeError("detail payload missing inventoryView.attributes")
|
||||
|
||||
inventory_id = parse_text(attributes.get("Id"))
|
||||
if not inventory_id and listing_vehicle is not None:
|
||||
inventory_id = listing_vehicle.inventory_id
|
||||
if not inventory_id and vehicle_url:
|
||||
inventory_id = self._inventory_id_from_url(vehicle_url)
|
||||
if not inventory_id:
|
||||
raise RuntimeError("missing inventory id")
|
||||
|
||||
brand = self._limit_text(self._attr_text(attributes, "Make") or "UNKNOWN", 50)
|
||||
model = self._build_model_name(
|
||||
self._attr_text(attributes, "Model"),
|
||||
self._attr_text(attributes, "Series", "Trim", "Variant"),
|
||||
) or "UNKNOWN"
|
||||
model = self._limit_text(model, 50)
|
||||
year = self._parse_year(
|
||||
self._attr_text(
|
||||
attributes,
|
||||
"Year",
|
||||
"FirstRegistration",
|
||||
"InitialRegistration",
|
||||
"Erstzulassung",
|
||||
"ModelYear",
|
||||
)
|
||||
)
|
||||
descriptor_text = " ".join(
|
||||
filter(
|
||||
None,
|
||||
[
|
||||
brand,
|
||||
model,
|
||||
self._attr_text(attributes, "BodyStyleName", "VehicleClass", "Category", "CategoryDescription"),
|
||||
self._attr_text(attributes, "DriveLineTypeDesc", "DriveType", "DriveTrain", "DriveTrainType", "DriveDescription"),
|
||||
self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType"),
|
||||
],
|
||||
)
|
||||
)
|
||||
|
||||
auction_info = detail_payload.get("auctionInformation")
|
||||
auction_info = auction_info if isinstance(auction_info, dict) else {}
|
||||
bidding_info = auction_info.get("biddingInformation")
|
||||
bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
|
||||
prebid_info = auction_info.get("prebidInformation")
|
||||
prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
|
||||
|
||||
high_bid = self._first_positive_int(
|
||||
prebid_info.get("decimalHighBidAmount"),
|
||||
prebid_info.get("highBidAmount"),
|
||||
bidding_info.get("highBidAmount"),
|
||||
)
|
||||
buy_now = self._first_positive_int(
|
||||
bidding_info.get("buyNowAmount"),
|
||||
prebid_info.get("buyNowPrice"),
|
||||
bidding_info.get("buyNowPrice"),
|
||||
)
|
||||
price = high_bid if high_bid is not None else buy_now
|
||||
|
||||
image_dimensions = inventory_view.get("imageDimensions")
|
||||
image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
|
||||
keys_container = image_dimensions.get("keys")
|
||||
keys_container = keys_container if isinstance(keys_container, dict) else {}
|
||||
image_keys = keys_container.get("$values")
|
||||
image_keys = image_keys if isinstance(image_keys, list) else []
|
||||
images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)]
|
||||
|
||||
primary_damage = self._attr_text(attributes, "PrimaryDamageDesc")
|
||||
secondary_damage = self._attr_text(attributes, "SecondaryDamageDesc")
|
||||
origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}"
|
||||
normalized_origin_id = self._normalize_origin_inventory_id(inventory_id)
|
||||
origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}"
|
||||
drive_source = self._attr_text(
|
||||
attributes,
|
||||
"DriveLineTypeDesc",
|
||||
"DriveType",
|
||||
"DriveTrain",
|
||||
"DriveTrainType",
|
||||
"DriveDescription",
|
||||
"Antrieb",
|
||||
) or descriptor_text
|
||||
gearbox_source = self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType") or descriptor_text
|
||||
body_type_source = self._attr_text(
|
||||
attributes,
|
||||
"BodyStyleName",
|
||||
"VehicleClass",
|
||||
"Category",
|
||||
"CategoryDescription",
|
||||
"BodyType",
|
||||
) or descriptor_text
|
||||
engine_source = self._attr_text(
|
||||
attributes,
|
||||
"EngineSize",
|
||||
"EngineInformation",
|
||||
"EngineDisplacement",
|
||||
"Displacement",
|
||||
"CubicCapacity",
|
||||
)
|
||||
color_source = self._attr_text(attributes, "ExteriorColor", "ColorDesc", "Color")
|
||||
|
||||
return CarRecord(
|
||||
parser_id=self._generate_parser_id(origin_id),
|
||||
brand=brand,
|
||||
model=model,
|
||||
year=year,
|
||||
price=price,
|
||||
currency=self._map_currency(self._attr_text(attributes, "Currency") or (listing_vehicle.currency if listing_vehicle else None)),
|
||||
mileage=self._parse_non_negative_int(self._attr_text(attributes, "ODOValue", "Mileage", "Kilometerstand", "Odometer")) or 0,
|
||||
country=self._map_country(inventory_id, tenant=listing_vehicle.tenant if listing_vehicle else None),
|
||||
is_sold=self._is_sold(listing_vehicle),
|
||||
color=self._normalize_color(color_source),
|
||||
drive=self._map_drive(drive_source),
|
||||
gearbox=self._map_gearbox(gearbox_source),
|
||||
steering_wheel="LEFT",
|
||||
body_type=self._map_body_type(body_type_source),
|
||||
engine_volume=self._parse_engine_volume(engine_source),
|
||||
selling_type="AUCTION",
|
||||
one_owner=False,
|
||||
new_car=False,
|
||||
is_hidden=not bool(images),
|
||||
origin="MOBILEDE",
|
||||
origin_url=origin_url,
|
||||
origin_id=origin_id,
|
||||
is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage),
|
||||
evaluation=self._attr_text(attributes, "VehicleGrade", "PriceRating"),
|
||||
non_smoking=True,
|
||||
rental=False,
|
||||
repair_history=False,
|
||||
slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))),
|
||||
last_seen_at=datetime.now(timezone.utc),
|
||||
images=images,
|
||||
)
|
||||
|
||||
def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]:
|
||||
inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {}
|
||||
inventory_view = inventory_view if isinstance(inventory_view, dict) else {}
|
||||
attr = inventory_view.get("attributes")
|
||||
attr = attr if isinstance(attr, dict) else {}
|
||||
auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {}
|
||||
auction_info = auction_info if isinstance(auction_info, dict) else {}
|
||||
bidding_info = auction_info.get("biddingInformation")
|
||||
bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
|
||||
prebid_info = auction_info.get("prebidInformation")
|
||||
prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
|
||||
image_dimensions = inventory_view.get("imageDimensions")
|
||||
image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
|
||||
keys_container = image_dimensions.get("keys")
|
||||
keys_container = keys_container if isinstance(keys_container, dict) else {}
|
||||
image_keys = keys_container.get("$values")
|
||||
image_keys = image_keys if isinstance(image_keys, list) else []
|
||||
image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)]
|
||||
engine_text = self._attr_text(attr, "EngineInformation", "EngineSize", "EngineDisplacement", "Displacement")
|
||||
return {
|
||||
"source_url": vehicle_url,
|
||||
"lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"),
|
||||
"year": attr.get("Year") or attr.get("FirstRegistration") or attr.get("Erstzulassung"),
|
||||
"make": attr.get("Make"),
|
||||
"model": attr.get("Model"),
|
||||
"trim": attr.get("Series"),
|
||||
"body_type": attr.get("BodyStyleName") or attr.get("VehicleClass") or attr.get("Category"),
|
||||
"drive": attr.get("DriveLineTypeDesc") or attr.get("DriveType") or attr.get("DriveTrain"),
|
||||
"engine": engine_text,
|
||||
"fuel_type": attr.get("FuelTypeCode"),
|
||||
"gearbox": attr.get("Transmission") or attr.get("Gearbox") or attr.get("TransmissionType"),
|
||||
"color": attr.get("ExteriorColor") or attr.get("ColorDesc") or attr.get("Color"),
|
||||
"primary_damage": attr.get("PrimaryDamageDesc"),
|
||||
"secondary_damage": attr.get("SecondaryDamageDesc"),
|
||||
"odometer": attr.get("ODOValue") or attr.get("Mileage") or attr.get("Kilometerstand"),
|
||||
"location": attr.get("BranchName"),
|
||||
"auction_date": attr.get("AuctionDateTime"),
|
||||
"title": attr.get("Title"),
|
||||
"current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"),
|
||||
"buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"),
|
||||
"actual_cash_value": attr.get("ProviderACV"),
|
||||
"estimated_repair_cost": attr.get("EstRepairCost"),
|
||||
"image_urls": image_urls,
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _inventory_id_from_url(vehicle_url: str) -> str | None:
|
||||
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
|
||||
return tail or None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_origin_inventory_id(inventory_id: str) -> str:
|
||||
return inventory_id.strip().split("~", 1)[0]
|
||||
|
||||
@staticmethod
|
||||
def _build_model_name(model: str | None, series: str | None) -> str:
|
||||
unique_parts: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for value in (model, series):
|
||||
if not value:
|
||||
continue
|
||||
normalized = " ".join(value.split())
|
||||
key = normalized.casefold()
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
unique_parts.append(normalized)
|
||||
return " ".join(unique_parts).strip()
|
||||
|
||||
@staticmethod
|
||||
def _parse_year(value: Any) -> int | None:
|
||||
if isinstance(value, str):
|
||||
match = re.search(r"(19|20)\d{2}", value)
|
||||
if match is not None:
|
||||
parsed = int(match.group(0))
|
||||
if 1900 <= parsed <= 2100:
|
||||
return parsed
|
||||
parsed = parse_int(value)
|
||||
if parsed is None or parsed < 1900 or parsed > 2100:
|
||||
return None
|
||||
return parsed
|
||||
|
||||
@staticmethod
|
||||
def _parse_non_negative_int(value: Any) -> int | None:
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
if not text:
|
||||
return None
|
||||
grouped_match = re.search(r"\d{1,3}(?:[.,\s]\d{3})+(?!\d)", text)
|
||||
if grouped_match is not None:
|
||||
digits_only = re.sub(r"\D", "", grouped_match.group(0))
|
||||
return int(digits_only) if digits_only else None
|
||||
if re.fullmatch(r"\d{1,3}(?:[.,\s]\d{3})+", text):
|
||||
digits_only = re.sub(r"\D", "", text)
|
||||
return int(digits_only) if digits_only else None
|
||||
parsed = parse_int(value)
|
||||
if parsed is None or parsed < 0:
|
||||
return None
|
||||
return parsed
|
||||
|
||||
@staticmethod
|
||||
def _attr_text(attributes: dict[str, Any], *keys: str) -> str | None:
|
||||
for key in keys:
|
||||
value = parse_text(attributes.get(key))
|
||||
if value:
|
||||
return value
|
||||
return None
|
||||
|
||||
@classmethod
|
||||
def _first_positive_int(cls, *values: Any) -> int | None:
|
||||
for value in values:
|
||||
parsed = cls._parse_non_negative_int(value)
|
||||
if parsed is not None and parsed > 0:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_color(value: str | None) -> str:
|
||||
if not value:
|
||||
return "other"
|
||||
normalized = FastCarMapper._simplify_text(value)
|
||||
if "/" in normalized:
|
||||
normalized = normalized.split("/", 1)[0].strip()
|
||||
for alias, mapped in COLOR_ALIASES.items():
|
||||
if alias in normalized:
|
||||
return mapped
|
||||
return normalized or "other"
|
||||
|
||||
@staticmethod
|
||||
def _map_currency(value: str | None) -> str:
|
||||
normalized = (value or "USD").strip().upper()
|
||||
return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD"
|
||||
|
||||
@staticmethod
|
||||
def _map_country(inventory_id: str, tenant: str | None = None) -> str:
|
||||
upper_id = inventory_id.strip().upper()
|
||||
if upper_id.endswith("~CA"):
|
||||
return "CA"
|
||||
if upper_id.endswith("~US"):
|
||||
return "US"
|
||||
tenant_normalized = (tenant or "").strip().upper()
|
||||
if tenant_normalized in {"US", "CA", "JP", "KR"}:
|
||||
return tenant_normalized
|
||||
return "NA"
|
||||
|
||||
@staticmethod
|
||||
def _map_drive(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
normalized = FastCarMapper._simplify_text(value)
|
||||
candidates: tuple[str, ...] | None = None
|
||||
if any(marker in normalized for marker in ("front wheel", "front-wheel", "frontantrieb", "fwd", "traction avant")):
|
||||
candidates = ("FWD",)
|
||||
elif any(
|
||||
marker in normalized
|
||||
for marker in (
|
||||
"all wheel",
|
||||
"all-wheel",
|
||||
"4x4",
|
||||
"awd",
|
||||
"four wheel",
|
||||
"allrad",
|
||||
"4matic",
|
||||
"4motion",
|
||||
"quattro",
|
||||
"xdrive",
|
||||
)
|
||||
):
|
||||
candidates = ("4WD", "FOUR_WD")
|
||||
elif any(marker in normalized for marker in ("rear wheel", "rear-wheel", "heckantrieb", "rwd", "propulsion")):
|
||||
candidates = ("RWD",)
|
||||
elif "2wd" in normalized or "two wheel" in normalized:
|
||||
candidates = ("2WD", "TWO_WD")
|
||||
elif "unknown" in normalized or normalized in {"na", "n/a"}:
|
||||
candidates = ("NA",)
|
||||
return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None
|
||||
|
||||
@staticmethod
|
||||
def _map_gearbox(value: str | None) -> str | None:
|
||||
if not value:
|
||||
return None
|
||||
normalized = FastCarMapper._simplify_text(value)
|
||||
candidates: tuple[str, ...] | None = None
|
||||
if "cvt" in normalized:
|
||||
candidates = ("CVT",)
|
||||
elif any(marker in normalized for marker in ("manual", "mt", "schaltgetriebe", "schaltung", "stick shift")):
|
||||
candidates = ("MT",)
|
||||
elif "electric" in normalized or normalized == "ev":
|
||||
candidates = ("EV",)
|
||||
elif any(marker in normalized for marker in ("auto", "at", "automatik", "automatic", "dsg", "doppelkupplung", "semi automatic", "halbautomatik")):
|
||||
candidates = ("AT",)
|
||||
elif "unknown" in normalized or normalized in {"na", "n/a"}:
|
||||
candidates = ("NA",)
|
||||
return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None
|
||||
|
||||
@staticmethod
|
||||
def _map_body_type(value: str | None) -> str:
|
||||
if not value:
|
||||
return "OTHER"
|
||||
normalized = FastCarMapper._simplify_text(value)
|
||||
candidates: tuple[str, ...] | None = None
|
||||
if any(marker in normalized for marker in ("sedan", "limousine", "saloon")):
|
||||
candidates = ("SEDAN",)
|
||||
elif any(marker in normalized for marker in ("sport utility", "suv", "crossover", "gelandewagen", "gelaendewagen", "off-road")):
|
||||
candidates = ("SUV",)
|
||||
elif any(marker in normalized for marker in ("hatch", "kleinwagen", "compact", "city car")):
|
||||
candidates = ("HATCHBACK",)
|
||||
elif any(marker in normalized for marker in ("wagon", "kombi", "estate", "touring", "variant", "avant", "shooting brake")):
|
||||
candidates = ("STATION_WAGON", "Station Wagon")
|
||||
elif "coupe" in normalized:
|
||||
candidates = ("COUPE",)
|
||||
elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized):
|
||||
candidates = ("PICKUP", "Pickup")
|
||||
elif any(marker in normalized for marker in ("convertible", "roadster", "cabrio", "cabriolet", "spyder")):
|
||||
candidates = ("OPEN", "Open")
|
||||
elif any(marker in normalized for marker in ("van", "bus", "people mover", "mpv", "minivan", "tourer")):
|
||||
candidates = ("MINIVAN",)
|
||||
elif any(marker in normalized for marker in ("truck", "chassis", "pritsche")):
|
||||
candidates = ("TRUCK", "Truck")
|
||||
elif "rv" in normalized or "motorized" in normalized:
|
||||
candidates = ("RV",)
|
||||
elif normalized in {"other", "unknown"}:
|
||||
candidates = ("OTHER", "Other")
|
||||
return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER"
|
||||
|
||||
@staticmethod
|
||||
def _parse_engine_volume(value: str | None) -> int | None:
|
||||
if not value:
|
||||
return None
|
||||
normalized = value.strip().replace(",", ".")
|
||||
match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", normalized)
|
||||
if not match:
|
||||
cc_match = re.search(r"(\d{3,5})\s*(?:ccm|cm3|cm³|cc)\b", normalized, flags=re.IGNORECASE)
|
||||
if cc_match is not None:
|
||||
cc = int(cc_match.group(1))
|
||||
return cc if 0 < cc <= 10000 else None
|
||||
raw_digits = re.fullmatch(r"\s*(\d{3,5})\s*", normalized)
|
||||
if raw_digits is not None:
|
||||
cc = int(raw_digits.group(1))
|
||||
return cc if 0 < cc <= 10000 else None
|
||||
return None
|
||||
try:
|
||||
liters = float(match.group(1))
|
||||
except ValueError:
|
||||
return None
|
||||
cc = int(round(liters * 1000))
|
||||
if cc <= 0 or cc > 10000:
|
||||
return None
|
||||
return cc
|
||||
|
||||
@staticmethod
|
||||
def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool:
|
||||
neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES}
|
||||
for value in (primary_damage, secondary_damage):
|
||||
if not value:
|
||||
continue
|
||||
normalized = value.replace(" ", "").strip().upper()
|
||||
if normalized and normalized not in neutral:
|
||||
return True
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool:
|
||||
if listing_vehicle is None:
|
||||
return False
|
||||
if listing_vehicle.timed_auction_closed:
|
||||
return True
|
||||
status = (listing_vehicle.inventory_status or "").strip().upper()
|
||||
return status in INACTIVE_STATUS_VALUES
|
||||
|
||||
@staticmethod
|
||||
def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None:
|
||||
if not candidates:
|
||||
return fallback if fallback in allowed else None
|
||||
allowed_set = set(allowed)
|
||||
for candidate in candidates:
|
||||
if candidate in allowed_set:
|
||||
return candidate
|
||||
return fallback if fallback in allowed_set else None
|
||||
|
||||
@staticmethod
|
||||
def _limit_text(value: str, max_length: int) -> str:
|
||||
return value if len(value) <= max_length else value[:max_length].rstrip()
|
||||
|
||||
@staticmethod
|
||||
def _simplify_text(value: str) -> str:
|
||||
return (
|
||||
value.strip()
|
||||
.lower()
|
||||
.replace("ä", "ae")
|
||||
.replace("ö", "oe")
|
||||
.replace("ü", "ue")
|
||||
.replace("ß", "ss")
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _slugify(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
|
||||
|
||||
@staticmethod
|
||||
def _generate_parser_id(origin_id: str) -> str:
|
||||
import hashlib
|
||||
from string import ascii_letters, digits
|
||||
|
||||
digest = hashlib.sha256(origin_id.encode()).digest()
|
||||
alphabet = ascii_letters + digits
|
||||
base = len(alphabet)
|
||||
num = int.from_bytes(digest[:17], "big")
|
||||
chars: list[str] = []
|
||||
for _ in range(22):
|
||||
num, idx = divmod(num, base)
|
||||
chars.append(alphabet[idx])
|
||||
return "car-" + "".join(chars)
|
||||
@@ -1,405 +0,0 @@
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from string import ascii_letters, digits
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ..core.utils import first_non_empty
|
||||
from ..storage.enums import (
|
||||
BODY_TYPE_ENUM_VALUES,
|
||||
COUNTRY_ENUM_VALUES,
|
||||
CURRENCY_ENUM_VALUES,
|
||||
DRIVE_ENUM_VALUES,
|
||||
GEARBOX_ENUM_VALUES,
|
||||
ORIGIN_ENUM_VALUES,
|
||||
SELLING_TYPE_ENUM_VALUES,
|
||||
STEERING_WHEEL_ENUM_VALUES,
|
||||
)
|
||||
from ..storage.schemas import CarRecord, ImageRecord
|
||||
|
||||
|
||||
class CarMapper:
|
||||
# Маппер MOBILEDE в CarRecord.
|
||||
|
||||
BODY_MAP = {
|
||||
"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
|
||||
"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
|
||||
"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
|
||||
"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
|
||||
"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
|
||||
"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
|
||||
}
|
||||
DRIVE_MAP = {
|
||||
"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
|
||||
"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
|
||||
"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
|
||||
}
|
||||
GEARBOX_MAP = {
|
||||
"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
|
||||
"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
|
||||
"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
|
||||
}
|
||||
STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
|
||||
COUNTRY_MAP = {
|
||||
"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
|
||||
"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
|
||||
}
|
||||
NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
|
||||
|
||||
def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
|
||||
# Собираем DB-модель.
|
||||
vehicle_summary = vehicle_summary or {}
|
||||
payload_insights = payload_insights or {}
|
||||
core = payload_insights.get("vehicle_core", {})
|
||||
pricing = payload_insights.get("pricing", {})
|
||||
damage = payload_insights.get("damage", {})
|
||||
auction = payload_insights.get("auction", {})
|
||||
images = payload_insights.get("images", {})
|
||||
|
||||
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
|
||||
parser_id = self._generate_parser_id(origin_id)
|
||||
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
|
||||
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
|
||||
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
|
||||
price = self._first_parsed_int(
|
||||
[
|
||||
pricing.get("buy_now"),
|
||||
pricing.get("current_bid"),
|
||||
vehicle_summary.get("buy_now"),
|
||||
vehicle_summary.get("current_bid"),
|
||||
pricing.get("actual_cash_value"),
|
||||
],
|
||||
self._to_money_int,
|
||||
)
|
||||
mileage = self._parse_odometer(
|
||||
first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
|
||||
)
|
||||
color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
|
||||
drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
|
||||
# Пробуем взять привод из двигателя.
|
||||
if not drive or drive == "NA":
|
||||
engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
|
||||
if engine_text:
|
||||
inferred_drive = self._normalize_drive(engine_text)
|
||||
if inferred_drive and inferred_drive != "NA":
|
||||
drive = inferred_drive
|
||||
gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
|
||||
steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
|
||||
body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
|
||||
engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
|
||||
title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
|
||||
seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
|
||||
location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
|
||||
country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
|
||||
is_damaged = self._bool_damage(damage, vehicle_summary)
|
||||
is_sold = self._bool_sold(auction)
|
||||
one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
|
||||
new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
|
||||
rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
|
||||
repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
|
||||
non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
|
||||
evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
|
||||
currency = self._normalize_currency(
|
||||
first_non_empty(
|
||||
[
|
||||
pricing.get("currency"),
|
||||
vehicle_summary.get("currency"),
|
||||
pricing.get("buy_now"),
|
||||
pricing.get("current_bid"),
|
||||
vehicle_summary.get("buy_now"),
|
||||
vehicle_summary.get("current_bid"),
|
||||
"USD",
|
||||
]
|
||||
)
|
||||
)
|
||||
slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
|
||||
images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
|
||||
origin = "MOBILEDE"
|
||||
|
||||
return CarRecord(
|
||||
parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
|
||||
mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
|
||||
steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
|
||||
selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
|
||||
is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
|
||||
evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
|
||||
slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _as_str(value: Any) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
@staticmethod
|
||||
def _to_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
digits = re.sub(r"[^\d]", "", str(value))
|
||||
return int(digits) if digits else None
|
||||
|
||||
@classmethod
|
||||
def _to_money_int(cls, value: Any) -> int | None:
|
||||
# Нормализация цены.
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
lowered = text.lower()
|
||||
if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
|
||||
return None
|
||||
|
||||
numbers = re.findall(r"\d[\d\s.,]*", text)
|
||||
if not numbers:
|
||||
return None
|
||||
|
||||
best: int | None = None
|
||||
for number in numbers:
|
||||
clean = number.replace(" ", "")
|
||||
if "," in clean and "." in clean:
|
||||
# Поддержка двух форматов.
|
||||
if clean.rfind(",") > clean.rfind("."):
|
||||
clean = clean.replace(".", "").replace(",", ".")
|
||||
else:
|
||||
clean = clean.replace(",", "")
|
||||
elif "," in clean:
|
||||
parts = clean.split(",")
|
||||
# Десятичный или тысячный разделитель.
|
||||
if len(parts[-1]) in {1, 2} and len(parts) == 2:
|
||||
clean = clean.replace(",", ".")
|
||||
else:
|
||||
clean = clean.replace(",", "")
|
||||
elif "." in clean:
|
||||
parts = clean.split(".")
|
||||
if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
|
||||
clean = clean.replace(".", "")
|
||||
|
||||
try:
|
||||
parsed = int(float(clean))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if parsed > 0 and (best is None or parsed > best):
|
||||
best = parsed
|
||||
|
||||
return best
|
||||
|
||||
@staticmethod
|
||||
def _first_parsed_int(values: list[Any], parser) -> int | None:
|
||||
for value in values:
|
||||
parsed = parser(value)
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _to_year(value: Any) -> int | None:
|
||||
parsed = CarMapper._to_int(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
if 1900 <= parsed <= 2100:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _parse_odometer(value: Any) -> int:
|
||||
# Разбор пробега.
|
||||
if value is None:
|
||||
return 0
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return 0
|
||||
lowered = text.lower()
|
||||
if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
|
||||
return 0
|
||||
# Ищем число.
|
||||
numbers = re.findall(r"[\d,]+", text)
|
||||
for num_str in numbers:
|
||||
clean = num_str.replace(",", "")
|
||||
if clean.isdigit() and int(clean) > 0:
|
||||
return int(clean)
|
||||
return 0
|
||||
|
||||
def _to_engine_cc(self, value: Any) -> int | None:
|
||||
# Поддержка литров и cc.
|
||||
text = str(value).lower().strip() if value is not None else ""
|
||||
if not text:
|
||||
return None
|
||||
if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
|
||||
return int(float(liters_match.group(1)) * 1000)
|
||||
if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
|
||||
return int(cubic_match.group(1))
|
||||
return int(text) if re.match(r"^\d+$", text) else None
|
||||
|
||||
def _normalize_currency(self, value: Any) -> str:
|
||||
text = self._as_str(value)
|
||||
upper = text.upper() or "USD"
|
||||
if any(token in text for token in ["€", "EUR"]):
|
||||
return "EUR"
|
||||
if any(token in text for token in ["¥", "JPY"]):
|
||||
return "JPY"
|
||||
if any(token in text for token in ["₩", "KRW"]):
|
||||
return "KRW"
|
||||
if any(token in text for token in ["£", "GBP"]):
|
||||
return "GBP"
|
||||
if any(token in text for token in ["₽", "RUB"]):
|
||||
return "RUB"
|
||||
if any(token in text for token in ["AED", "د.إ"]):
|
||||
return "AED"
|
||||
if any(token in text for token in ["CA$", "CAD"]):
|
||||
return "CAD"
|
||||
|
||||
text = upper
|
||||
if text in CURRENCY_ENUM_VALUES:
|
||||
return text
|
||||
return "USD" if "$" in str(value) else "USD"
|
||||
|
||||
def _normalize_drive(self, value: Any) -> str | None:
|
||||
return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
|
||||
|
||||
def _normalize_gearbox(self, value: Any) -> str | None:
|
||||
return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
|
||||
|
||||
def _normalize_steering(self, value: Any) -> str | None:
|
||||
return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
|
||||
|
||||
def _normalize_body_type(self, value: Any) -> str:
|
||||
return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
|
||||
|
||||
def _normalize_country(self, value: Any) -> str:
|
||||
fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
|
||||
return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
|
||||
|
||||
def _normalize_selling_type(self, value: Any) -> str:
|
||||
text = self._as_str(value) or "AUCTION"
|
||||
return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
|
||||
|
||||
def _map_value(
|
||||
self,
|
||||
value: Any,
|
||||
mapping: dict[str, str],
|
||||
allowed_values: tuple[str, ...],
|
||||
*,
|
||||
empty_default: str | None,
|
||||
fallback: str | None,
|
||||
) -> str | None:
|
||||
# Общая нормализация enum.
|
||||
text = self._as_str(value).lower()
|
||||
if not text:
|
||||
return empty_default
|
||||
if (mapped := mapping.get(text)) and mapped in allowed_values:
|
||||
return mapped
|
||||
for marker, mapped in mapping.items():
|
||||
if marker in text and mapped in allowed_values:
|
||||
return mapped
|
||||
return fallback
|
||||
|
||||
@staticmethod
|
||||
def _normalize_color(value: Any) -> str:
|
||||
text = str(value).strip() if value is not None else "other"
|
||||
if not text:
|
||||
return "other"
|
||||
if "/" in text:
|
||||
text = text.split("/")[0].strip()
|
||||
return text.lower() or "other"
|
||||
|
||||
@staticmethod
|
||||
def _boolish(value: Any) -> bool:
|
||||
return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
|
||||
|
||||
def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
|
||||
for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
|
||||
text = self._as_str(value).lower()
|
||||
if text and text not in self.NO_DAMAGE_MARKERS:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _bool_sold(self, auction: dict[str, Any]) -> bool:
|
||||
return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
|
||||
|
||||
def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
|
||||
# Для imageKeys берём самый большой размер.
|
||||
best_by_key: dict[str, str] = {}
|
||||
key_order: list[str] = []
|
||||
non_keyed: list[str] = []
|
||||
for item in urls:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
url = item.strip()
|
||||
if not url:
|
||||
continue
|
||||
if m := re.search(r'imageKeys=([^&]+)', url):
|
||||
img_key = m.group(1)
|
||||
w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
|
||||
existing = best_by_key.get(img_key)
|
||||
if existing is None:
|
||||
best_by_key[img_key] = url
|
||||
key_order.append(img_key)
|
||||
else:
|
||||
existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
|
||||
if w > existing_w:
|
||||
best_by_key[img_key] = url
|
||||
elif url not in non_keyed:
|
||||
non_keyed.append(url)
|
||||
deduped = [best_by_key[k] for k in key_order] + non_keyed
|
||||
return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
|
||||
|
||||
@staticmethod
|
||||
def _make_fullres_url(url: str) -> str:
|
||||
if "vis.MOBILEDE.com" in url:
|
||||
return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
|
||||
return url
|
||||
|
||||
@staticmethod
|
||||
def _make_preview_url(url: str) -> str:
|
||||
if "vis.MOBILEDE.com" in url:
|
||||
preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
|
||||
if preview != url:
|
||||
return preview
|
||||
return url
|
||||
|
||||
# Формирование parser_id.
|
||||
|
||||
_PARSER_ID_ALPHABET = ascii_letters + digits
|
||||
|
||||
@classmethod
|
||||
def _generate_parser_id(cls, origin_id: str) -> str:
|
||||
# Стабильный parser_id.
|
||||
digest = hashlib.sha256(origin_id.encode()).digest()
|
||||
alphabet = cls._PARSER_ID_ALPHABET
|
||||
base = len(alphabet)
|
||||
num = int.from_bytes(digest[:17], "big") # Хватает на 22 символа.
|
||||
chars: list[str] = []
|
||||
for _ in range(22):
|
||||
num, idx = divmod(num, base)
|
||||
chars.append(alphabet[idx])
|
||||
return "car-" + "".join(chars)
|
||||
|
||||
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
|
||||
# Формат: mobilede:{lot_number}.
|
||||
for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
|
||||
text = self._as_str(value)
|
||||
if text:
|
||||
return f"mobilede:{text}"
|
||||
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
|
||||
if "~" in tail:
|
||||
tail = tail.split("~")[0]
|
||||
raw = tail or self._slugify(vehicle_url)
|
||||
return f"mobilede:{raw}"
|
||||
|
||||
@staticmethod
|
||||
def _slugify(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
|
||||
|
||||
|
||||
@@ -1,408 +0,0 @@
|
||||
import html as html_module
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
|
||||
|
||||
logger = logging.getLogger("MOBILEDE_scraper.parsers")
|
||||
|
||||
|
||||
class VehicleParser:
|
||||
# Парсер страницы авто.
|
||||
|
||||
# Регулярки парсинга и защиты.
|
||||
_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
|
||||
_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
|
||||
_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
|
||||
_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
|
||||
_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
|
||||
|
||||
SUMMARY_KEY_MAP = {
|
||||
"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
|
||||
"year": {"year"},
|
||||
"make": {"make", "manufacturer", "brand"},
|
||||
"model": {"model"},
|
||||
"trim": {"trim", "series"},
|
||||
"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
|
||||
"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
|
||||
"secondary_damage": {"secondarydamage"},
|
||||
"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
|
||||
"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
|
||||
"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
|
||||
"actual_cash_value": {"actualcashvalue", "acv"},
|
||||
"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
|
||||
"keys": {"keys", "keystatus"},
|
||||
"title": {"titletype", "title", "documenttype"},
|
||||
"seller": {"seller", "sellername"},
|
||||
"location": {"location", "branchname", "auctionlocation", "branch"},
|
||||
"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
|
||||
"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
|
||||
"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
|
||||
"gearbox": {"transmission", "gearbox", "transmissiontype"},
|
||||
"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
|
||||
"fuel_type": {"fueltype", "fuel"},
|
||||
"cylinders": {"cylinders", "cylindercount"},
|
||||
"color": {"color", "primarycolor", "exteriorcolor"},
|
||||
}
|
||||
|
||||
DOM_LABEL_MAP: dict[str, str] = {
|
||||
"stock #": "lot_number",
|
||||
"stock": "lot_number",
|
||||
"primary damage": "primary_damage",
|
||||
"secondary damage": "secondary_damage",
|
||||
"odometer": "odometer",
|
||||
"odometer (miles)": "odometer",
|
||||
"mileage": "odometer",
|
||||
"body style": "body_type",
|
||||
"body type": "body_type",
|
||||
"vehicle type": "body_type",
|
||||
"engine": "engine",
|
||||
"engine type": "engine",
|
||||
"transmission": "gearbox",
|
||||
"drive line type": "drive",
|
||||
"driveline type": "drive",
|
||||
"drive line": "drive",
|
||||
"driveline": "drive",
|
||||
"drive type": "drive",
|
||||
"fuel type": "fuel_type",
|
||||
"fuel": "fuel_type",
|
||||
"cylinders": "cylinders",
|
||||
"exterior/interior": "color",
|
||||
"exterior color": "color",
|
||||
"color": "color",
|
||||
"model": "model",
|
||||
"series": "trim",
|
||||
"selling branch": "location",
|
||||
"vehicle location": "vehicle_location",
|
||||
"auction date and time": "auction_date",
|
||||
"sale date": "auction_date",
|
||||
"lane/run #": "lane",
|
||||
"actual cash value": "actual_cash_value",
|
||||
"estimated repair cost": "estimated_repair_cost",
|
||||
"seller": "seller",
|
||||
"title/sale doc": "title",
|
||||
"title/sale doc brand": "title_brand",
|
||||
"start code": "run_and_drive",
|
||||
"key": "keys",
|
||||
"keys": "keys",
|
||||
"manufactured in": "manufactured_in",
|
||||
"vehicle class": "vehicle_class",
|
||||
}
|
||||
|
||||
def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
if not dom_text:
|
||||
return result
|
||||
lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
|
||||
known_labels = set(self.DOM_LABEL_MAP.keys())
|
||||
skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
|
||||
max_fields = len(set(self.DOM_LABEL_MAP.values()))
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
# Ранний выход.
|
||||
if len(result) >= max_fields:
|
||||
break
|
||||
|
||||
# Метка и значение в одной строке.
|
||||
colon_pos = line.find(":")
|
||||
if colon_pos > 0:
|
||||
label_part = line[:colon_pos].strip().lower()
|
||||
value_part = line[colon_pos + 1:].strip()
|
||||
if label_part in known_labels and value_part and value_part.lower() not in skip_values:
|
||||
field_name = self.DOM_LABEL_MAP[label_part]
|
||||
if field_name not in result or not result[field_name]:
|
||||
result[field_name] = value_part
|
||||
continue
|
||||
|
||||
# Метка и значение в соседних строках.
|
||||
clean = line.rstrip(":").strip().lower()
|
||||
clean_alt = clean.rstrip("#").strip()
|
||||
matched_label = None
|
||||
if clean in known_labels:
|
||||
matched_label = clean
|
||||
elif clean_alt in known_labels:
|
||||
matched_label = clean_alt
|
||||
|
||||
if matched_label and i + 1 < len(lines):
|
||||
value = lines[i + 1].strip()
|
||||
if value.rstrip(":").lower().strip() in known_labels:
|
||||
continue
|
||||
if value.lower() in skip_values:
|
||||
continue
|
||||
field_name = self.DOM_LABEL_MAP[matched_label]
|
||||
if field_name not in result or not result[field_name]:
|
||||
result[field_name] = value
|
||||
return result
|
||||
|
||||
def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
|
||||
result: dict[str, str | None] = {"year": None, "make": None, "model": None}
|
||||
title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
|
||||
if title_match:
|
||||
result["year"] = title_match.group(1)
|
||||
result["make"] = title_match.group(2)
|
||||
result["model"] = title_match.group(3)
|
||||
return result
|
||||
dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
|
||||
if dom_match:
|
||||
result["year"] = dom_match.group(1)
|
||||
result["make"] = dom_match.group(2)
|
||||
result["model"] = dom_match.group(3).strip()
|
||||
return result
|
||||
|
||||
def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
|
||||
page_html = page_html or ""
|
||||
dom_text = dom_text or ""
|
||||
network_dump = network_dump or {}
|
||||
responses = network_dump.get("json_responses", [])
|
||||
payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
|
||||
dom_kv = self._parse_dom_key_value_pairs(dom_text)
|
||||
page_title = ""
|
||||
title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
|
||||
if title_match:
|
||||
page_title = title_match.group(1).strip()
|
||||
title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
|
||||
|
||||
# Один проход по payload.
|
||||
all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
|
||||
|
||||
summary: dict[str, Any] = {"source_url": vehicle_url}
|
||||
for field, values in all_found.items():
|
||||
if field in dom_kv:
|
||||
values.append(dom_kv[field])
|
||||
summary[field] = first_non_empty(values)
|
||||
|
||||
summary["year"] = summary.get("year") or title_parsed.get("year")
|
||||
summary["make"] = summary.get("make") or title_parsed.get("make")
|
||||
summary["model"] = summary.get("model") or title_parsed.get("model")
|
||||
summary["trim"] = summary.get("trim") or dom_kv.get("trim")
|
||||
summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
|
||||
summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
|
||||
for dom_field, dom_value in dom_kv.items():
|
||||
if dom_field not in summary or not summary[dom_field]:
|
||||
summary[dom_field] = dom_value
|
||||
prices = self._extract_prices_from_text(dom_text)
|
||||
if not summary.get("actual_cash_value") and prices:
|
||||
summary["actual_cash_value"] = prices[0]
|
||||
if not summary.get("buy_now"):
|
||||
buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
|
||||
if buy_now_match:
|
||||
summary["buy_now"] = buy_now_match.group(1)
|
||||
elif prices:
|
||||
summary["buy_now"] = prices[0]
|
||||
if not summary.get("current_bid") and len(prices) > 1:
|
||||
summary["current_bid"] = prices[1]
|
||||
|
||||
embedded = self._extract_embedded_json(page_html)
|
||||
for item in embedded:
|
||||
p = item.get("payload")
|
||||
if isinstance(p, (dict, list)):
|
||||
payloads.append(p)
|
||||
# Доп. проход по JSON.
|
||||
extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
|
||||
for field, vals in extra.items():
|
||||
if not summary.get(field):
|
||||
v = first_non_empty(vals)
|
||||
if v:
|
||||
summary[field] = v
|
||||
|
||||
# Передаём готовые image_urls.
|
||||
image_urls = summary.get("image_urls") or []
|
||||
return {
|
||||
"vehicle_summary": summary,
|
||||
"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
|
||||
"embedded_json": embedded,
|
||||
"dom_hints": self._dom_hints(dom_text),
|
||||
"access_notes": self._build_access_notes(summary, responses),
|
||||
}
|
||||
|
||||
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
|
||||
if image_urls is None:
|
||||
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
|
||||
return {
|
||||
"vehicle_core": {
|
||||
"lot_number": summary.get("lot_number"), "year": summary.get("year"),
|
||||
"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
|
||||
"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
|
||||
"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
|
||||
"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
|
||||
"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
|
||||
"color": summary.get("color"), "keys": summary.get("keys"),
|
||||
},
|
||||
"pricing": {
|
||||
"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
|
||||
"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
|
||||
"currency": self._guess_currency(summary),
|
||||
},
|
||||
"bids": self._build_bid_insights(summary, payloads),
|
||||
"damage": {
|
||||
"primary": summary.get("primary_damage"),
|
||||
"secondary": summary.get("secondary_damage"),
|
||||
"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
|
||||
},
|
||||
"auction": {
|
||||
"auction_date": summary.get("auction_date"),
|
||||
"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
|
||||
"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
|
||||
"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
|
||||
"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
|
||||
},
|
||||
"images": {"count": len(image_urls), "urls": image_urls},
|
||||
"source_endpoints": self._build_source_endpoints(responses),
|
||||
}
|
||||
|
||||
def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"amount": summary.get("current_bid"),
|
||||
"currency": self._guess_currency(summary),
|
||||
"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
|
||||
"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
|
||||
}
|
||||
|
||||
def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
|
||||
mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
|
||||
for item in responses:
|
||||
url = item.get("url", "")
|
||||
category = item.get("category", "other")
|
||||
if category == "vehicle":
|
||||
mapping["vehicle"].append(url)
|
||||
lowered = url.lower()
|
||||
if any(token in lowered for token in ["bid", "offer"]):
|
||||
mapping["bids"].append(url)
|
||||
if any(token in lowered for token in ["damage", "report"]):
|
||||
mapping["damage"].append(url)
|
||||
if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
|
||||
mapping["auction"].append(url)
|
||||
elif category in mapping:
|
||||
mapping[category].append(url)
|
||||
return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
|
||||
|
||||
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
endpoints = [item.get("url", "") for item in responses]
|
||||
dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
|
||||
return {
|
||||
"images_visible": bool(summary.get("image_urls")),
|
||||
"network_json_count": len(responses),
|
||||
"possible_captcha": bool(dom_hints.get("has_captcha_text")),
|
||||
"possible_antibot": bool(dom_hints.get("has_antibot_text")),
|
||||
"observed_endpoints": endpoints[:20],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
|
||||
lowered = {key.lower() for key in keys}
|
||||
values: list[Any] = []
|
||||
for payload in payloads:
|
||||
values.extend(deep_find_key(payload, lowered))
|
||||
return values
|
||||
|
||||
@staticmethod
|
||||
def _guess_currency(summary: dict[str, Any]) -> str:
|
||||
for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
|
||||
value = str(summary.get(key) or "")
|
||||
if "$" in value:
|
||||
return "USD"
|
||||
if "€" in value:
|
||||
return "EUR"
|
||||
if "¥" in value:
|
||||
return "JPY"
|
||||
return "USD"
|
||||
|
||||
@staticmethod
|
||||
def _extract_lot_number(text: str) -> str | None:
|
||||
match = LOT_RE.search(text or "")
|
||||
return match.group(1) if match else None
|
||||
|
||||
@staticmethod
|
||||
def _extract_prices_from_text(text: str) -> list[str]:
|
||||
return [match.group(1) for match in PRICE_RE.finditer(text or "")]
|
||||
|
||||
@staticmethod
|
||||
def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
|
||||
scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
|
||||
extracted: list[dict[str, Any]] = []
|
||||
for script_text in scripts:
|
||||
if "{" not in script_text and "[" not in script_text:
|
||||
continue
|
||||
# Пропускаем большие блоки.
|
||||
if len(script_text) > 51_200:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(script_text.strip())
|
||||
except Exception:
|
||||
continue
|
||||
extracted.append({"type": "inline_json", "payload": parsed})
|
||||
return extracted
|
||||
|
||||
@staticmethod
|
||||
def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
|
||||
vehicle_key = ""
|
||||
key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
|
||||
if key_match:
|
||||
vehicle_key = key_match.group(1)
|
||||
found: list[str] = []
|
||||
for payload in payloads:
|
||||
found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
|
||||
flat: list[str] = []
|
||||
seen_flat: set[str] = set()
|
||||
for item in found:
|
||||
if isinstance(item, str) and item.startswith("http"):
|
||||
cleaned = html_module.unescape(item)
|
||||
lowered = cleaned.lower()
|
||||
if vehicle_key and "vis.MOBILEDE.com" in lowered and vehicle_key not in cleaned:
|
||||
continue
|
||||
if cleaned not in seen_flat:
|
||||
seen_flat.add(cleaned)
|
||||
flat.append(cleaned)
|
||||
elif isinstance(item, list):
|
||||
for child in item:
|
||||
if isinstance(child, str) and child.startswith("http"):
|
||||
cleaned = html_module.unescape(child)
|
||||
lowered = cleaned.lower()
|
||||
if vehicle_key and "vis.MOBILEDE.com" in lowered and vehicle_key not in cleaned:
|
||||
continue
|
||||
if cleaned not in seen_flat:
|
||||
seen_flat.add(cleaned)
|
||||
flat.append(cleaned)
|
||||
for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
|
||||
for match in re.finditer(pattern, html or "", re.IGNORECASE):
|
||||
url = html_module.unescape(match.group(1).strip())
|
||||
if not url.startswith("http") or url in seen_flat:
|
||||
continue
|
||||
lowered = url.lower()
|
||||
if vehicle_key and vehicle_key in url:
|
||||
seen_flat.add(url)
|
||||
flat.append(url)
|
||||
elif any(token in lowered for token in ["vis.MOBILEDE.com", "anvis", "vehicleimage"]):
|
||||
if vehicle_key and vehicle_key not in url:
|
||||
continue
|
||||
seen_flat.add(url)
|
||||
flat.append(url)
|
||||
if vehicle_key:
|
||||
for url in re.findall(r'https?://vis\.MOBILEDE\.com[^\s"\'<>]+', html or ""):
|
||||
cleaned = html_module.unescape(url)
|
||||
if cleaned not in seen_flat and vehicle_key in cleaned:
|
||||
seen_flat.add(cleaned)
|
||||
flat.append(cleaned)
|
||||
filtered: list[str] = []
|
||||
for url in flat:
|
||||
lowered = url.lower()
|
||||
if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
|
||||
continue
|
||||
if "vis.MOBILEDE.com" in lowered and "/resizer" not in lowered:
|
||||
continue
|
||||
filtered.append(url)
|
||||
return filtered
|
||||
|
||||
@staticmethod
|
||||
def _dom_hints(text: str) -> dict[str, Any]:
|
||||
lowered = (text or "").lower()
|
||||
return {
|
||||
"has_buy_now_text": "buy now" in lowered,
|
||||
"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
|
||||
"has_damage_text": "damage" in lowered,
|
||||
"has_title_text": "title" in lowered,
|
||||
"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
|
||||
"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -21,6 +21,17 @@ CAR_DB_FIELDS = {
|
||||
|
||||
_IN_CHUNK_SIZE = 5000
|
||||
CAR_TABLE_NAME = Car.__tablename__
|
||||
MOBILEDE_ORIGIN_PREFIXES = ("mobile.de:", "mobilede:")
|
||||
|
||||
|
||||
def _origin_prefix_filter(column, prefixes: tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES):
|
||||
"""Match all supported mobile.de origin_id prefixes.
|
||||
|
||||
Older records were stored as ``mobile.de:<id>`` while some newer helper code
|
||||
used ``mobilede:<id>``. Cleanup and refresh queries must include both to avoid
|
||||
leaving stale active cars in the DB.
|
||||
"""
|
||||
return or_(*[column.like(f"{prefix}%") for prefix in prefixes])
|
||||
|
||||
|
||||
class PersistenceService:
|
||||
@@ -502,7 +513,7 @@ class PersistenceService:
|
||||
update(Car)
|
||||
.where(Car.origin_id.notin_(active_origin_ids))
|
||||
.where(Car.is_sold == False) # noqa: E712
|
||||
.where(Car.origin_id.like("mobilede:%"))
|
||||
.where(_origin_prefix_filter(Car.origin_id))
|
||||
.values(is_sold=True)
|
||||
)
|
||||
result = session.execute(stmt)
|
||||
@@ -533,7 +544,12 @@ class PersistenceService:
|
||||
if is_postgres:
|
||||
# Считаем кандидатов на sold.
|
||||
total_active = session.execute(
|
||||
text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%%'")
|
||||
text(f"""
|
||||
SELECT count(*)
|
||||
FROM {CAR_TABLE_NAME}
|
||||
WHERE is_sold = FALSE
|
||||
AND (origin_id LIKE 'mobile.de:%%' OR origin_id LIKE 'mobilede:%%')
|
||||
""")
|
||||
).scalar() or 0
|
||||
|
||||
if total_active == 0:
|
||||
@@ -561,7 +577,7 @@ class PersistenceService:
|
||||
LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
|
||||
WHERE a.url IS NULL
|
||||
AND c.is_sold = FALSE
|
||||
AND c.origin_id LIKE 'mobilede:%%'
|
||||
AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%')
|
||||
""".format(car_table=CAR_TABLE_NAME))).scalar() or 0
|
||||
|
||||
# Защита от аномалии.
|
||||
@@ -582,7 +598,7 @@ class PersistenceService:
|
||||
LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
|
||||
WHERE a.url IS NULL
|
||||
AND c.is_sold = FALSE
|
||||
AND c.origin_id LIKE 'mobilede:%%'
|
||||
AND (c.origin_id LIKE 'mobile.de:%%' OR c.origin_id LIKE 'mobilede:%%')
|
||||
) sub
|
||||
WHERE {car_table}.id = sub.id
|
||||
""".format(car_table=CAR_TABLE_NAME)))
|
||||
@@ -592,13 +608,13 @@ class PersistenceService:
|
||||
stmt = (
|
||||
update(Car)
|
||||
.where(Car.is_sold == False) # noqa: E712
|
||||
.where(Car.origin_id.like("mobilede:%"))
|
||||
.where(_origin_prefix_filter(Car.origin_id))
|
||||
.values(is_sold=True)
|
||||
)
|
||||
# Загружаем active URL.
|
||||
all_active = session.execute(
|
||||
select(Car.id, Car.origin_url).where(
|
||||
Car.is_sold == False, Car.origin_id.like("mobilede:%") # noqa: E712
|
||||
Car.is_sold == False, _origin_prefix_filter(Car.origin_id) # noqa: E712
|
||||
)
|
||||
).all()
|
||||
mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
|
||||
@@ -622,35 +638,99 @@ class PersistenceService:
|
||||
logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
|
||||
return count
|
||||
|
||||
def get_all_origin_ids_for_lane(self, prefix: str = "mobilede:") -> set[str]:
|
||||
def mark_sold_not_seen_since(
|
||||
self,
|
||||
since_ts: datetime,
|
||||
*,
|
||||
prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
|
||||
safety_ratio: float = 0.8,
|
||||
) -> int:
|
||||
"""Mark active cars as sold when they were not seen during a full refresh cycle.
|
||||
|
||||
Any unsold mobile.de car with ``last_seen_at < since_ts`` is considered absent
|
||||
from the latest completed refresh cycle and can be marked as sold.
|
||||
A safety guard prevents anomalous bulk updates.
|
||||
"""
|
||||
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
|
||||
with self.session_scope() as session:
|
||||
total_active = int(
|
||||
session.execute(
|
||||
select(text("count(*)")).select_from(Car).where(
|
||||
_origin_prefix_filter(Car.origin_id, prefixes),
|
||||
Car.is_sold == False, # noqa: E712
|
||||
)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
if total_active <= 0:
|
||||
return 0
|
||||
|
||||
would_mark = int(
|
||||
session.execute(
|
||||
select(text("count(*)")).select_from(Car).where(
|
||||
_origin_prefix_filter(Car.origin_id, prefixes),
|
||||
Car.is_sold == False, # noqa: E712
|
||||
Car.last_seen_at < since_ts,
|
||||
)
|
||||
).scalar()
|
||||
or 0
|
||||
)
|
||||
if would_mark <= 0:
|
||||
return 0
|
||||
|
||||
if total_active > 100 and would_mark > int(total_active * max(0.0, min(1.0, safety_ratio))):
|
||||
logger.error(
|
||||
"mark_sold(last_seen) safety abort: would mark %d/%d (%.0f%%) as sold",
|
||||
would_mark,
|
||||
total_active,
|
||||
(would_mark / max(1, total_active)) * 100,
|
||||
)
|
||||
return 0
|
||||
|
||||
result = session.execute(
|
||||
update(Car)
|
||||
.where(_origin_prefix_filter(Car.origin_id, prefixes))
|
||||
.where(Car.is_sold == False) # noqa: E712
|
||||
.where(Car.last_seen_at < since_ts)
|
||||
.values(is_sold=True)
|
||||
)
|
||||
count = int(result.rowcount or 0)
|
||||
if count:
|
||||
logger.info("Marked %d cars as sold by last_seen cutoff=%s", count, since_ts.isoformat())
|
||||
return count
|
||||
|
||||
def get_all_origin_ids_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]:
|
||||
"""Возвращает все известные origin_id для заданного префикса.
|
||||
|
||||
Использует yield_per для потоковой загрузки при большом количестве записей.
|
||||
"""
|
||||
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
|
||||
with self.session_scope() as session:
|
||||
result = session.execute(
|
||||
select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
|
||||
select(Car.origin_id).where(_origin_prefix_filter(Car.origin_id, prefixes)).execution_options(yield_per=10000)
|
||||
)
|
||||
return {str(row[0]) for row in result if row and row[0]}
|
||||
|
||||
def get_all_active_origin_urls_for_lane(self, prefix: str = "mobilede:") -> set[str]:
|
||||
def get_all_active_origin_urls_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> set[str]:
|
||||
"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
|
||||
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
|
||||
with self.session_scope() as session:
|
||||
result = session.execute(
|
||||
select(Car.origin_url).where(
|
||||
Car.origin_id.like(f"{prefix}%"),
|
||||
_origin_prefix_filter(Car.origin_id, prefixes),
|
||||
Car.is_sold == False, # noqa: E712
|
||||
).execution_options(yield_per=10000)
|
||||
)
|
||||
return {str(row[0]) for row in result if row and row[0]}
|
||||
|
||||
def count_active_cars_for_lane(self, prefix: str = "mobilede:") -> int:
|
||||
def count_active_cars_for_lane(self, prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES) -> int:
|
||||
"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
|
||||
from sqlalchemy import func as sa_func
|
||||
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
|
||||
with self.session_scope() as session:
|
||||
result = session.execute(
|
||||
select(sa_func.count()).select_from(Car).where(
|
||||
Car.origin_id.like(f"{prefix}%"),
|
||||
_origin_prefix_filter(Car.origin_id, prefixes),
|
||||
Car.is_sold == False, # noqa: E712
|
||||
)
|
||||
)
|
||||
@@ -658,7 +738,7 @@ class PersistenceService:
|
||||
|
||||
def get_active_origin_urls_batch_for_refresh(
|
||||
self,
|
||||
prefix: str = "mobilede:",
|
||||
prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
|
||||
offset: int = 0,
|
||||
limit: int = 500,
|
||||
) -> list[str]:
|
||||
@@ -666,10 +746,11 @@ class PersistenceService:
|
||||
|
||||
Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
|
||||
"""
|
||||
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
|
||||
with self.session_scope() as session:
|
||||
result = session.execute(
|
||||
select(Car.origin_url).where(
|
||||
Car.origin_id.like(f"{prefix}%"),
|
||||
_origin_prefix_filter(Car.origin_id, prefixes),
|
||||
Car.is_sold == False, # noqa: E712
|
||||
).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
|
||||
)
|
||||
|
||||
@@ -16,7 +16,7 @@ BODY_TYPE_ENUM_VALUES = (
|
||||
"OTHER",
|
||||
"NA",
|
||||
)
|
||||
COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
|
||||
COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "NA")
|
||||
ORIGIN_ENUM_VALUES = (
|
||||
"MOBILEDE",
|
||||
"MOBILE_DE",
|
||||
|
||||
@@ -127,7 +127,6 @@ celery_app.conf.update(
|
||||
"mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
|
||||
"mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
|
||||
"mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE},
|
||||
"MOBILEDE.sync_cars_feed": {"queue": MOBILEDE_SYNC_QUEUE},
|
||||
"mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE},
|
||||
},
|
||||
)
|
||||
@@ -137,7 +136,7 @@ celery_app.autodiscover_tasks(["mobilede_scraper.worker"])
|
||||
|
||||
@worker_ready.connect
|
||||
def _on_worker_ready(**kwargs):
|
||||
"""При старте worker отправляем первый sync_listing, если очередь пуста."""
|
||||
"""При старте worker отправляем первый canonical runtime sync, если очередь пуста."""
|
||||
if not _env_bool("MOBILEDE_STARTUP_SYNC_ENABLED", True):
|
||||
logger.info("Worker ready: startup sync dispatch disabled by MOBILEDE_STARTUP_SYNC_ENABLED")
|
||||
return
|
||||
@@ -154,16 +153,6 @@ def _on_worker_ready(**kwargs):
|
||||
)
|
||||
|
||||
has_fresh_progress = _has_fresh_active_progress(redis_client)
|
||||
for stale_key in ("mobilede:locks:sync_listing",):
|
||||
try:
|
||||
ttl = redis_client.ttl(stale_key)
|
||||
if ttl is not None and ttl != -2 and not has_fresh_progress:
|
||||
redis_client.delete(stale_key)
|
||||
logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
|
||||
elif ttl is not None and ttl != -2:
|
||||
logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl)
|
||||
except Exception:
|
||||
logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True)
|
||||
|
||||
try:
|
||||
queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
|
||||
|
||||
137
mobilede_scraper/worker/constants.py
Normal file
137
mobilede_scraper/worker/constants.py
Normal file
@@ -0,0 +1,137 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
|
||||
MOBILEDE_SYNC_QUEUE = "mobilede_sync"
|
||||
MOBILEDE_SEARCH_CURSOR_KEY = "mobilede:state:search_next_page"
|
||||
MOBILEDE_SEGMENT_CURSOR_KEY_FMT = "mobilede:state:search_next_page:{segment_key}"
|
||||
MOBILEDE_RUNTIME_SEGMENT_INDEX_KEY = "mobilede:state:runtime_segment_index"
|
||||
MOBILEDE_RUNTIME_SEGMENTS_TASK = "mobilede.sync_runtime_segments"
|
||||
MOBILEDE_SYNC_TASK_NAME = "mobilede.sync_search"
|
||||
MOBILEDE_SEGMENT_LOCK_KEY_FMT = "mobilede:locks:segment:{segment_key}"
|
||||
MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT = "mobilede:state:followup_pending:{segment_key}"
|
||||
MOBILEDE_PROGRESS_PAGE_COUNTER_KEY_FMT = "mobilede:state:progress_pages:{segment_key}"
|
||||
|
||||
MOBILEDE_CONTINUOUS_SYNC_ENABLED = os.getenv("MOBILEDE_CONTINUOUS_SYNC_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS", "15"))))
|
||||
MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS = max(60, int(float(os.getenv("MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS", "3600"))))
|
||||
MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS", "5"))))
|
||||
MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "10")))
|
||||
MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_RUNTIME_INITIAL_TASKS = max(1, int(os.getenv("MOBILEDE_RUNTIME_INITIAL_TASKS", "2")))
|
||||
MOBILEDE_SEGMENT_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_SEGMENT_PAGE_WINDOW", "10")))
|
||||
MOBILEDE_RESULTS_PER_PAGE = max(1, int(os.getenv("MOBILEDE_RESULTS_PER_PAGE", "20")))
|
||||
MOBILEDE_MAX_PAGE_NUMBER = max(1, int(os.getenv("MOBILEDE_MAX_PAGE_NUMBER", "50")))
|
||||
MOBILEDE_SEGMENT_TARGET_RESULTS = max(
|
||||
MOBILEDE_RESULTS_PER_PAGE,
|
||||
int(os.getenv("MOBILEDE_SEGMENT_TARGET_RESULTS", str(MOBILEDE_RESULTS_PER_PAGE * MOBILEDE_MAX_PAGE_NUMBER))),
|
||||
)
|
||||
MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "false").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_PREPLAN_SEGMENT_PROBES = os.getenv("MOBILEDE_PREPLAN_SEGMENT_PROBES", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_PREPLAN_MAX_SEGMENTS = max(1, int(os.getenv("MOBILEDE_PREPLAN_MAX_SEGMENTS", "1000")))
|
||||
MOBILEDE_PREPLAN_MAX_PROBES = max(0, int(os.getenv("MOBILEDE_PREPLAN_MAX_PROBES", "40")))
|
||||
MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = min(
|
||||
5.0,
|
||||
max(1.0, float(os.getenv("MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO", "2.5"))),
|
||||
)
|
||||
MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = max(0, min(3, int(os.getenv("MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH", "1"))))
|
||||
MOBILEDE_COMPACT_SEGMENTS = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000")))
|
||||
MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018")))
|
||||
MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_MILEAGE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_HOT_MILEAGE_PRICE_MIN = max(1, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MIN", "15001")))
|
||||
MOBILEDE_HOT_MILEAGE_PRICE_MAX = max(MOBILEDE_HOT_MILEAGE_PRICE_MIN, int(os.getenv("MOBILEDE_HOT_MILEAGE_PRICE_MAX", "30000")))
|
||||
MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX = max(1, int(os.getenv("MOBILEDE_HOT_OLD_CHEAP_PRICE_MAX", "5000")))
|
||||
MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED = os.getenv("MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP = os.getenv("MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_INCREMENTAL_PAGE_WINDOW = max(1, int(os.getenv("MOBILEDE_INCREMENTAL_PAGE_WINDOW", "1")))
|
||||
MOBILEDE_ONLY_NEW_NEWEST_FIRST = os.getenv("MOBILEDE_ONLY_NEW_NEWEST_FIRST", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS = os.getenv("MOBILEDE_INCREMENTAL_STRICT_FIRST_PASS", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK = max(1, int(os.getenv("MOBILEDE_ONLY_NEW_ZERO_INSERT_STREAK", "1")))
|
||||
MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS = max(60, int(os.getenv("MOBILEDE_ONLY_NEW_COOLDOWN_SECONDS", "3600")))
|
||||
MOBILEDE_ONLY_NEW_HOT_ONLY = os.getenv("MOBILEDE_ONLY_NEW_HOT_ONLY", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_ONLY_NEW_HOT_TTL_SECONDS", "10800")))
|
||||
MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO = min(1.0, max(0.0, float(os.getenv("MOBILEDE_ONLY_NEW_MIN_INSERT_RATIO", "0.95"))))
|
||||
|
||||
MOBILEDE_BOOTSTRAP_DONE_KEY = "mobilede:state:bootstrap_full_scan_done"
|
||||
MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY = "mobilede:state:bootstrap_segments_total"
|
||||
MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY = "mobilede:state:bootstrap_segments_done"
|
||||
MOBILEDE_BOOTSTRAP_LISTINGS_TOTAL_KEY = "mobilede:state:bootstrap_listings_total"
|
||||
MOBILEDE_BOOTSTRAP_UNIQUE_TOTAL_KEY = "mobilede:state:bootstrap_unique_total"
|
||||
MOBILEDE_BOOTSTRAP_INSERTED_TOTAL_KEY = "mobilede:state:bootstrap_inserted_total"
|
||||
MOBILEDE_BOOTSTRAP_UPDATED_TOTAL_KEY = "mobilede:state:bootstrap_updated_total"
|
||||
MOBILEDE_BOOTSTRAP_IMAGES_TOTAL_KEY = "mobilede:state:bootstrap_images_total"
|
||||
MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY = "mobilede:state:bootstrap_dispatched_segments"
|
||||
MOBILEDE_BOOTSTRAP_INCREMENTAL_TRANSITION_KEY = "mobilede:state:bootstrap_incremental_transition"
|
||||
MOBILEDE_RUNTIME_SEGMENTS_CACHE_KEY = "mobilede:state:runtime_segments_cache"
|
||||
MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY = "mobilede:state:runtime_segments_plan_finalized"
|
||||
MOBILEDE_RUNTIME_SEGMENTS_BUILDING_KEY = "mobilede:state:runtime_segments_building"
|
||||
MOBILEDE_RUNTIME_SEGMENTS_PENDING_KEY = "mobilede:state:runtime_segments_pending"
|
||||
MOBILEDE_RUNTIME_SEGMENTS_CACHE_LOCK_KEY = "mobilede:locks:runtime_segments_cache"
|
||||
MOBILEDE_OVERFLOW_EXPANDED_PARENTS_KEY = "mobilede:state:overflow_expanded_parents"
|
||||
MOBILEDE_REFRESH_CYCLE_ID_KEY = "mobilede:state:refresh_cycle:id"
|
||||
MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY = "mobilede:state:refresh_cycle:started_at"
|
||||
MOBILEDE_REFRESH_CYCLE_TOTAL_KEY = "mobilede:state:refresh_cycle:total"
|
||||
MOBILEDE_REFRESH_CYCLE_DONE_KEY = "mobilede:state:refresh_cycle:done"
|
||||
MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT = "mobilede:state:refresh_cycle:done_segments:{cycle_id}"
|
||||
MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT = "mobilede:state:refresh_cycle:finalized:{cycle_id}"
|
||||
MOBILEDE_REFRESH_CYCLE_TTL_SECONDS = max(60 * 60, int(os.getenv("MOBILEDE_REFRESH_CYCLE_TTL_SECONDS", str(24 * 60 * 60))))
|
||||
|
||||
MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min(
|
||||
1.0,
|
||||
max(0.5, float(os.getenv("MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO", "0.98"))),
|
||||
)
|
||||
MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = max(
|
||||
1,
|
||||
min(5, int(os.getenv("MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS", "3"))),
|
||||
)
|
||||
MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN = max(
|
||||
250,
|
||||
int(os.getenv("MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN", "1000")),
|
||||
)
|
||||
MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max(
|
||||
1,
|
||||
min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))),
|
||||
)
|
||||
|
||||
MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle"
|
||||
MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count"
|
||||
MOBILEDE_INCREMENTAL_CYCLE_SEEN_SET_KEY_FMT = "mobilede:state:incremental_cycle_seen:{cycle_id}"
|
||||
|
||||
TASK_PROGRESS_KEY_FMT = "mobilede:state:task_progress:{task_id}"
|
||||
GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts"
|
||||
GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts"
|
||||
DB_PROGRESS_STAGES = {
|
||||
"db_upsert_done",
|
||||
}
|
||||
|
||||
STALL_WATCHDOG_NAVIGATION_STAGES = {
|
||||
"page_collected",
|
||||
}
|
||||
STALL_WATCHDOG_LONG_RUNNING_STAGES = {
|
||||
"search_collection_done",
|
||||
"records_mapped",
|
||||
}
|
||||
STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max(
|
||||
300,
|
||||
int(os.getenv("STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS", "900")),
|
||||
)
|
||||
STALL_WATCHDOG_DETAIL_GRACE_SECONDS = max(
|
||||
600,
|
||||
int(os.getenv("STALL_WATCHDOG_DETAIL_GRACE_SECONDS", "1200")),
|
||||
)
|
||||
DB_IDLE_RESTART_SECONDS = max(60, int(os.getenv("MOBILEDE_DB_IDLE_RESTART_SECONDS", "3600")))
|
||||
TERMINAL_PROGRESS_STAGES = {
|
||||
"segment_done",
|
||||
"segment_failed",
|
||||
"segment_task_completed",
|
||||
"segment_task_failed",
|
||||
"segment_task_soft_timeout",
|
||||
"sync_done",
|
||||
"failed",
|
||||
}
|
||||
105
mobilede_scraper/worker/progress.py
Normal file
105
mobilede_scraper/worker/progress.py
Normal file
@@ -0,0 +1,105 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
|
||||
from redis import Redis
|
||||
|
||||
from .constants import (
|
||||
DB_PROGRESS_STAGES,
|
||||
GLOBAL_DB_PROGRESS_TS_KEY,
|
||||
GLOBAL_PROGRESS_TS_KEY,
|
||||
MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT,
|
||||
MOBILEDE_SEGMENT_LOCK_KEY_FMT,
|
||||
STALL_WATCHDOG_DETAIL_GRACE_SECONDS,
|
||||
STALL_WATCHDOG_LONG_RUNNING_STAGES,
|
||||
STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS,
|
||||
STALL_WATCHDOG_NAVIGATION_STAGES,
|
||||
TASK_PROGRESS_KEY_FMT,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("mobilede_scraper.worker.progress")
|
||||
|
||||
|
||||
def _safe_int(value) -> int | None:
|
||||
try:
|
||||
if value is None:
|
||||
return None
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _task_progress_key(task_id: str) -> str:
|
||||
return TASK_PROGRESS_KEY_FMT.format(task_id=task_id)
|
||||
|
||||
|
||||
def _mobilede_segment_lock_key(segment_key: str) -> str:
|
||||
return MOBILEDE_SEGMENT_LOCK_KEY_FMT.format(segment_key=segment_key)
|
||||
|
||||
|
||||
def _mobilede_followup_pending_key(segment_key: str) -> str:
|
||||
return MOBILEDE_SEGMENT_FOLLOWUP_PENDING_KEY_FMT.format(segment_key=segment_key)
|
||||
|
||||
|
||||
def _update_task_progress(
|
||||
redis_client: Redis,
|
||||
*,
|
||||
task_id: str,
|
||||
stage: str,
|
||||
ttl_seconds: int,
|
||||
**payload,
|
||||
) -> None:
|
||||
try:
|
||||
now_ts = int(time.time())
|
||||
existing_task_started_ts: int | None = None
|
||||
try:
|
||||
existing_raw = redis_client.get(_task_progress_key(task_id))
|
||||
if existing_raw:
|
||||
existing_payload = json.loads(existing_raw)
|
||||
existing_task_started_ts = _safe_int(existing_payload.get("task_started_ts"))
|
||||
except Exception:
|
||||
existing_task_started_ts = None
|
||||
payload.setdefault("task_started_ts", existing_task_started_ts or now_ts)
|
||||
if stage not in DB_PROGRESS_STAGES and "last_db_progress_ts" not in payload:
|
||||
last_db_progress_ts = _safe_int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY))
|
||||
if last_db_progress_ts is not None:
|
||||
payload["last_db_progress_ts"] = last_db_progress_ts
|
||||
data = {
|
||||
"task_id": task_id,
|
||||
"stage": stage,
|
||||
"ts": now_ts,
|
||||
**payload,
|
||||
}
|
||||
ttl = max(60, int(ttl_seconds))
|
||||
pipe = redis_client.pipeline()
|
||||
pipe.set(
|
||||
_task_progress_key(task_id),
|
||||
json.dumps(data, ensure_ascii=False),
|
||||
ex=ttl,
|
||||
)
|
||||
# Глобальный маркер активности для внешнего guard-процесса.
|
||||
# Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера
|
||||
# (когда PID жив, но прогресс по задачам не двигается).
|
||||
pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))
|
||||
if stage in DB_PROGRESS_STAGES:
|
||||
pipe.set(GLOBAL_DB_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))
|
||||
pipe.execute()
|
||||
except Exception:
|
||||
logger.warning("Failed to update task progress for %s", task_id, exc_info=True)
|
||||
|
||||
|
||||
def _clear_task_progress(redis_client: Redis, task_id: str) -> None:
|
||||
try:
|
||||
redis_client.delete(_task_progress_key(task_id))
|
||||
except Exception:
|
||||
logger.warning("Failed to clear task progress for %s", task_id, exc_info=True)
|
||||
|
||||
|
||||
def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int:
|
||||
if stage in STALL_WATCHDOG_NAVIGATION_STAGES:
|
||||
return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS)
|
||||
if stage in STALL_WATCHDOG_LONG_RUNNING_STAGES:
|
||||
return max(int(default_timeout), STALL_WATCHDOG_DETAIL_GRACE_SECONDS)
|
||||
return int(default_timeout)
|
||||
@@ -7,17 +7,14 @@ import time
|
||||
|
||||
from redis import Redis
|
||||
|
||||
from .constants import DB_PROGRESS_STAGES
|
||||
|
||||
logger = logging.getLogger("MOBILEDE_scraper.worker.self_heal")
|
||||
|
||||
MOBILEDE_SYNC_QUEUE = "MOBILEDE_sync"
|
||||
MOBILEDE_SYNC_QUEUE = "mobilede_sync"
|
||||
GLOBAL_PROGRESS_TS_KEY = "mobilede:state:last_progress_ts"
|
||||
GLOBAL_DB_PROGRESS_TS_KEY = "mobilede:state:last_db_progress_ts"
|
||||
SELF_HEAL_RESTART_LOCK_KEY = "mobilede:state:self_heal_restart_in_progress"
|
||||
SYNC_LISTING_LOCK_KEY = "mobilede:locks:sync_listing"
|
||||
SYNC_FULL_SCAN_DONE_KEY = "mobilede:state:sync_full_scan_done"
|
||||
SYNC_LISTING_CHECKPOINT_KEY = "mobilede:state:sync_listing_checkpoint"
|
||||
SYNC_LISTING_FOLLOWUP_PENDING_KEY = "mobilede:state:sync_listing_followup_pending"
|
||||
SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
|
||||
|
||||
|
||||
@@ -97,7 +94,7 @@ def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
|
||||
if not payload:
|
||||
continue
|
||||
data = json.loads(payload)
|
||||
if str(data.get("stage") or "") == "fast_db_progress":
|
||||
if str(data.get("stage") or "") in DB_PROGRESS_STAGES:
|
||||
ts = _safe_int(data.get("ts"), 0)
|
||||
else:
|
||||
ts = _safe_int(data.get("last_db_progress_ts"), 0)
|
||||
@@ -110,28 +107,28 @@ def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
|
||||
|
||||
def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None:
|
||||
pipe = redis_client.pipeline()
|
||||
pipe.delete(SYNC_LISTING_CHECKPOINT_KEY)
|
||||
pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
|
||||
pipe.delete(GLOBAL_PROGRESS_TS_KEY)
|
||||
pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY)
|
||||
pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0")
|
||||
pipe.execute()
|
||||
|
||||
|
||||
def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
|
||||
"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
|
||||
queue_len = _safe_int(redis_client.llen(queue_name), 0)
|
||||
has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0
|
||||
has_segment_lock = 0
|
||||
for _ in redis_client.scan_iter(match="mobilede:locks:segment:*"):
|
||||
has_segment_lock = 1
|
||||
break
|
||||
has_task_progress = 0
|
||||
for _ in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
|
||||
has_task_progress = 1
|
||||
break
|
||||
flags = {
|
||||
"queue_len": queue_len,
|
||||
"has_lock": has_lock,
|
||||
"has_segment_lock": has_segment_lock,
|
||||
"has_task_progress": has_task_progress,
|
||||
}
|
||||
return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags
|
||||
return (queue_len > 0 or has_segment_lock == 1 or has_task_progress == 1), flags
|
||||
|
||||
|
||||
def _kill_worker_process() -> None:
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user