add helper scripts
This commit is contained in:
qananasikq
1
dubizzle_scraper/__init__.py
Normal file
1
dubizzle_scraper/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
__all__: list[str] = []
|
||||
3
dubizzle_scraper/api/__init__.py
Normal file
3
dubizzle_scraper/api/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from .app import create_app
|
||||
|
||||
__all__ = ["create_app"]
|
||||
40
dubizzle_scraper/api/app.py
Normal file
40
dubizzle_scraper/api/app.py
Normal file
@@ -0,0 +1,40 @@
|
||||
# Создание FastAPI-приложения и настройка его жизненного цикла.
|
||||
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from fastapi import FastAPI
|
||||
|
||||
from ..core.config import Settings
|
||||
from ..storage.db import PersistenceService
|
||||
from .routes import cars, health, tasks
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
# Жизненный цикл.
|
||||
# Таблицы создаются через Alembic-миграции (сервис migrate).
|
||||
yield
|
||||
|
||||
|
||||
def create_app(settings: Settings | None = None) -> FastAPI:
|
||||
_settings = settings or Settings()
|
||||
|
||||
app = FastAPI(
|
||||
title="Dubizzle Scraper API",
|
||||
description="REST API для управления задачами скрапинга Dubizzle и просмотра данных",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
app.state.settings = _settings
|
||||
app.state.persistence = PersistenceService(_settings)
|
||||
|
||||
app.include_router(health.router, tags=["health"])
|
||||
app.include_router(cars.router, prefix="/api/v1", tags=["cars"])
|
||||
app.include_router(tasks.router, prefix="/api/v1", tags=["tasks"])
|
||||
|
||||
return app
|
||||
|
||||
|
||||
# Экземпляр приложения для запуска через uvicorn.
|
||||
app = create_app()
|
||||
9
dubizzle_scraper/api/deps.py
Normal file
9
dubizzle_scraper/api/deps.py
Normal file
@@ -0,0 +1,9 @@
|
||||
# Вспомогательные зависимости для FastAPI-роутов.
|
||||
|
||||
from fastapi import Request
|
||||
|
||||
from ..storage.db import PersistenceService
|
||||
|
||||
|
||||
def get_persistence(request: Request) -> PersistenceService:
|
||||
return request.app.state.persistence
|
||||
0
dubizzle_scraper/api/routes/__init__.py
Normal file
0
dubizzle_scraper/api/routes/__init__.py
Normal file
103
dubizzle_scraper/api/routes/cars.py
Normal file
103
dubizzle_scraper/api/routes/cars.py
Normal file
@@ -0,0 +1,103 @@
|
||||
# Роуты для просмотра автомобилей и агрегированной статистики.
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, Query
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from ..deps import get_persistence
|
||||
from ...storage.db import PersistenceService
|
||||
from ...storage.models import Car, Image
|
||||
from ...storage.schemas import CarRead
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
@router.get("/cars")
|
||||
def list_cars(
|
||||
page: int = Query(1, ge=1),
|
||||
per_page: int = Query(20, ge=1, le=100),
|
||||
brand: str | None = None,
|
||||
model: str | None = None,
|
||||
year_min: int | None = None,
|
||||
year_max: int | None = None,
|
||||
is_sold: bool | None = None,
|
||||
persistence: PersistenceService = Depends(get_persistence),
|
||||
):
|
||||
# Список автомобилей с пагинацией и фильтрами
|
||||
with persistence.session_scope() as session:
|
||||
query = select(Car)
|
||||
|
||||
if brand:
|
||||
query = query.where(Car.brand.ilike(f"%{brand}%"))
|
||||
if model:
|
||||
query = query.where(Car.model.ilike(f"%{model}%"))
|
||||
if year_min is not None:
|
||||
query = query.where(Car.year >= year_min)
|
||||
if year_max is not None:
|
||||
query = query.where(Car.year <= year_max)
|
||||
if is_sold is not None:
|
||||
query = query.where(Car.is_sold == is_sold)
|
||||
|
||||
count_query = select(func.count()).select_from(query.subquery())
|
||||
total = session.execute(count_query).scalar() or 0
|
||||
|
||||
offset = (page - 1) * per_page
|
||||
cars = session.execute(
|
||||
query.order_by(Car.last_seen_at.desc()).offset(offset).limit(per_page)
|
||||
).scalars().all()
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"pages": (total + per_page - 1) // per_page if per_page else 0,
|
||||
"items": [CarRead.model_validate(car).model_dump(mode="json") for car in cars],
|
||||
}
|
||||
|
||||
|
||||
@router.get("/cars/{car_id}")
|
||||
def get_car(
|
||||
car_id: int,
|
||||
persistence: PersistenceService = Depends(get_persistence),
|
||||
):
|
||||
# Детальная информация об автомобиле с изображениями
|
||||
with persistence.session_scope() as session:
|
||||
car = session.get(Car, car_id)
|
||||
if car is None:
|
||||
raise HTTPException(status_code=404, detail="Car not found")
|
||||
return CarRead.model_validate(car).model_dump(mode="json")
|
||||
|
||||
|
||||
@router.get("/cars/by-origin/{origin_id}")
|
||||
def get_car_by_origin(
|
||||
origin_id: str,
|
||||
persistence: PersistenceService = Depends(get_persistence),
|
||||
):
|
||||
# Поиск автомобиля по origin_id
|
||||
with persistence.session_scope() as session:
|
||||
car = session.execute(
|
||||
select(Car).where(Car.origin_id == origin_id)
|
||||
).scalars().first()
|
||||
if car is None:
|
||||
raise HTTPException(status_code=404, detail="Car not found")
|
||||
return CarRead.model_validate(car).model_dump(mode="json")
|
||||
|
||||
|
||||
@router.get("/stats")
|
||||
def get_stats(persistence: PersistenceService = Depends(get_persistence)):
|
||||
# Общая статистика по БД
|
||||
with persistence.session_scope() as session:
|
||||
total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
|
||||
total_images = session.execute(select(func.count(Image.id))).scalar() or 0
|
||||
brands = session.execute(
|
||||
select(Car.brand, func.count(Car.id))
|
||||
.group_by(Car.brand)
|
||||
.order_by(func.count(Car.id).desc())
|
||||
.limit(20)
|
||||
).all()
|
||||
|
||||
return {
|
||||
"total_cars": total_cars,
|
||||
"total_images": total_images,
|
||||
"top_brands": [{"brand": b, "count": c} for b, c in brands],
|
||||
}
|
||||
|
||||
30
dubizzle_scraper/api/routes/health.py
Normal file
30
dubizzle_scraper/api/routes/health.py
Normal file
@@ -0,0 +1,30 @@
|
||||
# Роут проверки доступности сервиса и соединения с БД.
|
||||
|
||||
import logging
|
||||
|
||||
from fastapi import APIRouter, Depends
|
||||
from sqlalchemy import text
|
||||
|
||||
from ..deps import get_persistence
|
||||
from ...storage.db import PersistenceService
|
||||
|
||||
router = APIRouter()
|
||||
logger = logging.getLogger("dubizzle_scraper.api.health")
|
||||
|
||||
|
||||
@router.get("/health")
|
||||
def health_check(persistence: PersistenceService = Depends(get_persistence)):
|
||||
# Проверка API и БД
|
||||
db_ok = False
|
||||
try:
|
||||
with persistence.session_scope() as session:
|
||||
session.execute(text("SELECT 1"))
|
||||
db_ok = True
|
||||
except Exception:
|
||||
logger.warning("Health DB check failed", exc_info=True)
|
||||
|
||||
return {
|
||||
"status": "ok" if db_ok else "degraded",
|
||||
"service": "dubizzle-scraper-api",
|
||||
"database": "connected" if db_ok else "unavailable",
|
||||
}
|
||||
120
dubizzle_scraper/api/routes/tasks.py
Normal file
120
dubizzle_scraper/api/routes/tasks.py
Normal file
@@ -0,0 +1,120 @@
|
||||
# Роуты запуска задач синхронизации и просмотра истории sync-runs.
|
||||
|
||||
from fastapi import APIRouter, Depends, Query
|
||||
from pydantic import BaseModel
|
||||
from sqlalchemy import select, func
|
||||
|
||||
from ..deps import get_persistence
|
||||
from ...storage.db import PersistenceService
|
||||
from ...storage.models import SyncRun
|
||||
from ...worker.celery_app import celery_app
|
||||
from ...worker.tasks import sync_vehicle_task, sync_listing_task
|
||||
|
||||
router = APIRouter()
|
||||
|
||||
|
||||
class SyncVehicleRequest(BaseModel):
|
||||
vehicle_url: str
|
||||
lane: str = "dubizzle"
|
||||
|
||||
|
||||
class SyncListingRequest(BaseModel):
|
||||
make: str | None = None
|
||||
model: str | None = None
|
||||
lane: str = "dubizzle_cars"
|
||||
limit: int | None = None
|
||||
only_new: bool | None = None
|
||||
|
||||
|
||||
@router.post("/tasks/sync-vehicle")
|
||||
def start_sync_vehicle(
|
||||
body: SyncVehicleRequest,
|
||||
):
|
||||
# Запустить задачу скрапинга одного автомобиля через Celery
|
||||
result = sync_vehicle_task.apply_async(
|
||||
kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
|
||||
queue="scraping",
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
"vehicle_url": body.vehicle_url,
|
||||
}
|
||||
|
||||
|
||||
@router.post("/tasks/sync-listing")
|
||||
def start_sync_listing(
|
||||
body: SyncListingRequest,
|
||||
):
|
||||
# Запустить задачу полного цикла листинга через Celery
|
||||
result = sync_listing_task.apply_async(
|
||||
kwargs={
|
||||
"make": body.make,
|
||||
"model": body.model,
|
||||
"lane": body.lane,
|
||||
"limit": body.limit,
|
||||
"only_new": body.only_new,
|
||||
},
|
||||
queue="scraping",
|
||||
)
|
||||
|
||||
return {
|
||||
"task_id": result.id,
|
||||
"status": "queued",
|
||||
}
|
||||
|
||||
|
||||
@router.get("/tasks/{task_id}")
|
||||
def get_task_status(task_id: str):
|
||||
result = celery_app.AsyncResult(task_id)
|
||||
|
||||
payload: dict = {
|
||||
"task_id": task_id,
|
||||
"state": result.state,
|
||||
}
|
||||
|
||||
if result.successful():
|
||||
payload["result"] = result.result
|
||||
elif result.failed():
|
||||
payload["error"] = str(result.result)
|
||||
elif result.info is not None:
|
||||
payload["meta"] = result.info
|
||||
|
||||
return payload
|
||||
|
||||
|
||||
@router.get("/sync-runs")
|
||||
def list_sync_runs(
|
||||
page: int = Query(1, ge=1),
|
||||
per_page: int = Query(20, ge=1, le=100),
|
||||
persistence: PersistenceService = Depends(get_persistence),
|
||||
):
|
||||
# История запусков синхронизации
|
||||
with persistence.session_scope() as session:
|
||||
total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
|
||||
offset = (page - 1) * per_page
|
||||
runs = session.execute(
|
||||
select(SyncRun).order_by(SyncRun.started_at.desc()).offset(offset).limit(per_page)
|
||||
).scalars().all()
|
||||
|
||||
return {
|
||||
"total": total,
|
||||
"page": page,
|
||||
"per_page": per_page,
|
||||
"items": [
|
||||
{
|
||||
"id": r.id,
|
||||
"started_at": r.started_at.isoformat() if r.started_at else None,
|
||||
"finished_at": r.finished_at.isoformat() if r.finished_at else None,
|
||||
"status": r.status,
|
||||
"lane": r.lane,
|
||||
"ids_fetched": r.ids_fetched,
|
||||
"cars_upserted": r.cars_upserted,
|
||||
"cars_failed": r.cars_failed,
|
||||
"images_upserted": r.images_upserted,
|
||||
"error_summary": r.error_summary,
|
||||
}
|
||||
for r in runs
|
||||
],
|
||||
}
|
||||
6
dubizzle_scraper/browser/__init__.py
Normal file
6
dubizzle_scraper/browser/__init__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
from .factory import BrowserFactory
|
||||
from .listing import ListingCollector
|
||||
from .network import NetworkCapture
|
||||
from .pace import HumanPacer
|
||||
|
||||
__all__ = ["BrowserFactory", "ListingCollector", "NetworkCapture", "HumanPacer"]
|
||||
214
dubizzle_scraper/browser/factory.py
Normal file
214
dubizzle_scraper/browser/factory.py
Normal file
@@ -0,0 +1,214 @@
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
|
||||
from playwright.sync_api import Browser, BrowserContext, Playwright
|
||||
|
||||
try:
|
||||
from playwright_stealth import stealth_sync
|
||||
except ImportError:
|
||||
stealth_sync = None
|
||||
|
||||
from ..core.config import Settings
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.browser")
|
||||
|
||||
|
||||
def _build_init_script() -> str:
|
||||
# Маскировка браузера.
|
||||
hardware_concurrency = random.choice([4, 8, 12, 16])
|
||||
device_memory = random.choice([4, 8, 16])
|
||||
languages = ["en-US", "en"]
|
||||
|
||||
return f"""
|
||||
(() => {{
|
||||
const define = (obj, prop, value) => {{
|
||||
try {{
|
||||
Object.defineProperty(obj, prop, {{ get: () => value, configurable: true }});
|
||||
}} catch (e) {{}}
|
||||
}};
|
||||
|
||||
define(navigator, 'webdriver', undefined);
|
||||
define(navigator, 'platform', 'Win32');
|
||||
define(navigator, 'vendor', 'Google Inc.');
|
||||
define(navigator, 'language', '{languages[0]}');
|
||||
define(navigator, 'languages', {json.dumps(languages)});
|
||||
define(navigator, 'hardwareConcurrency', {hardware_concurrency});
|
||||
define(navigator, 'deviceMemory', {device_memory});
|
||||
define(navigator, 'maxTouchPoints', 0);
|
||||
|
||||
if (!window.chrome) {{
|
||||
Object.defineProperty(window, 'chrome', {{
|
||||
value: {{ runtime: {{}}, app: {{}}, csi: () => ({{}}), loadTimes: () => ({{}}) }},
|
||||
configurable: true
|
||||
}});
|
||||
}}
|
||||
|
||||
const originalQuery = navigator.permissions && navigator.permissions.query;
|
||||
if (originalQuery) {{
|
||||
navigator.permissions.query = (params) => (
|
||||
params && params.name === 'notifications'
|
||||
? Promise.resolve({{ state: Notification.permission }})
|
||||
: originalQuery(params)
|
||||
);
|
||||
}}
|
||||
|
||||
const originalGetParameter = WebGLRenderingContext.prototype.getParameter;
|
||||
WebGLRenderingContext.prototype.getParameter = function(parameter) {{
|
||||
if (parameter === 37445) return 'Intel Inc.';
|
||||
if (parameter === 37446) return 'Intel Iris OpenGL Engine';
|
||||
return originalGetParameter.call(this, parameter);
|
||||
}};
|
||||
}})();
|
||||
"""
|
||||
|
||||
|
||||
class BrowserFactory:
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.settings = settings
|
||||
|
||||
def _resolve_engine(self) -> str:
|
||||
# Выбор движка.
|
||||
engine = self.settings.browser_engine.strip().lower()
|
||||
if engine == "auto":
|
||||
# Для DUBIZZLE стабильнее Chromium.
|
||||
return "chromium"
|
||||
if engine in ("firefox", "chromium"):
|
||||
return engine
|
||||
logger.warning("Unknown DUBIZZLE_BROWSER_ENGINE=%r, falling back to auto", engine)
|
||||
return "chromium"
|
||||
|
||||
def create_browser(self, playwright: Playwright) -> Browser:
|
||||
engine = self._resolve_engine()
|
||||
proxy_dict = self.settings.proxy.to_playwright_dict()
|
||||
logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
|
||||
|
||||
if engine == "firefox":
|
||||
launch_kwargs: dict = {"headless": self.settings.headless}
|
||||
if proxy_dict:
|
||||
launch_kwargs["proxy"] = proxy_dict
|
||||
logger.info("Using proxy: %s", self.settings.proxy.server)
|
||||
# Настройки Firefox.
|
||||
launch_kwargs["firefox_user_prefs"] = {
|
||||
"dom.webdriver.enabled": False,
|
||||
"useAutomationExtension": False,
|
||||
# Базовые оптимизации.
|
||||
"media.autoplay.default": 5,
|
||||
"media.volume_scale": "0.0",
|
||||
"media.audio.playback.standalone": False,
|
||||
"dom.ipc.processCount": 1,
|
||||
"dom.ipc.plugins.enabled": False,
|
||||
"browser.cache.disk.enable": False,
|
||||
"browser.cache.memory.enable": True,
|
||||
"browser.cache.memory.max_entry_size": 8192,
|
||||
"network.prefetch-next": False,
|
||||
"network.dns.disablePrefetch": True,
|
||||
"permissions.default.image": 2,
|
||||
"javascript.options.mem.gc_incremental_mark_slice_ms": 20,
|
||||
}
|
||||
logger.info("Launching Firefox (headless=%s)", self.settings.headless)
|
||||
return playwright.firefox.launch(**launch_kwargs)
|
||||
|
||||
# Запуск Chromium.
|
||||
args = [
|
||||
"--disable-blink-features=AutomationControlled",
|
||||
"--no-default-browser-check",
|
||||
"--disable-dev-shm-usage",
|
||||
"--disable-features=IsolateOrigins,site-per-process",
|
||||
]
|
||||
if self.settings.headless:
|
||||
args.append("--headless=new")
|
||||
pw_headless = False
|
||||
logger.info("Using Chromium new-headless mode (--headless=new)")
|
||||
else:
|
||||
pw_headless = False
|
||||
|
||||
launch_kwargs = {"headless": pw_headless, "args": args}
|
||||
if proxy_dict:
|
||||
launch_kwargs["proxy"] = proxy_dict
|
||||
logger.info("Using proxy: %s", self.settings.proxy.server)
|
||||
try:
|
||||
logger.info("Trying to launch real Chrome channel")
|
||||
return playwright.chromium.launch(channel="chrome", **launch_kwargs)
|
||||
except Exception:
|
||||
logger.warning("Chrome channel launch failed, falling back to Chromium")
|
||||
return playwright.chromium.launch(**launch_kwargs)
|
||||
|
||||
def create_context(self, browser: Browser) -> BrowserContext:
|
||||
viewport = random.choice(self.settings.fingerprint.viewport_presets)
|
||||
timezone_id = random.choice(self.settings.fingerprint.timezone_candidates)
|
||||
color_scheme = random.choice(["light", "dark"])
|
||||
|
||||
is_firefox = browser.browser_type.name == "firefox"
|
||||
|
||||
ctx_kwargs: dict = {
|
||||
"viewport": viewport,
|
||||
"screen": viewport,
|
||||
"locale": self.settings.fingerprint.locale,
|
||||
"timezone_id": timezone_id,
|
||||
"color_scheme": color_scheme,
|
||||
"java_script_enabled": True,
|
||||
"ignore_https_errors": False,
|
||||
}
|
||||
|
||||
if is_firefox:
|
||||
# Заголовки Firefox.
|
||||
ctx_kwargs["user_agent"] = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
|
||||
"Gecko/20100101 Firefox/128.0"
|
||||
)
|
||||
ctx_kwargs["extra_http_headers"] = {
|
||||
"Accept-Language": "en-US,en;q=0.5",
|
||||
"DNT": "1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
}
|
||||
else:
|
||||
ctx_kwargs["user_agent"] = self.settings.fingerprint.user_agent
|
||||
ctx_kwargs["device_scale_factor"] = random.choice([1, 1.25])
|
||||
ctx_kwargs["is_mobile"] = False
|
||||
ctx_kwargs["has_touch"] = False
|
||||
ctx_kwargs["extra_http_headers"] = {
|
||||
"Accept-Language": "en-US,en;q=0.9",
|
||||
"DNT": "1",
|
||||
"Upgrade-Insecure-Requests": "1",
|
||||
"Sec-CH-UA": self.settings.fingerprint.sec_ch_ua,
|
||||
"Sec-CH-UA-Mobile": "?0",
|
||||
"Sec-CH-UA-Platform": '"Windows"',
|
||||
}
|
||||
|
||||
context = browser.new_context(**ctx_kwargs)
|
||||
context.set_default_timeout(self.settings.default_timeout_ms)
|
||||
context.set_default_navigation_timeout(self.settings.default_timeout_ms)
|
||||
|
||||
if not is_firefox:
|
||||
# Маскировка Chromium.
|
||||
context.add_init_script(_build_init_script())
|
||||
if stealth_sync:
|
||||
context.on("page", lambda page: stealth_sync(page))
|
||||
logger.debug("playwright-stealth attached to context")
|
||||
|
||||
return context
|
||||
|
||||
@staticmethod
|
||||
def enable_resource_blocking(page) -> None:
|
||||
# Блокируем тяжёлые ресурсы.
|
||||
BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
|
||||
BLOCKED_URL_PATTERNS = (
|
||||
"google-analytics", "googletagmanager", "facebook.net",
|
||||
"doubleclick.net", "hotjar", "newrelic", ".woff", ".woff2",
|
||||
"analytics", "tracking", "adservice",
|
||||
)
|
||||
|
||||
def _handle_route(route):
|
||||
req = route.request
|
||||
if req.resource_type in BLOCKED_TYPES:
|
||||
route.abort()
|
||||
return
|
||||
url = req.url.lower()
|
||||
if any(pat in url for pat in BLOCKED_URL_PATTERNS):
|
||||
route.abort()
|
||||
return
|
||||
route.continue_()
|
||||
|
||||
page.route("**/*", _handle_route)
|
||||
725
dubizzle_scraper/browser/listing.py
Normal file
725
dubizzle_scraper/browser/listing.py
Normal file
@@ -0,0 +1,725 @@
|
||||
import logging
|
||||
import re
|
||||
import time
|
||||
from dataclasses import asdict, dataclass, field
|
||||
from typing import Any
|
||||
from urllib.parse import urljoin
|
||||
|
||||
from playwright.sync_api import Page
|
||||
|
||||
from .pace import HumanPacer
|
||||
from ..core.config import Settings
|
||||
from ..core.utils import first_non_empty
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.listing")
|
||||
VEHICLE_HREF_RE = re.compile(
|
||||
r'(?:/VehicleDetail/(?P<veh_id>\d+)(?:~[A-Z]{2})?)|(?:/motors/used-cars/[^"\s]+?(?:/ad-(?P<ad_id>\d+)/?|---(?P<slug_id>[a-f0-9]{32})/?))|(?:/s/(?P<short_id>[A-Za-z0-9]+))',
|
||||
re.IGNORECASE,
|
||||
)
|
||||
VEHICLE_LINK_SELECTOR = (
|
||||
"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail'], "
|
||||
"a[href*='/motors/used-cars/'], a[href*='/s/']"
|
||||
)
|
||||
COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
|
||||
"button:has-text('Accept All')",
|
||||
"button:has-text('Accept all')",
|
||||
"button:has-text('I Agree')",
|
||||
"button:has-text('Agree')",
|
||||
"button:has-text('Only necessary')",
|
||||
"button:has-text('Только необходимые')",
|
||||
"button:has-text('Принять все')",
|
||||
"[id*='accept']",
|
||||
"[class*='accept']",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ListingVehicleLink:
|
||||
href: str
|
||||
title: str = ""
|
||||
lot_number: str | None = None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ListingPageResult:
|
||||
source_url: str
|
||||
page_number: int
|
||||
vehicle_links: list[ListingVehicleLink] = field(default_factory=list)
|
||||
pagination_available: bool = False
|
||||
next_page_detected: bool = False
|
||||
|
||||
|
||||
class ListingCollector:
|
||||
_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
|
||||
"a[aria-label*='Next']",
|
||||
"button[aria-label*='Next']",
|
||||
"a[aria-label*='next']",
|
||||
"button[aria-label*='next']",
|
||||
"a[title*='Next']",
|
||||
"button[title*='Next']",
|
||||
"a[title*='next']",
|
||||
"button[title*='next']",
|
||||
"a[rel='next']",
|
||||
"link[rel='next']",
|
||||
"a.pagination-next",
|
||||
"button.pagination-next",
|
||||
"a.next",
|
||||
"button.next",
|
||||
"a:has-text('Next')",
|
||||
"button:has-text('Next')",
|
||||
"a:has-text('NEXT')",
|
||||
"button:has-text('NEXT')",
|
||||
"a:has-text('›')",
|
||||
"button:has-text('›')",
|
||||
"a:has-text('»')",
|
||||
"button:has-text('»')",
|
||||
"a:has(img[src*='icon-arrow-right'])",
|
||||
"button:has(img[src*='icon-arrow-right'])",
|
||||
"a:has(img[src*='arrow-right'])",
|
||||
"button:has(img[src*='arrow-right'])",
|
||||
)
|
||||
|
||||
def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
|
||||
self.settings = settings
|
||||
self.pacer = pacer
|
||||
|
||||
@staticmethod
|
||||
def _get_current_page_number(page: Page) -> int | None:
|
||||
try:
|
||||
value = page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
|
||||
const current = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
||||
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
|
||||
});
|
||||
if (current) {
|
||||
return parseInt((current.textContent || '').trim(), 10);
|
||||
}
|
||||
const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
|
||||
return match ? parseInt(match[1], 10) : null;
|
||||
}
|
||||
"""
|
||||
)
|
||||
return int(value) if value is not None else None
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
|
||||
if old_first_href:
|
||||
try:
|
||||
page.wait_for_function(
|
||||
f"""() => {{
|
||||
const a = document.querySelector(\"{VEHICLE_LINK_SELECTOR}\");
|
||||
return a && a.getAttribute('href') !== '{old_first_href}';
|
||||
}}""",
|
||||
timeout=12000,
|
||||
)
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if expected_page_number is not None:
|
||||
current_page = ListingCollector._get_current_page_number(page)
|
||||
if current_page == expected_page_number:
|
||||
return True
|
||||
|
||||
try:
|
||||
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
current_page = ListingCollector._get_current_page_number(page)
|
||||
if expected_page_number is not None and current_page == expected_page_number:
|
||||
return True
|
||||
|
||||
return not old_first_href
|
||||
|
||||
@staticmethod
|
||||
def has_page_number(page: Page, target_page_number: int) -> bool:
|
||||
try:
|
||||
return bool(page.evaluate(
|
||||
"""
|
||||
(targetPageNumber) => {
|
||||
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
|
||||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
|
||||
return controls.some((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
|
||||
const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
|
||||
return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
|
||||
});
|
||||
}
|
||||
""",
|
||||
target_page_number,
|
||||
))
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
|
||||
url = url_override or self.settings.listing.cars_url
|
||||
logger.info("Opening cars listing page: %s", url)
|
||||
last_err = None
|
||||
for attempt in range(3):
|
||||
try:
|
||||
page.goto(url, wait_until="commit", timeout=60_000)
|
||||
last_err = None
|
||||
break
|
||||
except Exception as e:
|
||||
last_err = e
|
||||
logger.warning("goto listing attempt %d failed: %s", attempt + 1, e)
|
||||
# Небольшой backoff при ошибках открытия листинга.
|
||||
time.sleep(5 * (attempt + 1))
|
||||
if last_err:
|
||||
logger.warning("All goto attempts failed, trying JS navigation")
|
||||
try:
|
||||
page.evaluate(f"window.location.href = '{url}'")
|
||||
except Exception:
|
||||
pass
|
||||
# Быстрая проверка готовности страницы.
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=12_000)
|
||||
except Exception:
|
||||
pass
|
||||
self._accept_cookie_banner(page)
|
||||
self._wait_for_listing_content(page)
|
||||
logger.info("Listing page URL: %s", page.url)
|
||||
self.pacer.after_listing_open()
|
||||
|
||||
def _accept_cookie_banner(self, page: Page) -> None:
|
||||
for selector in COOKIE_ACCEPT_SELECTORS:
|
||||
locator = page.locator(selector).first
|
||||
try:
|
||||
if locator.count() == 0:
|
||||
continue
|
||||
if not locator.is_visible(timeout=500):
|
||||
continue
|
||||
locator.click(timeout=2_000)
|
||||
logger.info("Accepted cookie banner using selector: %s", selector)
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=3_000)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
def _wait_for_listing_content(self, page: Page) -> None:
|
||||
try:
|
||||
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
|
||||
try:
|
||||
page.wait_for_function(
|
||||
"""() => {
|
||||
const html = document.documentElement?.innerHTML || '';
|
||||
const text = document.body?.innerText || '';
|
||||
return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
|
||||
}""",
|
||||
timeout=12_000,
|
||||
)
|
||||
except Exception:
|
||||
# Короткая пауза вместо длинного sleep.
|
||||
time.sleep(1.0)
|
||||
|
||||
def apply_filters(
|
||||
self,
|
||||
page: Page,
|
||||
make: str | None = None,
|
||||
model: str | None = None,
|
||||
year_min: int | None = None,
|
||||
year_max: int | None = None,
|
||||
) -> dict[str, str | int | None]:
|
||||
applied: dict[str, str | int | None] = {"make": None, "model": None, "year_min": None, "year_max": None}
|
||||
if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
|
||||
applied["make"] = make
|
||||
self.pacer.after_filter_action()
|
||||
if model and self._try_fill_filter_input(page, ["input[placeholder*='Model']", "input[aria-label*='Model']"], model):
|
||||
applied["model"] = model
|
||||
self.pacer.after_filter_action()
|
||||
if year_min is not None or year_max is not None:
|
||||
if self._apply_year_range(page, year_min, year_max):
|
||||
applied["year_min"] = year_min
|
||||
applied["year_max"] = year_max
|
||||
self.pacer.after_filter_action()
|
||||
return applied
|
||||
|
||||
def _apply_year_range(self, page: Page, year_min: int | None, year_max: int | None) -> bool:
|
||||
"""Заполняет поля фильтра Year и нажимает Apply Year."""
|
||||
if year_min is None and year_max is None:
|
||||
return False
|
||||
try:
|
||||
success = page.evaluate(
|
||||
"""([yearMin, yearMax]) => {
|
||||
const inputs = Array.from(document.querySelectorAll('input'));
|
||||
const yearInputs = inputs.filter(inp => {
|
||||
const v = parseInt(inp.value, 10);
|
||||
return !isNaN(v) && v >= 1900 && v <= 2100;
|
||||
});
|
||||
if (yearInputs.length < 2) return false;
|
||||
yearInputs.sort((a, b) => parseInt(a.value) - parseInt(b.value));
|
||||
const setVal = (el, val) => {
|
||||
const setter = Object.getOwnPropertyDescriptor(
|
||||
HTMLInputElement.prototype, 'value'
|
||||
).set;
|
||||
setter.call(el, String(val));
|
||||
el.dispatchEvent(new Event('input', {bubbles: true}));
|
||||
el.dispatchEvent(new Event('change', {bubbles: true}));
|
||||
};
|
||||
if (yearMin !== null) setVal(yearInputs[0], yearMin);
|
||||
if (yearMax !== null) setVal(yearInputs[yearInputs.length - 1], yearMax);
|
||||
const container = yearInputs[0].closest(
|
||||
'[class*="filter"], [class*="year"], section, fieldset'
|
||||
) || yearInputs[0].parentElement.parentElement;
|
||||
if (container) {
|
||||
const btn = Array.from(container.querySelectorAll(
|
||||
'button, a, [role="button"], span[class*="apply"]'
|
||||
)).find(el => /apply|\u043f\u0440\u0438\u043c\u0435\u043d/i.test(el.textContent));
|
||||
if (btn) { btn.click(); return true; }
|
||||
}
|
||||
yearInputs[yearInputs.length - 1].dispatchEvent(
|
||||
new KeyboardEvent('keydown', {
|
||||
key: 'Enter', code: 'Enter', keyCode: 13, bubbles: true
|
||||
})
|
||||
);
|
||||
return true;
|
||||
}""",
|
||||
[year_min, year_max],
|
||||
)
|
||||
if success:
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=15_000)
|
||||
except Exception:
|
||||
pass
|
||||
self._wait_for_listing_content(page)
|
||||
logger.info("Applied year range filter: %s — %s", year_min, year_max)
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to apply year range filter: %s", exc)
|
||||
return False
|
||||
|
||||
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
|
||||
# Считываем ссылки одним проходом по DOM.
|
||||
self._accept_cookie_banner(page)
|
||||
self._wait_for_listing_content(page)
|
||||
try:
|
||||
raw_items = page.eval_on_selector_all(
|
||||
"a[href], [data-href], [href]",
|
||||
"""
|
||||
(nodes) => nodes.map((node) => ({
|
||||
href:
|
||||
node.getAttribute('href') ||
|
||||
node.getAttribute('data-href') ||
|
||||
node.getAttribute('data-url') ||
|
||||
'',
|
||||
title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
|
||||
text: (node.textContent || '').trim(),
|
||||
}))
|
||||
""",
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
|
||||
raw_items = []
|
||||
total = min(len(raw_items), self.settings.listing.page_link_limit)
|
||||
links: list[ListingVehicleLink] = []
|
||||
seen: set[str] = set()
|
||||
for idx in range(total):
|
||||
item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
|
||||
href = str(item.get("href") or "")
|
||||
match = VEHICLE_HREF_RE.search(href)
|
||||
if not match:
|
||||
continue
|
||||
lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or None
|
||||
if lot_number is None:
|
||||
short = match.group("short_id")
|
||||
lot_number = short if short else None
|
||||
absolute = urljoin(self.settings.home_url, match.group(0))
|
||||
if absolute in seen:
|
||||
continue
|
||||
seen.add(absolute)
|
||||
title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
|
||||
links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
|
||||
if len(links) >= self.settings.listing.max_vehicles_per_run:
|
||||
break
|
||||
|
||||
# Fallback: на DUBIZZLE ссылки иногда не рендерятся как <a>,
|
||||
# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
|
||||
if not links:
|
||||
try:
|
||||
page.wait_for_timeout(1_500)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
html = page.content()
|
||||
except Exception as exc:
|
||||
logger.debug("page.content() failed on page %d: %s", page_number, exc)
|
||||
html = ""
|
||||
|
||||
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
|
||||
if absolute in seen:
|
||||
continue
|
||||
seen.add(absolute)
|
||||
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
|
||||
if len(links) >= self.settings.listing.max_vehicles_per_run:
|
||||
break
|
||||
|
||||
if links:
|
||||
logger.info(
|
||||
"Page %d: recovered %d vehicle links from HTML fallback",
|
||||
page_number,
|
||||
len(links),
|
||||
)
|
||||
next_page_detected = self._has_next_page(page)
|
||||
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
|
||||
|
||||
def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
|
||||
if not html:
|
||||
return []
|
||||
|
||||
# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
|
||||
normalized = html.replace("\\/", "/")
|
||||
found: list[tuple[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
|
||||
for match in VEHICLE_HREF_RE.finditer(normalized):
|
||||
lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or match.group("short_id") or ""
|
||||
absolute = urljoin(self.settings.home_url, match.group(0))
|
||||
if absolute in seen:
|
||||
continue
|
||||
seen.add(absolute)
|
||||
found.append((absolute, lot_number))
|
||||
if len(found) >= self.settings.listing.page_link_limit:
|
||||
break
|
||||
|
||||
return found
|
||||
|
||||
def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
|
||||
# Запоминаем первую ссылку текущей страницы для определения смены контента.
|
||||
old_first_href = ""
|
||||
try:
|
||||
first_link = page.locator(VEHICLE_LINK_SELECTOR).first
|
||||
if first_link.count() > 0:
|
||||
old_first_href = first_link.get_attribute("href") or ""
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
for selector in self._NEXT_PAGE_SELECTORS:
|
||||
locator = page.locator(selector).first
|
||||
if locator.count() == 0:
|
||||
continue
|
||||
try:
|
||||
disabled = (locator.get_attribute("disabled", timeout=1500) or "").lower()
|
||||
aria_disabled = (locator.get_attribute("aria-disabled", timeout=1500) or "").lower()
|
||||
classes = (locator.get_attribute("class", timeout=1500) or "").lower()
|
||||
except Exception:
|
||||
continue
|
||||
if disabled or aria_disabled == "true" or "disabled" in classes:
|
||||
continue
|
||||
try:
|
||||
self.pacer.move_mouse_to(page, locator)
|
||||
locator.click(timeout=8000)
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
|
||||
self.pacer.after_page_change()
|
||||
return True
|
||||
|
||||
# Fallback для DUBIZZLE: пагинация часто рендерится как набор номеров страниц
|
||||
# + стрелка с иконкой, без явного текста Next.
|
||||
try:
|
||||
clicked = bool(page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
|
||||
const disabled = (el) => {
|
||||
if (!el) return true;
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
|
||||
return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
|
||||
};
|
||||
|
||||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
|
||||
.filter((el) => visible(el) && !disabled(el));
|
||||
|
||||
const current = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
||||
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
|
||||
});
|
||||
|
||||
if (current) {
|
||||
const currentPage = parseInt((current.textContent || '').trim(), 10);
|
||||
const nextNumber = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
|
||||
});
|
||||
if (nextNumber) {
|
||||
nextNumber.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
const iconNext = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim().toLowerCase();
|
||||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||||
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
|
||||
return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
|
||||
});
|
||||
|
||||
if (iconNext) {
|
||||
iconNext.click();
|
||||
return true;
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
"""
|
||||
))
|
||||
if clicked:
|
||||
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
|
||||
self.pacer.after_page_change()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug("Numeric/icon pagination fallback failed: %s", exc)
|
||||
|
||||
# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
|
||||
try:
|
||||
clicked = bool(page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
|
||||
for (const el of candidates) {
|
||||
const text = (el.textContent || '').trim().toLowerCase();
|
||||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||||
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
|
||||
const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
|
||||
const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
|
||||
if (!disabled && visible && looksNext) {
|
||||
el.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
"""
|
||||
))
|
||||
if clicked:
|
||||
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
|
||||
self.pacer.after_page_change()
|
||||
return True
|
||||
except Exception as exc:
|
||||
logger.debug("JS next-page fallback failed: %s", exc)
|
||||
|
||||
logger.warning("Could not navigate to next page from %s", page.url)
|
||||
return False
|
||||
|
||||
def collect_listing_links(
|
||||
self,
|
||||
page: Page,
|
||||
*,
|
||||
make: str | None = None,
|
||||
model: str | None = None,
|
||||
known_origin_ids: set[str] | None = None,
|
||||
max_duration_seconds: float | None = None,
|
||||
) -> dict[str, Any]:
|
||||
self.open_cars_listing(page)
|
||||
applied_filters = self.apply_filters(page, make=make, model=model)
|
||||
started_at = time.perf_counter()
|
||||
truncated_by_time_budget = False
|
||||
pages: list[dict[str, object]] = []
|
||||
all_links: list[str] = []
|
||||
early_stopped = False
|
||||
threshold = self.settings.listing.early_stop_threshold
|
||||
|
||||
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
|
||||
if max_duration_seconds is not None and max_duration_seconds > 0:
|
||||
elapsed = time.perf_counter() - started_at
|
||||
if elapsed >= max_duration_seconds:
|
||||
truncated_by_time_budget = True
|
||||
logger.warning(
|
||||
"Listing collection stopped by time budget: page=%d elapsed=%.1fs budget=%.1fs",
|
||||
page_number,
|
||||
elapsed,
|
||||
max_duration_seconds,
|
||||
)
|
||||
break
|
||||
page_result = self.collect_current_page(page, page_number=page_number)
|
||||
pages.append({
|
||||
"page_number": page_result.page_number,
|
||||
"source_url": page_result.source_url,
|
||||
"links_found": len(page_result.vehicle_links),
|
||||
"vehicle_links": [asdict(item) for item in page_result.vehicle_links],
|
||||
"next_page_detected": page_result.next_page_detected,
|
||||
})
|
||||
for item in page_result.vehicle_links:
|
||||
if item.href not in all_links:
|
||||
all_links.append(item.href)
|
||||
if len(all_links) >= self.settings.listing.max_vehicles_per_run:
|
||||
break
|
||||
|
||||
# Ранний останов: если на этой странице много известных И нет новых — дальше нет смысла.
|
||||
# Важно: если есть хоть одна новая машина — продолжаем листать (новые могут быть на любой странице).
|
||||
if (
|
||||
known_origin_ids is not None
|
||||
and threshold > 0.0
|
||||
and page_result.vehicle_links
|
||||
):
|
||||
page_known = sum(
|
||||
1 for item in page_result.vehicle_links
|
||||
if item.lot_number and f"dubizzle:{item.lot_number}" in known_origin_ids
|
||||
)
|
||||
page_new = len(page_result.vehicle_links) - page_known
|
||||
ratio = page_known / len(page_result.vehicle_links)
|
||||
# Останавливаемся только если нет новых И порог превышен
|
||||
if ratio >= threshold and page_new == 0:
|
||||
logger.info(
|
||||
"Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%",
|
||||
page_number, ratio * 100, page_known,
|
||||
len(page_result.vehicle_links), threshold * 100,
|
||||
)
|
||||
early_stopped = True
|
||||
break
|
||||
elif page_new > 0 and ratio >= threshold:
|
||||
logger.info(
|
||||
"Page %d: %.0f%% known but %d new found — продолжаем",
|
||||
page_number, ratio * 100, page_new,
|
||||
)
|
||||
|
||||
if (
|
||||
len(all_links) >= self.settings.listing.max_vehicles_per_run
|
||||
or self.settings.listing.collect_current_page_only
|
||||
or not self.settings.listing.include_pagination
|
||||
or not page_result.next_page_detected
|
||||
):
|
||||
break
|
||||
if not self.go_to_next_page(page):
|
||||
break
|
||||
|
||||
return {
|
||||
"listing_url": self.settings.listing.cars_url,
|
||||
"applied_filters": applied_filters,
|
||||
"pages_collected": len(pages),
|
||||
"vehicles_collected": len(all_links),
|
||||
"vehicle_urls": all_links,
|
||||
"early_stopped": early_stopped,
|
||||
"truncated_by_time_budget": truncated_by_time_budget,
|
||||
"pages": pages,
|
||||
"strategy": {
|
||||
"sequential": True,
|
||||
"collect_current_page_only": self.settings.listing.collect_current_page_only,
|
||||
"include_pagination": self.settings.listing.include_pagination,
|
||||
"max_pages_per_run": self.settings.listing.max_pages_per_run,
|
||||
"max_vehicles_per_run": self.settings.listing.max_vehicles_per_run,
|
||||
"early_stop_threshold": threshold,
|
||||
},
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _try_fill_filter_input(page: Page, selectors: list[str], value: str) -> bool:
|
||||
for selector in selectors:
|
||||
locator = page.locator(selector).first
|
||||
if locator.count() == 0:
|
||||
continue
|
||||
try:
|
||||
locator.click()
|
||||
locator.fill(value)
|
||||
page.keyboard.press("Enter")
|
||||
try:
|
||||
page.wait_for_load_state("domcontentloaded", timeout=15000)
|
||||
except Exception:
|
||||
pass
|
||||
try:
|
||||
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _has_next_page(page: Page) -> bool:
|
||||
for selector in ListingCollector._NEXT_PAGE_SELECTORS:
|
||||
locator = page.locator(selector)
|
||||
count = locator.count()
|
||||
if count == 0:
|
||||
continue
|
||||
# Тестовые/fake локаторы могут не поддерживать nth/get_attribute.
|
||||
# В таком случае считаем наличие селектора достаточным признаком next.
|
||||
if not hasattr(locator, "nth"):
|
||||
return True
|
||||
# Проверяем, что хотя бы один элемент не disabled.
|
||||
# Disabled "Next" на последней странице не означает наличия следующей.
|
||||
for i in range(min(count, 3)):
|
||||
try:
|
||||
el = locator.nth(i)
|
||||
disabled_attr = el.get_attribute("disabled", timeout=300)
|
||||
aria_disabled = el.get_attribute("aria-disabled", timeout=300)
|
||||
cls = (el.get_attribute("class", timeout=300) or "").lower()
|
||||
if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
try:
|
||||
return bool(page.evaluate(
|
||||
"""
|
||||
() => {
|
||||
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
|
||||
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')).filter((el) => {
|
||||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||||
const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
|
||||
return !disabled && visible(el);
|
||||
});
|
||||
|
||||
const hasExplicitNext = controls.some((el) => {
|
||||
const text = (el.textContent || '').trim().toLowerCase();
|
||||
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
|
||||
const title = (el.getAttribute('title') || '').trim().toLowerCase();
|
||||
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
|
||||
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
|
||||
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
|
||||
return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
|
||||
});
|
||||
|
||||
if (hasExplicitNext) {
|
||||
return true;
|
||||
}
|
||||
|
||||
const current = controls.find((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
const cls = (el.getAttribute('class') || '').toLowerCase();
|
||||
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
|
||||
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
|
||||
});
|
||||
|
||||
if (!current) {
|
||||
return false;
|
||||
}
|
||||
|
||||
const currentPage = parseInt((current.textContent || '').trim(), 10);
|
||||
return controls.some((el) => {
|
||||
const text = (el.textContent || '').trim();
|
||||
return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
|
||||
});
|
||||
}
|
||||
"""
|
||||
))
|
||||
except Exception:
|
||||
return False
|
||||
return False
|
||||
130
dubizzle_scraper/browser/network.py
Normal file
130
dubizzle_scraper/browser/network.py
Normal file
@@ -0,0 +1,130 @@
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from playwright.sync_api import Page, Request, Response
|
||||
|
||||
from ..core.config import Settings
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.network")
|
||||
|
||||
|
||||
@dataclass
|
||||
class NetworkCapture:
|
||||
# Перехватчик сетевых запросов.
|
||||
|
||||
settings: Settings
|
||||
requests: list[dict[str, Any]] = field(default_factory=list)
|
||||
json_responses: list[dict[str, Any]] = field(default_factory=list)
|
||||
_seen_req: set[str] = field(default_factory=set)
|
||||
_seen_resp: set[str] = field(default_factory=set)
|
||||
_origin: str | None = None
|
||||
_page: Any = field(default=None)
|
||||
|
||||
def attach(self, page: Page, origin_url: str | None = None) -> None:
|
||||
# Снимаем старые подписки перед повторным attach.
|
||||
try:
|
||||
page.remove_listener("request", self._on_request)
|
||||
page.remove_listener("response", self._on_response)
|
||||
except Exception:
|
||||
pass
|
||||
# Подписка на сетевые события
|
||||
try:
|
||||
self._origin = urlparse(origin_url or page.url).netloc.lower() or None
|
||||
except Exception:
|
||||
self._origin = None
|
||||
self._page = page
|
||||
page.on("request", self._on_request)
|
||||
page.on("response", self._on_response)
|
||||
|
||||
def _is_same_origin(self, url: str) -> bool:
|
||||
# Фильтр по домену
|
||||
if not self.settings.capture.capture_same_origin_only or not self._origin:
|
||||
return True
|
||||
netloc = urlparse(url).netloc.lower()
|
||||
return netloc == self._origin or netloc.endswith(".dubizzle.com")
|
||||
|
||||
def _on_request(self, request: Request) -> None:
|
||||
# Берём только xhr/fetch
|
||||
if request.resource_type not in {"xhr", "fetch"}:
|
||||
return
|
||||
if not self._is_same_origin(request.url):
|
||||
return
|
||||
if len(self.requests) >= self.settings.capture.max_requests:
|
||||
return
|
||||
key = f"{request.method}:{request.url}:{request.post_data or ''}"
|
||||
# Убираем дубли
|
||||
if key in self._seen_req:
|
||||
return
|
||||
self._seen_req.add(key)
|
||||
self.requests.append({
|
||||
"url": request.url, "method": request.method,
|
||||
"resource_type": request.resource_type, "post_data": request.post_data,
|
||||
})
|
||||
|
||||
def _on_response(self, response: Response) -> None:
|
||||
# Сохраняем JSON
|
||||
request = response.request
|
||||
if request.resource_type not in {"xhr", "fetch"}:
|
||||
return
|
||||
if not self._is_same_origin(response.url):
|
||||
return
|
||||
if len(self.json_responses) >= self.settings.capture.max_json_responses:
|
||||
return
|
||||
if not self._is_json(response):
|
||||
return
|
||||
key = f"{request.method}:{response.url}:{response.status}"
|
||||
if key in self._seen_resp:
|
||||
return
|
||||
self._seen_resp.add(key)
|
||||
try:
|
||||
payload = response.json()
|
||||
except Exception:
|
||||
try:
|
||||
payload = json.loads(response.text())
|
||||
except Exception:
|
||||
return
|
||||
self.json_responses.append({
|
||||
"url": response.url, "status": response.status,
|
||||
"request_method": request.method, "post_data": request.post_data,
|
||||
"resource_type": request.resource_type, "payload": payload,
|
||||
"category": self._categorize(response.url),
|
||||
})
|
||||
|
||||
@staticmethod
|
||||
def _is_json(response: Response) -> bool:
|
||||
ct = (response.headers.get("content-type") or "").lower()
|
||||
if "application/json" in ct or "+json" in ct:
|
||||
return True
|
||||
url = response.url.lower()
|
||||
return any(m in url for m in ["/api/", "/graphql", "vehicledetail", "vehicle", "auction", "bid", "images", "media"])
|
||||
|
||||
@staticmethod
|
||||
def _categorize(url: str) -> str:
|
||||
# Простая категория URL
|
||||
low = url.lower()
|
||||
mapping = {
|
||||
"images": ["image", "media", "photos", "gallery"],
|
||||
"bids": ["bid", "offer", "buy-now", "buynow"],
|
||||
"auction": ["auction", "sale", "lane", "branch"],
|
||||
"vehicle": ["vehicle", "detail", "vin", "damage", "runanddrive"],
|
||||
"documents": ["title", "document", "report"],
|
||||
}
|
||||
for cat, markers in mapping.items():
|
||||
if any(m in low for m in markers):
|
||||
return cat
|
||||
return "other"
|
||||
|
||||
def export(self) -> dict[str, Any]:
|
||||
responses = sorted(self.json_responses, key=lambda i: (i["category"] == "other", i["url"]))
|
||||
return {
|
||||
"requests": self.requests,
|
||||
"json_responses": responses,
|
||||
"capture_limits": {
|
||||
"same_origin_only": self.settings.capture.capture_same_origin_only,
|
||||
"max_requests": self.settings.capture.max_requests,
|
||||
"max_json_responses": self.settings.capture.max_json_responses,
|
||||
},
|
||||
}
|
||||
46
dubizzle_scraper/browser/pace.py
Normal file
46
dubizzle_scraper/browser/pace.py
Normal file
@@ -0,0 +1,46 @@
|
||||
import random
|
||||
import time
|
||||
|
||||
from playwright.sync_api import Locator, Page
|
||||
|
||||
from ..core.config import Settings
|
||||
|
||||
|
||||
class HumanPacer:
|
||||
# Случайные паузы между действиями.
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.settings = settings
|
||||
|
||||
def pause(self, min_s: float, max_s: float) -> None:
|
||||
if self.settings.pace.enabled:
|
||||
time.sleep(random.uniform(min_s, max_s))
|
||||
|
||||
def after_listing_open(self) -> None:
|
||||
self.pause(self.settings.pace.after_listing_open_min_s, self.settings.pace.after_listing_open_max_s)
|
||||
|
||||
def after_filter_action(self) -> None:
|
||||
self.pause(self.settings.pace.after_filter_action_min_s, self.settings.pace.after_filter_action_max_s)
|
||||
|
||||
def before_vehicle_open(self) -> None:
|
||||
self.pause(self.settings.pace.before_vehicle_open_min_s, self.settings.pace.before_vehicle_open_max_s)
|
||||
|
||||
def after_vehicle_open(self) -> None:
|
||||
self.pause(self.settings.pace.after_vehicle_open_min_s, self.settings.pace.after_vehicle_open_max_s)
|
||||
|
||||
def between_vehicles(self) -> None:
|
||||
self.pause(self.settings.pace.between_vehicles_min_s, self.settings.pace.between_vehicles_max_s)
|
||||
|
||||
def after_page_change(self) -> None:
|
||||
self.pause(self.settings.pace.after_page_change_min_s, self.settings.pace.after_page_change_max_s)
|
||||
|
||||
def move_mouse_to(self, page: Page, locator: Locator) -> None:
|
||||
try:
|
||||
box = locator.bounding_box()
|
||||
except Exception:
|
||||
box = None
|
||||
if not box:
|
||||
return
|
||||
x = box["x"] + box["width"] * random.uniform(0.2, 0.8)
|
||||
y = box["y"] + box["height"] * random.uniform(0.2, 0.8)
|
||||
page.mouse.move(x, y, steps=random.randint(8, 18))
|
||||
97
dubizzle_scraper/cli.py
Normal file
97
dubizzle_scraper/cli.py
Normal file
@@ -0,0 +1,97 @@
|
||||
import argparse
|
||||
from pathlib import Path
|
||||
|
||||
from .core.config import Settings, parse_listing_segments
|
||||
from .core.utils import save_to_json
|
||||
from .scraper import DUBIZZLEScraper
|
||||
|
||||
|
||||
def build_parser() -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(description="Dubizzle scraper CLI")
|
||||
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
|
||||
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
|
||||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||||
|
||||
default_output_dir = Path("artifacts/json")
|
||||
|
||||
subparsers.add_parser("init-db", help="Create DB tables")
|
||||
|
||||
listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
|
||||
listing_parser.add_argument("--make", default=None)
|
||||
listing_parser.add_argument("--model", default=None)
|
||||
listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_listing_links.json"))
|
||||
|
||||
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
|
||||
scrape_parser.add_argument("vehicle_url")
|
||||
scrape_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_vehicle_detail.json"))
|
||||
|
||||
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
|
||||
sync_vehicle_parser.add_argument("vehicle_url")
|
||||
sync_vehicle_parser.add_argument("--lane", default="dubizzle")
|
||||
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_vehicle.json"))
|
||||
|
||||
sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
|
||||
sync_listing_parser.add_argument("--make", default=None)
|
||||
sync_listing_parser.add_argument("--model", default=None)
|
||||
sync_listing_parser.add_argument("--lane", default="dubizzle_cars")
|
||||
sync_listing_parser.add_argument("--limit", type=int, default=None)
|
||||
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
|
||||
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_listing.json"))
|
||||
|
||||
return parser
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = build_parser()
|
||||
args = parser.parse_args()
|
||||
|
||||
runtime_settings: Settings | None = None
|
||||
if args.headless is not None or args.debug:
|
||||
runtime_settings = Settings()
|
||||
if args.headless is not None:
|
||||
runtime_settings.headless = args.headless == "true"
|
||||
if args.debug:
|
||||
runtime_settings.log_level = "DEBUG"
|
||||
|
||||
with DUBIZZLEScraper(runtime_settings) as scraper:
|
||||
if args.command == "init-db":
|
||||
data = scraper.init_db()
|
||||
print(f"DB initialized: {data}")
|
||||
return
|
||||
elif args.command == "collect-listing":
|
||||
data = scraper.collect_listing(make=args.make, model=args.model)
|
||||
elif args.command == "scrape-vehicle":
|
||||
data = scraper.scrape_vehicle_detail(args.vehicle_url)
|
||||
elif args.command == "sync-vehicle":
|
||||
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
|
||||
else:
|
||||
only_new = None if args.only_new is None else args.only_new == "true"
|
||||
segments = parse_listing_segments(scraper.settings.listing.listing_segments_json)
|
||||
use_segmented = (
|
||||
bool(segments)
|
||||
and args.make is None
|
||||
and args.model is None
|
||||
and args.limit is None
|
||||
and scraper.settings.discovery.mode in {"listing", "algolia"}
|
||||
)
|
||||
if use_segmented:
|
||||
data = scraper.sync_listing_segmented(
|
||||
segments=segments,
|
||||
lane=args.lane,
|
||||
only_new=only_new,
|
||||
)
|
||||
else:
|
||||
data = scraper.sync_listing(
|
||||
make=args.make,
|
||||
model=args.model,
|
||||
lane=args.lane,
|
||||
limit=args.limit,
|
||||
only_new=only_new,
|
||||
)
|
||||
|
||||
save_to_json(data, Path(args.output))
|
||||
print(f"Saved to {Path(args.output).resolve()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
1
dubizzle_scraper/core/__init__.py
Normal file
1
dubizzle_scraper/core/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
__all__: list[str] = []
|
||||
315
dubizzle_scraper/core/config.py
Normal file
315
dubizzle_scraper/core/config.py
Normal file
@@ -0,0 +1,315 @@
|
||||
import json
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
TRUE_VALUES = {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _resolve_env_value(name: str) -> str | None:
|
||||
"""Возвращает значение env с поддержкой legacy-префиксов DUBIZZLE_/IAAI_."""
|
||||
value = os.getenv(name)
|
||||
if value is not None:
|
||||
return value
|
||||
if name.startswith("DUBIZZLE_"):
|
||||
return os.getenv("IAAI_" + name[len("DUBIZZLE_"):])
|
||||
return None
|
||||
|
||||
|
||||
# Хелперы для чтения env-переменных с приведением типов
|
||||
|
||||
def _env_str(name: str, default: str) -> str:
|
||||
value = _resolve_env_value(name)
|
||||
return value if value is not None else default
|
||||
|
||||
|
||||
def _env_optional_str(name: str) -> str | None:
|
||||
value = _resolve_env_value(name)
|
||||
if value is None:
|
||||
return None
|
||||
value = value.strip()
|
||||
return value or None
|
||||
|
||||
|
||||
def _env_path_str(name: str) -> str | None:
|
||||
value = _env_optional_str(name)
|
||||
if value is None:
|
||||
return None
|
||||
return str(Path(value).expanduser())
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
fallback = "true" if default else "false"
|
||||
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
|
||||
|
||||
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
return int(_env_str(name, str(default)).strip())
|
||||
|
||||
|
||||
def _env_float(name: str, default: float) -> float:
|
||||
return float(_env_str(name, str(default)).strip())
|
||||
|
||||
|
||||
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
|
||||
|
||||
@dataclass(slots=True)
|
||||
class FingerprintConfig:
|
||||
user_agent: str = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/135.0.0.0 Safari/537.36"
|
||||
)
|
||||
viewport_presets: tuple[dict[str, int], ...] = (
|
||||
{"width": 1920, "height": 1080},
|
||||
{"width": 1600, "height": 900},
|
||||
{"width": 1536, "height": 864},
|
||||
{"width": 1440, "height": 900},
|
||||
{"width": 1366, "height": 768},
|
||||
)
|
||||
timezone_candidates: tuple[str, ...] = (
|
||||
"America/New_York",
|
||||
"America/Chicago",
|
||||
"America/Los_Angeles",
|
||||
)
|
||||
locale: str = "en-US"
|
||||
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
|
||||
|
||||
|
||||
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CaptureConfig:
|
||||
capture_same_origin_only: bool = _env_bool("DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY", True)
|
||||
max_requests: int = _env_int("DUBIZZLE_MAX_CAPTURED_REQUESTS", 40)
|
||||
max_json_responses: int = _env_int("DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES", 30)
|
||||
|
||||
|
||||
# Конфиг пауз между действиями (имитация человека)
|
||||
|
||||
@dataclass(slots=True)
|
||||
class HumanPaceConfig:
|
||||
enabled: bool = _env_bool("DUBIZZLE_HUMAN_PACE_ENABLED", True)
|
||||
after_listing_open_min_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MIN_S", 0.5)
|
||||
after_listing_open_max_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MAX_S", 1.2)
|
||||
after_filter_action_min_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MIN_S", 0.5)
|
||||
after_filter_action_max_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MAX_S", 1.2)
|
||||
before_vehicle_open_min_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
|
||||
before_vehicle_open_max_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
|
||||
after_vehicle_open_min_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
|
||||
after_vehicle_open_max_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
|
||||
between_vehicles_min_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MIN_S", 0.05)
|
||||
between_vehicles_max_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MAX_S", 0.15)
|
||||
after_page_change_min_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S", 0.8)
|
||||
after_page_change_max_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S", 1.8)
|
||||
|
||||
|
||||
# Конфиг сбора листинга (URL, лимиты страниц и машин)
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ListingConfig:
|
||||
cars_url: str = _env_str("DUBIZZLE_CARS_LISTING_URL", "https://dubai.dubizzle.com/motors/used-cars/")
|
||||
max_pages_per_run: int = _env_int("DUBIZZLE_MAX_PAGES_PER_RUN", 9999)
|
||||
max_vehicles_per_run: int = _env_int("DUBIZZLE_MAX_VEHICLES_PER_RUN", 50000)
|
||||
page_link_limit: int = _env_int("DUBIZZLE_PAGE_LINK_LIMIT", 500)
|
||||
include_pagination: bool = _env_bool("DUBIZZLE_INCLUDE_PAGINATION", True)
|
||||
collect_current_page_only: bool = _env_bool("DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY", False)
|
||||
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
|
||||
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
|
||||
early_stop_threshold: float = _env_float("DUBIZZLE_EARLY_STOP_THRESHOLD", 0.8)
|
||||
# Сегментация листинга по брендам для обхода лимита пагинации DUBIZZLE (~22 600 машин).
|
||||
# JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
|
||||
# Пустая строка = без сегментации (backward compatible).
|
||||
listing_segments_json: str = _env_str("DUBIZZLE_LISTING_SEGMENTS", "")
|
||||
|
||||
|
||||
# Пагинационный потолок одного Algolia-запроса: ~100 страниц × 100 = 10 000 результатов.
|
||||
DUBIZZLE_PAGINATION_CEILING = 10_000
|
||||
|
||||
# Авто-сегментация по году. Эти диапазоны подобраны так, чтобы каждый сегмент
|
||||
# оставался ниже лимита Algolia и собирался быстрее, чем старая make/year схема.
|
||||
_AUTO_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
|
||||
(1900, 2012),
|
||||
(2013, 2015),
|
||||
(2016, 2017),
|
||||
(2018, 2019),
|
||||
(2020, 2021),
|
||||
(2022, 2023),
|
||||
(2024, 2025),
|
||||
(2026, 2027),
|
||||
)
|
||||
|
||||
|
||||
def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
|
||||
"""Парсит DUBIZZLE_LISTING_SEGMENTS в список сегментов.
|
||||
|
||||
Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
|
||||
Специальное значение ``"auto"`` генерирует быстрые year-only сегменты,
|
||||
которые гарантированно проходят через Algolia без упора в лимит ~10k.
|
||||
"""
|
||||
raw = raw.strip()
|
||||
if not raw:
|
||||
return []
|
||||
if raw.lower() == "auto":
|
||||
return [
|
||||
{"make": None, "year_min": yr_min, "year_max": yr_max}
|
||||
for yr_min, yr_max in _AUTO_YEAR_SPLITS
|
||||
]
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return []
|
||||
if not isinstance(data, list):
|
||||
return []
|
||||
segments = []
|
||||
for item in data:
|
||||
if isinstance(item, str):
|
||||
segments.append({"make": item.upper(), "year_min": None, "year_max": None})
|
||||
elif isinstance(item, dict):
|
||||
segments.append({
|
||||
"make": str(item.get("make") or "").upper() or None,
|
||||
"year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
|
||||
"year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
|
||||
})
|
||||
return segments
|
||||
|
||||
|
||||
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DatabaseConfig:
|
||||
url: str = _env_str("DUBIZZLE_DATABASE_URL", "postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper")
|
||||
echo: bool = _env_bool("DUBIZZLE_DATABASE_ECHO", False)
|
||||
pool_size: int = _env_int("DUBIZZLE_DATABASE_POOL_SIZE", 5)
|
||||
max_overflow: int = _env_int("DUBIZZLE_DATABASE_MAX_OVERFLOW", 10)
|
||||
pool_recycle_seconds: int = _env_int("DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
|
||||
auto_create_tables: bool = _env_bool("DUBIZZLE_DATABASE_AUTO_CREATE_TABLES", False)
|
||||
|
||||
|
||||
# --- Конфиг Redis (URL для Celery broker) ---
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RedisConfig:
|
||||
url: str = _env_str("DUBIZZLE_REDIS_URL", "redis://localhost:6379/0")
|
||||
socket_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
|
||||
socket_connect_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
|
||||
health_check_interval_seconds: int = _env_int("DUBIZZLE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
|
||||
|
||||
|
||||
# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
|
||||
|
||||
@dataclass(slots=True)
|
||||
class DiscoveryConfig:
|
||||
mode: str = _env_str("DUBIZZLE_DISCOVERY_MODE", "algolia")
|
||||
hourly_mode: str = _env_str("DUBIZZLE_HOURLY_MODE", "rolling_refresh")
|
||||
hourly_refresh_batch_size: int = _env_int("DUBIZZLE_HOURLY_REFRESH_BATCH_SIZE", 500)
|
||||
always_full_scan: bool = _env_bool("DUBIZZLE_ALWAYS_FULL_SCAN", False)
|
||||
sitemap_fallback_on_empty: bool = _env_bool("DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY", False)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AlgoliaConfig:
|
||||
application_id: str = _env_str("DUBIZZLE_ALGOLIA_APPLICATION_ID", "WD0PTZ13ZS")
|
||||
api_key: str = _env_str(
|
||||
"DUBIZZLE_ALGOLIA_API_KEY",
|
||||
"cef139620248f1bc328a00fddc7107a6",
|
||||
)
|
||||
index_name: str = _env_str("DUBIZZLE_ALGOLIA_INDEX_NAME", "motors.com")
|
||||
category_slug: str = _env_str("DUBIZZLE_ALGOLIA_CATEGORY_SLUG", "motors/used-cars")
|
||||
base_url: str = _env_str("DUBIZZLE_ALGOLIA_BASE_URL", "https://WD0PTZ13ZS-dsn.algolia.net")
|
||||
hits_per_page: int = _env_int("DUBIZZLE_ALGOLIA_HITS_PER_PAGE", 100)
|
||||
|
||||
|
||||
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
|
||||
|
||||
@dataclass(slots=True)
|
||||
class CeleryConfig:
|
||||
broker_url: str = _env_str("CELERY_BROKER_URL", "")
|
||||
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
|
||||
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
|
||||
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
|
||||
task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
|
||||
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
|
||||
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
|
||||
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
|
||||
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
|
||||
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
|
||||
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
|
||||
parallel_tabs: int = _env_int("DUBIZZLE_PARALLEL_TABS", 8)
|
||||
parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
|
||||
block_resources: bool = _env_bool("DUBIZZLE_BLOCK_RESOURCES", True)
|
||||
|
||||
|
||||
# --- Конфиг прокси (server, username, password) ---
|
||||
|
||||
@dataclass(slots=True)
|
||||
class ProxyConfig:
|
||||
server: str | None = _env_optional_str("DUBIZZLE_PROXY_SERVER")
|
||||
username: str | None = _env_optional_str("DUBIZZLE_PROXY_USERNAME")
|
||||
password: str | None = _env_optional_str("DUBIZZLE_PROXY_PASSWORD")
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return bool(self.server)
|
||||
|
||||
def to_playwright_dict(self) -> dict[str, str] | None:
|
||||
if not self.server:
|
||||
return None
|
||||
result: dict[str, str] = {"server": self.server}
|
||||
if self.username:
|
||||
result["username"] = self.username
|
||||
if self.password:
|
||||
result["password"] = self.password
|
||||
return result
|
||||
|
||||
|
||||
# Главный объект настроек: собирает все блоки конфигурации
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Settings:
|
||||
home_url: str = "https://www.dubizzle.com/"
|
||||
default_timeout_ms: int = _env_int("DUBIZZLE_TIMEOUT_MS", 45000)
|
||||
network_settle_ms: int = _env_int("DUBIZZLE_NETWORK_SETTLE_MS", 400)
|
||||
fast_path_timeout_ms: int = _env_int("DUBIZZLE_FAST_PATH_TIMEOUT_MS", 5000)
|
||||
fast_path_max_attempts: int = _env_int("DUBIZZLE_FAST_PATH_MAX_ATTEMPTS", 1)
|
||||
fallback_navigation_timeout_ms: int = _env_int("DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS", 15000)
|
||||
max_retries: int = _env_int("DUBIZZLE_MAX_RETRIES", 3)
|
||||
retry_delay_seconds: float = _env_float("DUBIZZLE_RETRY_DELAY_SECONDS", 2.5)
|
||||
retry_backoff_multiplier: float = _env_float("DUBIZZLE_RETRY_BACKOFF_MULTIPLIER", 2.0)
|
||||
retry_jitter_seconds: float = _env_float("DUBIZZLE_RETRY_JITTER_SECONDS", 0.25)
|
||||
headless: bool = _env_bool("DUBIZZLE_HEADLESS", True)
|
||||
browser_engine: str = _env_str("DUBIZZLE_BROWSER_ENGINE", "auto")
|
||||
log_level: str = _env_str("DUBIZZLE_LOG_LEVEL", "INFO")
|
||||
log_file: str | None = _env_optional_str("DUBIZZLE_LOG_FILE")
|
||||
enable_trace_id_logs: bool = _env_bool("DUBIZZLE_ENABLE_TRACE_ID_LOGS", True)
|
||||
sync_only_new: bool = _env_bool("DUBIZZLE_SYNC_ONLY_NEW", False)
|
||||
raw_output_json: str | None = _env_optional_str("DUBIZZLE_RAW_OUTPUT_JSON")
|
||||
tokens_file: str | None = _env_path_str("DUBIZZLE_TOKENS_FILE")
|
||||
runtime_config_file: str | None = _env_path_str("DUBIZZLE_RUNTIME_CONFIG_FILE")
|
||||
scheduler_interval_minutes: int = _env_int("DUBIZZLE_SCHEDULER_INTERVAL_MINUTES", 60)
|
||||
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
|
||||
capture: CaptureConfig = field(default_factory=CaptureConfig)
|
||||
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
|
||||
listing: ListingConfig = field(default_factory=ListingConfig)
|
||||
database: DatabaseConfig = field(default_factory=DatabaseConfig)
|
||||
redis: RedisConfig = field(default_factory=RedisConfig)
|
||||
celery: CeleryConfig = field(default_factory=CeleryConfig)
|
||||
proxy: ProxyConfig = field(default_factory=ProxyConfig)
|
||||
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
|
||||
algolia: AlgoliaConfig = field(default_factory=AlgoliaConfig)
|
||||
|
||||
@property
|
||||
def parallel_tabs(self) -> int:
|
||||
return self.celery.parallel_tabs
|
||||
|
||||
@property
|
||||
def block_resources(self) -> bool:
|
||||
return self.celery.block_resources
|
||||
|
||||
# Глобальный синглтон — используется по умолчанию во всех модулях.
|
||||
settings = Settings()
|
||||
14
dubizzle_scraper/core/exceptions.py
Normal file
14
dubizzle_scraper/core/exceptions.py
Normal file
@@ -0,0 +1,14 @@
|
||||
class ScraperError(Exception):
|
||||
"""Базовое исключение скрапера."""
|
||||
|
||||
|
||||
class AntiBotDetectedError(ScraperError):
|
||||
"""Вызывается, когда сайт блокирует автоматизацию."""
|
||||
|
||||
|
||||
class SiteStructureChangedError(ScraperError):
|
||||
"""Вызывается, когда структура страницы изменилась и данных не хватает."""
|
||||
|
||||
|
||||
class ListingResumeError(ScraperError):
|
||||
"""Вызывается, когда resume по checkpoint больше недостижим."""
|
||||
39
dubizzle_scraper/core/logs.py
Normal file
39
dubizzle_scraper/core/logs.py
Normal file
@@ -0,0 +1,39 @@
|
||||
import logging
|
||||
import sys
|
||||
from contextvars import ContextVar
|
||||
|
||||
# Храним trace_id текущего потока/корутины.
|
||||
TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
|
||||
|
||||
|
||||
class TraceIdFilter(logging.Filter):
|
||||
# Добавляет trace_id в каждую запись лога для сквозной трассировки.
|
||||
def filter(self, record: logging.LogRecord) -> bool:
|
||||
record.trace_id = TRACE_ID.get()
|
||||
return True
|
||||
|
||||
|
||||
def set_trace_id(trace_id: str) -> None:
|
||||
TRACE_ID.set(trace_id)
|
||||
|
||||
|
||||
def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
|
||||
# stderr — Docker и Celery prefork корректно его подхватывают.
|
||||
handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
|
||||
if log_file:
|
||||
handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
|
||||
trace_filter = TraceIdFilter()
|
||||
for handler in handlers:
|
||||
handler.addFilter(trace_filter)
|
||||
handler.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
root = logging.getLogger()
|
||||
root.setLevel(getattr(logging, level.upper(), logging.INFO))
|
||||
# Убираем старые хендлеры, чтобы не дублировать после fork.
|
||||
root.handlers.clear()
|
||||
for handler in handlers:
|
||||
root.addHandler(handler)
|
||||
fmt = logging.Formatter(
|
||||
"%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
|
||||
)
|
||||
for handler in root.handlers:
|
||||
handler.setFormatter(fmt)
|
||||
53
dubizzle_scraper/core/retry.py
Normal file
53
dubizzle_scraper/core/retry.py
Normal file
@@ -0,0 +1,53 @@
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from functools import wraps
|
||||
from typing import Any
|
||||
|
||||
from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
|
||||
|
||||
from .exceptions import AntiBotDetectedError
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.retry")
|
||||
|
||||
# Типы исключений, при которых retry имеет смысл.
|
||||
RETRYABLE_EXCEPTIONS = (
|
||||
PlaywrightTimeoutError,
|
||||
Error,
|
||||
ConnectionError,
|
||||
OSError,
|
||||
TimeoutError,
|
||||
AntiBotDetectedError,
|
||||
)
|
||||
|
||||
|
||||
def retryable(
|
||||
max_attempts: int,
|
||||
delay_seconds: float = 2.5,
|
||||
backoff_multiplier: float = 2.0,
|
||||
jitter_seconds: float = 0.0,
|
||||
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
|
||||
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
|
||||
@wraps(func)
|
||||
def wrapper(*args: Any, **kwargs: Any) -> Any:
|
||||
last_error: Exception | None = None
|
||||
for attempt in range(1, max_attempts + 1):
|
||||
try:
|
||||
return func(*args, **kwargs)
|
||||
except RETRYABLE_EXCEPTIONS as exc:
|
||||
last_error = exc
|
||||
logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
|
||||
if attempt < max_attempts:
|
||||
sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
|
||||
if jitter_seconds > 0:
|
||||
sleep_for += random.uniform(0, jitter_seconds)
|
||||
logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
|
||||
time.sleep(sleep_for)
|
||||
if last_error is not None:
|
||||
raise last_error
|
||||
raise RuntimeError("Retry wrapper failed without a captured exception")
|
||||
|
||||
return wrapper
|
||||
|
||||
return decorator
|
||||
301
dubizzle_scraper/core/runtime_config.py
Normal file
301
dubizzle_scraper/core/runtime_config.py
Normal file
@@ -0,0 +1,301 @@
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.runtime_config")
|
||||
|
||||
|
||||
def _normalize_text(value: str) -> str:
|
||||
return value.strip().casefold()
|
||||
|
||||
|
||||
def _text_tuple(values: Any) -> tuple[str, ...]:
|
||||
return tuple(
|
||||
str(item).strip()
|
||||
for item in (values or [])
|
||||
if str(item).strip()
|
||||
)
|
||||
|
||||
|
||||
def _int_tuple(values: Any) -> tuple[int, ...]:
|
||||
result: list[int] = []
|
||||
for value in values or []:
|
||||
try:
|
||||
result.append(int(value))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
return tuple(result)
|
||||
|
||||
|
||||
def _optional_int(value: Any) -> int | None:
|
||||
if value in (None, ""):
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _optional_bool(value: Any) -> bool | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip().casefold()
|
||||
if normalized in {"1", "true", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"0", "false", "no", "off"}:
|
||||
return False
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RuntimeSyncConfig:
|
||||
name: str | None = None
|
||||
ids_initial_size: int | None = None
|
||||
ids_next_size: int | None = None
|
||||
ids_max_pages: int | None = None
|
||||
condition_check_enabled: bool | None = None
|
||||
lane: str | None = None
|
||||
only_new: bool | None = None
|
||||
limit: int | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeSyncConfig":
|
||||
data = data or {}
|
||||
name = str(data.get("name")).strip() if data.get("name") else None
|
||||
lane = str(data.get("lane")).strip() if data.get("lane") else None
|
||||
return cls(
|
||||
name=name or None,
|
||||
ids_initial_size=_optional_int(data.get("ids_initial_size")),
|
||||
ids_next_size=_optional_int(data.get("ids_next_size")),
|
||||
ids_max_pages=_optional_int(data.get("ids_max_pages")),
|
||||
condition_check_enabled=_optional_bool(data.get("condition_check_enabled")),
|
||||
lane=lane or None,
|
||||
only_new=_optional_bool(data.get("only_new")),
|
||||
limit=_optional_int(data.get("limit")),
|
||||
)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RuntimeListingConfig:
|
||||
make: str | None = None
|
||||
model: str | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeListingConfig":
|
||||
data = data or {}
|
||||
make = str(data.get("make")).strip() if data.get("make") else None
|
||||
model = str(data.get("model")).strip() if data.get("model") else None
|
||||
return cls(make=make or None, model=model or None)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RuntimeFieldFilters:
|
||||
brands: tuple[str, ...] = ()
|
||||
models: tuple[str, ...] = ()
|
||||
years: tuple[int, ...] = ()
|
||||
body_types: tuple[str, ...] = ()
|
||||
colors: tuple[str, ...] = ()
|
||||
drives: tuple[str, ...] = ()
|
||||
gearboxes: tuple[str, ...] = ()
|
||||
locations: tuple[str, ...] = ()
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFieldFilters":
|
||||
data = data or {}
|
||||
return cls(
|
||||
brands=_text_tuple(data.get("brands")),
|
||||
models=_text_tuple(data.get("models")),
|
||||
years=_int_tuple(data.get("years")),
|
||||
body_types=_text_tuple(data.get("body_types")),
|
||||
colors=_text_tuple(data.get("colors")),
|
||||
drives=_text_tuple(data.get("drives")),
|
||||
gearboxes=_text_tuple(data.get("gearboxes")),
|
||||
locations=_text_tuple(data.get("locations")),
|
||||
)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return not any([
|
||||
self.brands,
|
||||
self.models,
|
||||
self.years,
|
||||
self.body_types,
|
||||
self.colors,
|
||||
self.drives,
|
||||
self.gearboxes,
|
||||
self.locations,
|
||||
])
|
||||
|
||||
def matches(self, values: dict[str, Any]) -> bool:
|
||||
return all([
|
||||
self._match_text(self.brands, values.get("brand")),
|
||||
self._match_text(self.models, values.get("model")),
|
||||
self._match_int(self.years, values.get("year")),
|
||||
self._match_text(self.body_types, values.get("body_type")),
|
||||
self._match_text(self.colors, values.get("color")),
|
||||
self._match_text(self.drives, values.get("drive")),
|
||||
self._match_text(self.gearboxes, values.get("gearbox")),
|
||||
self._match_text(self.locations, values.get("location")),
|
||||
])
|
||||
|
||||
@staticmethod
|
||||
def _match_text(allowed: tuple[str, ...], value: Any) -> bool:
|
||||
if not allowed:
|
||||
return True
|
||||
normalized = _normalize_text(str(value or ""))
|
||||
return normalized in {_normalize_text(item) for item in allowed}
|
||||
|
||||
@staticmethod
|
||||
def _match_int(allowed: tuple[int, ...], value: Any) -> bool:
|
||||
if not allowed:
|
||||
return True
|
||||
parsed = _optional_int(value)
|
||||
return parsed in set(allowed)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RuntimeRangeFilter:
|
||||
min: int | None = None
|
||||
max: int | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeRangeFilter":
|
||||
data = data or {}
|
||||
return cls(min=_optional_int(data.get("min")), max=_optional_int(data.get("max")))
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return self.min is None and self.max is None
|
||||
|
||||
def matches(self, value: Any) -> bool:
|
||||
parsed = _optional_int(value)
|
||||
if parsed is None:
|
||||
return self.is_empty()
|
||||
if self.min is not None and parsed < self.min:
|
||||
return False
|
||||
if self.max is not None and parsed > self.max:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RuntimeFlagFilters:
|
||||
damaged_only: bool | None = None
|
||||
run_and_drive: bool | None = None
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFlagFilters":
|
||||
data = data or {}
|
||||
return cls(
|
||||
damaged_only=_optional_bool(data.get("damaged_only")),
|
||||
run_and_drive=_optional_bool(data.get("run_and_drive")),
|
||||
)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return self.damaged_only is None and self.run_and_drive is None
|
||||
|
||||
def matches(self, values: dict[str, Any]) -> bool:
|
||||
if self.damaged_only is not None and _optional_bool(values.get("is_damaged")) is not self.damaged_only:
|
||||
return False
|
||||
if self.run_and_drive is not None and _optional_bool(values.get("run_and_drive")) is not self.run_and_drive:
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RuntimeFiltersConfig:
|
||||
include: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
|
||||
exclude: RuntimeFieldFilters = field(default_factory=RuntimeFieldFilters)
|
||||
price: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
|
||||
mileage: RuntimeRangeFilter = field(default_factory=RuntimeRangeFilter)
|
||||
flags: RuntimeFlagFilters = field(default_factory=RuntimeFlagFilters)
|
||||
|
||||
@classmethod
|
||||
def from_dict(cls, data: dict[str, Any] | None) -> "RuntimeFiltersConfig":
|
||||
data = data or {}
|
||||
legacy_fields = RuntimeFieldFilters.from_dict(data)
|
||||
include_payload = data.get("include")
|
||||
exclude_payload = data.get("exclude")
|
||||
|
||||
flat_exclude_payload = {
|
||||
"brands": data.get("exclude_brands"),
|
||||
"models": data.get("exclude_models"),
|
||||
"years": data.get("exclude_years"),
|
||||
"body_types": data.get("exclude_body_types"),
|
||||
"colors": data.get("exclude_colors"),
|
||||
"drives": data.get("exclude_drives"),
|
||||
"gearboxes": data.get("exclude_gearboxes"),
|
||||
"locations": data.get("exclude_locations"),
|
||||
}
|
||||
|
||||
include = RuntimeFieldFilters.from_dict(include_payload) if include_payload is not None else legacy_fields
|
||||
exclude = (
|
||||
RuntimeFieldFilters.from_dict(exclude_payload)
|
||||
if exclude_payload is not None
|
||||
else RuntimeFieldFilters.from_dict(flat_exclude_payload)
|
||||
)
|
||||
|
||||
return cls(
|
||||
include=include,
|
||||
exclude=exclude,
|
||||
price=RuntimeRangeFilter.from_dict(data.get("price")),
|
||||
mileage=RuntimeRangeFilter.from_dict(data.get("mileage")),
|
||||
flags=RuntimeFlagFilters.from_dict(data.get("flags")),
|
||||
)
|
||||
|
||||
def is_empty(self) -> bool:
|
||||
return all([
|
||||
self.include.is_empty(),
|
||||
self.exclude.is_empty(),
|
||||
self.price.is_empty(),
|
||||
self.mileage.is_empty(),
|
||||
self.flags.is_empty(),
|
||||
])
|
||||
|
||||
def matches(self, values: dict[str, Any]) -> bool:
|
||||
if not self.include.matches(values):
|
||||
return False
|
||||
if not self._matches_exclude(values):
|
||||
return False
|
||||
if not self.price.matches(values.get("price")):
|
||||
return False
|
||||
if not self.mileage.matches(values.get("mileage")):
|
||||
return False
|
||||
if not self.flags.matches(values):
|
||||
return False
|
||||
return True
|
||||
|
||||
def _matches_exclude(self, values: dict[str, Any]) -> bool:
|
||||
if self.exclude.is_empty():
|
||||
return True
|
||||
return not self.exclude.matches(values)
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class RuntimeConfig:
|
||||
sync: RuntimeSyncConfig = field(default_factory=RuntimeSyncConfig)
|
||||
listing: RuntimeListingConfig = field(default_factory=RuntimeListingConfig)
|
||||
filters: RuntimeFiltersConfig = field(default_factory=RuntimeFiltersConfig)
|
||||
|
||||
@classmethod
|
||||
def from_file(cls, config_path: str | None) -> "RuntimeConfig":
|
||||
if not config_path:
|
||||
return cls()
|
||||
path = Path(config_path)
|
||||
if not path.exists():
|
||||
logger.info("Runtime config file not found: %s", path)
|
||||
return cls()
|
||||
try:
|
||||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to read runtime config %s: %s", path, exc)
|
||||
return cls()
|
||||
return cls(
|
||||
sync=RuntimeSyncConfig.from_dict(payload.get("sync")),
|
||||
listing=RuntimeListingConfig.from_dict(payload.get("listing")),
|
||||
filters=RuntimeFiltersConfig.from_dict(payload.get("filters")),
|
||||
)
|
||||
72
dubizzle_scraper/core/utils.py
Normal file
72
dubizzle_scraper/core/utils.py
Normal file
@@ -0,0 +1,72 @@
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import Any, Callable, Iterable
|
||||
|
||||
|
||||
def save_to_json(data: Any, filename: str | Path) -> None:
|
||||
path = Path(filename)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
|
||||
|
||||
def first_non_empty(values: Iterable[Any]) -> Any | None:
|
||||
for value in values:
|
||||
if value not in (None, "", [], {}, ()):
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
# Регулярные выражения для VIN, lot и price.
|
||||
VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
|
||||
LOT_RE = re.compile(r"\b(\d{7,10})\b")
|
||||
PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
|
||||
|
||||
|
||||
def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
|
||||
# Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
|
||||
found = []
|
||||
if _depth >= max_depth:
|
||||
return found
|
||||
if isinstance(obj, dict):
|
||||
for key, value in obj.items():
|
||||
if key.lower() in target_keys:
|
||||
found.append(value)
|
||||
found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
|
||||
return found
|
||||
|
||||
|
||||
def deep_find_all_keys(
|
||||
payloads: list,
|
||||
field_map: dict[str, set[str]],
|
||||
max_depth: int = 64,
|
||||
) -> dict[str, list]:
|
||||
"""Извлекает все нужные поля за один проход по JSON."""
|
||||
# Готовим обратную карту: нормализованный ключ -> имя поля.
|
||||
reverse: dict[str, str] = {}
|
||||
for field_name, keys in field_map.items():
|
||||
for k in keys:
|
||||
reverse[k.lower()] = field_name
|
||||
|
||||
result: dict[str, list] = {f: [] for f in field_map}
|
||||
|
||||
def _recurse(obj: Any, depth: int) -> None:
|
||||
if depth >= max_depth:
|
||||
return
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
field = reverse.get(k.lower())
|
||||
if field is not None:
|
||||
result[field].append(v)
|
||||
_recurse(v, depth + 1)
|
||||
elif isinstance(obj, list):
|
||||
for item in obj:
|
||||
_recurse(item, depth + 1)
|
||||
|
||||
for payload in payloads:
|
||||
_recurse(payload, 0)
|
||||
|
||||
return result
|
||||
23
dubizzle_scraper/discovery/__init__.py
Normal file
23
dubizzle_scraper/discovery/__init__.py
Normal file
@@ -0,0 +1,23 @@
|
||||
from .sitemap import (
|
||||
SitemapDiscoveryError,
|
||||
SitemapDiscoveryResult,
|
||||
SitemapDiscoveryStats,
|
||||
discover_vehicle_urls_from_sitemap,
|
||||
discover_vehicle_urls_from_sitemap_with_stats,
|
||||
)
|
||||
from .algolia import (
|
||||
AlgoliaDiscoveryError,
|
||||
AlgoliaDiscoveryResult,
|
||||
discover_vehicle_hits_from_algolia,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"SitemapDiscoveryError",
|
||||
"SitemapDiscoveryResult",
|
||||
"SitemapDiscoveryStats",
|
||||
"discover_vehicle_urls_from_sitemap",
|
||||
"discover_vehicle_urls_from_sitemap_with_stats",
|
||||
"AlgoliaDiscoveryError",
|
||||
"AlgoliaDiscoveryResult",
|
||||
"discover_vehicle_hits_from_algolia",
|
||||
]
|
||||
479
dubizzle_scraper/discovery/algolia.py
Normal file
479
dubizzle_scraper/discovery/algolia.py
Normal file
@@ -0,0 +1,479 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
from urllib.parse import parse_qs, urlencode, urlparse
|
||||
from urllib.request import Request, urlopen
|
||||
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.discovery.algolia")
|
||||
|
||||
ALGOLIA_HARD_PAGE_LIMIT = 100
|
||||
ALGOLIA_MAX_HITS_PER_QUERY = ALGOLIA_HARD_PAGE_LIMIT * 100
|
||||
ALGOLIA_ID_RANGE_START = 0
|
||||
ALGOLIA_ID_RANGE_END = 20_000_000
|
||||
ALGOLIA_ID_RANGE_MIN_WINDOW = 100
|
||||
|
||||
|
||||
class AlgoliaDiscoveryError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AlgoliaDiscoveryPageStat:
|
||||
page_number: int
|
||||
hits_count: int
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class AlgoliaDiscoveryResult:
|
||||
vehicle_urls: list[str] = field(default_factory=list)
|
||||
hit_records: dict[str, dict[str, Any]] = field(default_factory=dict)
|
||||
origin_ids_by_url: dict[str, str] = field(default_factory=dict)
|
||||
pages: list[dict[str, int]] = field(default_factory=list)
|
||||
early_stopped: bool = False
|
||||
truncated_by_time_budget: bool = False
|
||||
total_hits: int = 0
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _AlgoliaShard:
|
||||
category_slug: str
|
||||
id_min: int | None = None
|
||||
id_max: int | None = None
|
||||
year_min: int | None = None
|
||||
year_max: int | None = None
|
||||
|
||||
def filter_expr(self) -> str:
|
||||
parts = [f"(category_v2.slug_paths:{self.category_slug})"]
|
||||
if self.year_min is not None:
|
||||
parts.append(f"year>={int(self.year_min)}")
|
||||
if self.year_max is not None:
|
||||
parts.append(f"year<={int(self.year_max)}")
|
||||
if self.id_min is not None:
|
||||
parts.append(f"id>={int(self.id_min)}")
|
||||
if self.id_max is not None:
|
||||
parts.append(f"id<={int(self.id_max)}")
|
||||
return " AND ".join(parts)
|
||||
|
||||
def label(self) -> str:
|
||||
label = self.category_slug
|
||||
if self.year_min is not None or self.year_max is not None:
|
||||
label += f"[year:{self.year_min}-{self.year_max}]"
|
||||
if self.id_min is None and self.id_max is None:
|
||||
return label
|
||||
return f"{label}[id:{self.id_min}-{self.id_max}]"
|
||||
|
||||
|
||||
@dataclass(slots=True)
|
||||
class _AlgoliaHttpClient:
|
||||
endpoint: str
|
||||
app_id: str
|
||||
api_key: str
|
||||
user_agent: str
|
||||
index_name: str
|
||||
hits_per_page: int
|
||||
|
||||
def request(self, *, page: int, filters: str) -> dict[str, Any]:
|
||||
params = urlencode(
|
||||
{
|
||||
"query": "",
|
||||
"page": page,
|
||||
"hitsPerPage": self.hits_per_page,
|
||||
"filters": filters,
|
||||
}
|
||||
)
|
||||
payload = {"requests": [{"indexName": self.index_name, "params": params}]}
|
||||
request = Request(
|
||||
self.endpoint,
|
||||
data=json.dumps(payload).encode("utf-8"),
|
||||
headers={
|
||||
"content-type": "application/json",
|
||||
"x-algolia-application-id": self.app_id,
|
||||
"x-algolia-api-key": self.api_key,
|
||||
"accept": "application/json",
|
||||
"user-agent": self.user_agent,
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urlopen(request, timeout=30) as response:
|
||||
body = response.read().decode("utf-8", errors="ignore")
|
||||
parsed = json.loads(body)
|
||||
results = parsed.get("results") or []
|
||||
return results[0] if results and isinstance(results[0], dict) else {}
|
||||
|
||||
|
||||
def _build_origin_id_from_hit(hit: dict[str, Any]) -> str | None:
|
||||
for key in ("id", "objectID", "uuid"):
|
||||
value = hit.get(key)
|
||||
if value is None:
|
||||
continue
|
||||
text = str(value).strip()
|
||||
if text:
|
||||
return f"dubizzle:{text}"
|
||||
permalink = str(hit.get("permalink") or "").strip()
|
||||
if permalink:
|
||||
tail = permalink.rstrip("/").split("/")[-1]
|
||||
if tail:
|
||||
return f"dubizzle:{tail}"
|
||||
return None
|
||||
|
||||
|
||||
def _build_vehicle_url_from_hit(hit: dict[str, Any]) -> str | None:
|
||||
permalink = str(hit.get("permalink") or "").strip()
|
||||
if permalink:
|
||||
if permalink.startswith("http://") or permalink.startswith("https://"):
|
||||
return permalink
|
||||
if permalink.startswith("/"):
|
||||
return f"https://www.dubizzle.com{permalink}"
|
||||
return f"https://www.dubizzle.com/{permalink.lstrip('/')}"
|
||||
|
||||
short = str(hit.get("short_url") or "").strip()
|
||||
if short:
|
||||
if short.startswith("http://") or short.startswith("https://"):
|
||||
return short
|
||||
return f"https://dubizzle.com/s/{short.strip('/')}"
|
||||
|
||||
hit_id = hit.get("id") or hit.get("objectID")
|
||||
if hit_id is not None:
|
||||
return f"https://www.dubizzle.com/motors/used-cars/ad-{hit_id}/"
|
||||
return None
|
||||
|
||||
|
||||
def _extract_make_from_listing_url(listing_url: str | None) -> str | None:
|
||||
if not listing_url:
|
||||
return None
|
||||
try:
|
||||
parsed = urlparse(listing_url)
|
||||
params = parse_qs(parsed.query)
|
||||
candidate = (params.get("Make") or params.get("make") or [None])[0]
|
||||
if candidate:
|
||||
return str(candidate).strip()
|
||||
parts = [p for p in parsed.path.split("/") if p]
|
||||
if len(parts) >= 3 and parts[0].lower() == "motors" and parts[1].lower() == "used-cars":
|
||||
slug = parts[2].strip().lower()
|
||||
if slug and slug != "s":
|
||||
return slug.replace("-", " ")
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def _make_slug(make: str | None) -> str | None:
|
||||
if not make:
|
||||
return None
|
||||
slug = make.strip().lower().replace(" ", "-")
|
||||
return slug or None
|
||||
|
||||
|
||||
def _hit_matches_filters(
|
||||
hit: dict[str, Any],
|
||||
*,
|
||||
make: str | None,
|
||||
model: str | None,
|
||||
year_min: int | None,
|
||||
year_max: int | None,
|
||||
) -> bool:
|
||||
if make:
|
||||
hit_make = str(hit.get("make") or "").strip().lower()
|
||||
if hit_make != make.strip().lower():
|
||||
return False
|
||||
if model:
|
||||
hit_model = str(hit.get("model") or "").strip().lower()
|
||||
if hit_model != model.strip().lower():
|
||||
return False
|
||||
|
||||
hit_year_raw = hit.get("year")
|
||||
hit_year: int | None = None
|
||||
if hit_year_raw is not None:
|
||||
try:
|
||||
hit_year = int(hit_year_raw)
|
||||
except Exception:
|
||||
hit_year = None
|
||||
|
||||
if year_min is not None and (hit_year is None or hit_year < year_min):
|
||||
return False
|
||||
if year_max is not None and (hit_year is None or hit_year > year_max):
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def _probe_total_hits(client: _AlgoliaHttpClient, shard: _AlgoliaShard) -> int:
|
||||
first = client.request(page=0, filters=shard.filter_expr())
|
||||
return int(first.get("nbHits") or 0)
|
||||
|
||||
|
||||
def _split_id_range(shard: _AlgoliaShard) -> tuple[_AlgoliaShard, _AlgoliaShard] | None:
|
||||
lo = shard.id_min if shard.id_min is not None else ALGOLIA_ID_RANGE_START
|
||||
hi = shard.id_max if shard.id_max is not None else ALGOLIA_ID_RANGE_END
|
||||
if hi - lo <= ALGOLIA_ID_RANGE_MIN_WINDOW:
|
||||
return None
|
||||
mid = (lo + hi) // 2
|
||||
return (
|
||||
_AlgoliaShard(
|
||||
category_slug=shard.category_slug,
|
||||
id_min=lo,
|
||||
id_max=mid,
|
||||
year_min=shard.year_min,
|
||||
year_max=shard.year_max,
|
||||
),
|
||||
_AlgoliaShard(
|
||||
category_slug=shard.category_slug,
|
||||
id_min=mid + 1,
|
||||
id_max=hi,
|
||||
year_min=shard.year_min,
|
||||
year_max=shard.year_max,
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _expand_shards(client: _AlgoliaHttpClient, initial_shard: _AlgoliaShard, max_duration_seconds: float | None, started_at: float) -> tuple[list[_AlgoliaShard], bool, int]:
|
||||
queue: list[_AlgoliaShard] = [initial_shard]
|
||||
ready: list[_AlgoliaShard] = []
|
||||
truncated = False
|
||||
total_hits = 0
|
||||
|
||||
while queue:
|
||||
if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
|
||||
truncated = True
|
||||
break
|
||||
shard = queue.pop(0)
|
||||
shard_total = _probe_total_hits(client, shard)
|
||||
if shard is initial_shard:
|
||||
total_hits = shard_total
|
||||
if shard_total == 0:
|
||||
continue
|
||||
if shard_total <= ALGOLIA_MAX_HITS_PER_QUERY:
|
||||
ready.append(shard)
|
||||
continue
|
||||
split = _split_id_range(shard)
|
||||
if split is None:
|
||||
logger.warning(
|
||||
"Shard %s still exceeds limit=%d but id-window is too small to split further (hits=%d)",
|
||||
shard.label(),
|
||||
ALGOLIA_MAX_HITS_PER_QUERY,
|
||||
shard_total,
|
||||
)
|
||||
ready.append(shard)
|
||||
continue
|
||||
logger.warning(
|
||||
"Splitting Algolia shard %s with hits=%d into %s and %s",
|
||||
shard.label(),
|
||||
shard_total,
|
||||
split[0].label(),
|
||||
split[1].label(),
|
||||
)
|
||||
queue.insert(0, split[1])
|
||||
queue.insert(0, split[0])
|
||||
|
||||
return ready, truncated, total_hits
|
||||
|
||||
|
||||
def _fetch_shard_hits(
|
||||
*,
|
||||
client: _AlgoliaHttpClient,
|
||||
shard: _AlgoliaShard,
|
||||
make: str | None,
|
||||
model: str | None,
|
||||
year_min: int | None,
|
||||
year_max: int | None,
|
||||
known_origin_ids: set[str] | None,
|
||||
threshold: float,
|
||||
max_duration_seconds: float | None,
|
||||
started_at: float,
|
||||
) -> tuple[list[str], dict[str, dict[str, Any]], dict[str, str], list[dict[str, int]], bool, bool]:
|
||||
urls: list[str] = []
|
||||
hit_records: dict[str, dict[str, Any]] = {}
|
||||
origin_ids_by_url: dict[str, str] = {}
|
||||
pages: list[dict[str, int]] = []
|
||||
early_stopped = False
|
||||
truncated_by_time_budget = False
|
||||
page = 0
|
||||
nb_pages = None
|
||||
|
||||
while True:
|
||||
if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
|
||||
truncated_by_time_budget = True
|
||||
break
|
||||
|
||||
try:
|
||||
first = client.request(page=page, filters=shard.filter_expr())
|
||||
except Exception as exc:
|
||||
raise AlgoliaDiscoveryError(
|
||||
f"Algolia request failed for shard={shard.label()} page={page}: {exc}"
|
||||
) from exc
|
||||
|
||||
hits = first.get("hits") or []
|
||||
if not isinstance(hits, list):
|
||||
hits = []
|
||||
|
||||
if page == 0:
|
||||
nb_pages = min(int(first.get("nbPages") or 0), ALGOLIA_HARD_PAGE_LIMIT)
|
||||
|
||||
pages.append({"page_number": page + 1, "links_found": len(hits), "shard": shard.label()})
|
||||
if not hits:
|
||||
break
|
||||
|
||||
page_known = 0
|
||||
page_new = 0
|
||||
for raw_hit in hits:
|
||||
if not isinstance(raw_hit, dict):
|
||||
continue
|
||||
if not _hit_matches_filters(
|
||||
raw_hit,
|
||||
make=make,
|
||||
model=model,
|
||||
year_min=year_min,
|
||||
year_max=year_max,
|
||||
):
|
||||
continue
|
||||
|
||||
url = _build_vehicle_url_from_hit(raw_hit)
|
||||
if not url:
|
||||
continue
|
||||
origin_id = _build_origin_id_from_hit(raw_hit)
|
||||
if origin_id and known_origin_ids is not None and origin_id in known_origin_ids:
|
||||
page_known += 1
|
||||
else:
|
||||
page_new += 1
|
||||
|
||||
if url in hit_records:
|
||||
continue
|
||||
urls.append(url)
|
||||
hit_records[url] = raw_hit
|
||||
if origin_id:
|
||||
origin_ids_by_url[url] = origin_id
|
||||
|
||||
if known_origin_ids is not None and threshold > 0 and (page_known + page_new) > 0:
|
||||
ratio = page_known / (page_known + page_new)
|
||||
if ratio >= threshold and page_new == 0:
|
||||
early_stopped = True
|
||||
break
|
||||
|
||||
page += 1
|
||||
if nb_pages is not None and page >= nb_pages:
|
||||
break
|
||||
|
||||
return urls, hit_records, origin_ids_by_url, pages, early_stopped, truncated_by_time_budget
|
||||
|
||||
|
||||
def discover_vehicle_hits_from_algolia(
|
||||
*,
|
||||
settings,
|
||||
make: str | None = None,
|
||||
model: str | None = None,
|
||||
limit: int | None = None,
|
||||
year_min: int | None = None,
|
||||
year_max: int | None = None,
|
||||
listing_url: str | None = None,
|
||||
known_origin_ids: set[str] | None = None,
|
||||
max_duration_seconds: float | None = None,
|
||||
) -> AlgoliaDiscoveryResult:
|
||||
app_id = settings.algolia.application_id.strip()
|
||||
api_key = settings.algolia.api_key.strip()
|
||||
index_name = settings.algolia.index_name.strip()
|
||||
if not app_id or not api_key or not index_name:
|
||||
raise AlgoliaDiscoveryError("Algolia credentials/index are not configured")
|
||||
|
||||
effective_make = make or _extract_make_from_listing_url(listing_url)
|
||||
hits_per_page = max(1, min(100, int(settings.algolia.hits_per_page)))
|
||||
base_url = settings.algolia.base_url.strip().rstrip("/")
|
||||
if not base_url:
|
||||
base_url = f"https://{app_id}-dsn.algolia.net"
|
||||
|
||||
category_slug = settings.algolia.category_slug.strip() or "motors/used-cars"
|
||||
make_slug = _make_slug(effective_make)
|
||||
if make_slug:
|
||||
category_slug = f"{category_slug}/{make_slug}"
|
||||
|
||||
client = _AlgoliaHttpClient(
|
||||
endpoint=f"{base_url}/1/indexes/*/queries",
|
||||
app_id=app_id,
|
||||
api_key=api_key,
|
||||
user_agent=settings.fingerprint.user_agent,
|
||||
index_name=index_name,
|
||||
hits_per_page=hits_per_page,
|
||||
)
|
||||
threshold = float(settings.listing.early_stop_threshold)
|
||||
started_at = time.perf_counter()
|
||||
|
||||
initial_shard = _AlgoliaShard(
|
||||
category_slug=category_slug,
|
||||
id_min=ALGOLIA_ID_RANGE_START,
|
||||
id_max=ALGOLIA_ID_RANGE_END,
|
||||
year_min=year_min,
|
||||
year_max=year_max,
|
||||
)
|
||||
shards, shard_build_truncated, total_hits = _expand_shards(
|
||||
client,
|
||||
initial_shard,
|
||||
max_duration_seconds,
|
||||
started_at,
|
||||
)
|
||||
|
||||
collected_urls: list[str] = []
|
||||
hits_by_url: dict[str, dict[str, Any]] = {}
|
||||
origin_ids_by_url: dict[str, str] = {}
|
||||
pages: list[dict[str, int]] = []
|
||||
early_stopped = False
|
||||
truncated_by_time_budget = shard_build_truncated
|
||||
|
||||
logger.warning(
|
||||
"Algolia shard plan ready: total_hits=%d shards=%d category=%s",
|
||||
total_hits,
|
||||
len(shards),
|
||||
category_slug,
|
||||
)
|
||||
|
||||
for shard in shards:
|
||||
shard_urls, shard_hits, shard_origin_ids, shard_pages, shard_early_stop, shard_truncated = _fetch_shard_hits(
|
||||
client=client,
|
||||
shard=shard,
|
||||
make=effective_make,
|
||||
model=model,
|
||||
year_min=year_min,
|
||||
year_max=year_max,
|
||||
known_origin_ids=known_origin_ids,
|
||||
threshold=threshold,
|
||||
max_duration_seconds=max_duration_seconds,
|
||||
started_at=started_at,
|
||||
)
|
||||
pages.extend(shard_pages)
|
||||
early_stopped = early_stopped or shard_early_stop
|
||||
truncated_by_time_budget = truncated_by_time_budget or shard_truncated
|
||||
for url in shard_urls:
|
||||
if url in hits_by_url:
|
||||
continue
|
||||
collected_urls.append(url)
|
||||
hits_by_url[url] = shard_hits[url]
|
||||
if url in shard_origin_ids:
|
||||
origin_ids_by_url[url] = shard_origin_ids[url]
|
||||
if limit is not None and limit > 0 and len(collected_urls) >= limit:
|
||||
break
|
||||
if truncated_by_time_budget:
|
||||
break
|
||||
|
||||
logger.info(
|
||||
"Algolia discovery done: urls=%d total_hits=%d pages=%d shards=%d",
|
||||
len(collected_urls),
|
||||
total_hits,
|
||||
len(pages),
|
||||
len(shards),
|
||||
)
|
||||
|
||||
if limit is not None and limit > 0 and len(collected_urls) > limit:
|
||||
collected_urls = collected_urls[:limit]
|
||||
|
||||
return AlgoliaDiscoveryResult(
|
||||
vehicle_urls=collected_urls,
|
||||
hit_records={url: hits_by_url[url] for url in collected_urls if url in hits_by_url},
|
||||
origin_ids_by_url={url: origin_ids_by_url[url] for url in collected_urls if url in origin_ids_by_url},
|
||||
pages=pages,
|
||||
early_stopped=early_stopped,
|
||||
truncated_by_time_budget=truncated_by_time_budget,
|
||||
total_hits=total_hits,
|
||||
)
|
||||
210
dubizzle_scraper/discovery/sitemap.py
Normal file
210
dubizzle_scraper/discovery/sitemap.py
Normal file
@@ -0,0 +1,210 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import gzip
|
||||
import io
|
||||
import logging
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Iterable
|
||||
from urllib.parse import urlsplit, urlunsplit
|
||||
from urllib.request import Request, urlopen, ProxyHandler, build_opener
|
||||
import xml.etree.ElementTree as ET
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.discovery.sitemap")
|
||||
|
||||
DEFAULT_SITEMAP_INDEX_URL = "https://www.dubizzle.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
|
||||
_SITEMAP_TIMEOUT_SECONDS = 30
|
||||
_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
|
||||
|
||||
|
||||
class SitemapDiscoveryError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def _is_vehicle_sitemap_url(url: str) -> bool:
|
||||
lowered = url.strip().lower()
|
||||
# Берём только sitemap с авто.
|
||||
if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
|
||||
return False
|
||||
return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
|
||||
|
||||
|
||||
def _normalize_vehicle_url(url: str) -> str:
|
||||
parts = urlsplit(url.strip())
|
||||
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
|
||||
|
||||
|
||||
def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
|
||||
request = Request(
|
||||
url,
|
||||
headers={
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
|
||||
"Accept-Encoding": "gzip",
|
||||
},
|
||||
)
|
||||
if proxy_url:
|
||||
handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
|
||||
opener = build_opener(handler)
|
||||
response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
|
||||
else:
|
||||
response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
|
||||
with response:
|
||||
payload = response.read()
|
||||
encoding = str(response.headers.get("Content-Encoding") or "").lower()
|
||||
if encoding == "gzip" or url.lower().endswith(".gz"):
|
||||
return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
|
||||
return payload
|
||||
|
||||
|
||||
def _local_name(tag: str) -> str:
|
||||
if "}" in tag:
|
||||
return tag.rsplit("}", 1)[1]
|
||||
return tag
|
||||
|
||||
|
||||
def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
|
||||
for match in _LOC_TAG_RE.finditer(xml_bytes):
|
||||
try:
|
||||
value = match.group(1).decode("utf-8", errors="ignore").strip()
|
||||
except Exception:
|
||||
continue
|
||||
if value:
|
||||
yield value
|
||||
|
||||
|
||||
def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
|
||||
try:
|
||||
root = ET.fromstring(xml_bytes)
|
||||
except ET.ParseError as exc:
|
||||
fallback_values = list(_iter_loc_values_fallback(xml_bytes))
|
||||
if fallback_values:
|
||||
logger.warning(
|
||||
"Falling back to regex sitemap loc extraction after XML parse error: %s",
|
||||
exc,
|
||||
)
|
||||
yield from fallback_values
|
||||
return
|
||||
raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
|
||||
|
||||
for element in root.iter():
|
||||
if _local_name(element.tag) != "loc":
|
||||
continue
|
||||
if not element.text:
|
||||
continue
|
||||
value = element.text.strip()
|
||||
if value:
|
||||
yield value
|
||||
|
||||
|
||||
def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
|
||||
result: list[str] = []
|
||||
seen: set[str] = set()
|
||||
for url in urls:
|
||||
normalized = _normalize_vehicle_url(url)
|
||||
if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
|
||||
continue
|
||||
if normalized in seen:
|
||||
continue
|
||||
seen.add(normalized)
|
||||
result.append(normalized)
|
||||
return result
|
||||
|
||||
|
||||
def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
|
||||
return any("/vehicledetail/" in url.lower() for url in urls)
|
||||
|
||||
|
||||
def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
|
||||
logger.info("Downloading sitemap index: %s", index_url)
|
||||
index_xml = _download_bytes(index_url, proxy_url=proxy_url)
|
||||
sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
|
||||
if not sitemap_urls:
|
||||
raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
|
||||
|
||||
all_vehicle_urls: list[str] = []
|
||||
for sitemap_url in sitemap_urls:
|
||||
logger.info("Downloading sitemap: %s", sitemap_url)
|
||||
try:
|
||||
sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
|
||||
raw_urls = list(_iter_loc_values(sitemap_xml))
|
||||
except SitemapDiscoveryError as exc:
|
||||
logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
|
||||
continue
|
||||
|
||||
vehicle_urls = _filter_vehicle_urls(raw_urls)
|
||||
if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
|
||||
logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
|
||||
continue
|
||||
logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
|
||||
all_vehicle_urls.extend(vehicle_urls)
|
||||
|
||||
deduped = _filter_vehicle_urls(all_vehicle_urls)
|
||||
if not deduped:
|
||||
raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
|
||||
logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
|
||||
return deduped
|
||||
|
||||
|
||||
@dataclass
|
||||
class SitemapDiscoveryStats:
|
||||
transport: str = "http"
|
||||
fetched_sitemaps: int = 0
|
||||
blocked_sitemaps: int = 0
|
||||
malformed_sitemaps: int = 0
|
||||
direct_probe_hits: int = 0
|
||||
direct_probe_misses: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class SitemapDiscoveryResult:
|
||||
vehicle_urls: list[str] = field(default_factory=list)
|
||||
stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
|
||||
|
||||
|
||||
def discover_vehicle_urls_from_sitemap_with_stats(
|
||||
settings=None,
|
||||
index_url: str = DEFAULT_SITEMAP_INDEX_URL,
|
||||
) -> SitemapDiscoveryResult:
|
||||
"""Возвращает URL и статистику."""
|
||||
proxy_url = None
|
||||
if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
|
||||
proxy_url = settings.proxy.server
|
||||
stats = SitemapDiscoveryStats(transport="http")
|
||||
logger.info("Downloading sitemap index: %s", index_url)
|
||||
index_xml = _download_bytes(index_url, proxy_url=proxy_url)
|
||||
sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
|
||||
if not sitemap_urls:
|
||||
raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
|
||||
|
||||
all_vehicle_urls: list[str] = []
|
||||
for sitemap_url in sitemap_urls:
|
||||
logger.info("Downloading sitemap: %s", sitemap_url)
|
||||
try:
|
||||
sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
|
||||
raw_urls = list(_iter_loc_values(sitemap_xml))
|
||||
stats.fetched_sitemaps += 1
|
||||
except SitemapDiscoveryError as exc:
|
||||
logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
|
||||
stats.malformed_sitemaps += 1
|
||||
continue
|
||||
except Exception as exc:
|
||||
logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
|
||||
stats.blocked_sitemaps += 1
|
||||
continue
|
||||
|
||||
vehicle_urls = _filter_vehicle_urls(raw_urls)
|
||||
if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
|
||||
logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
|
||||
continue
|
||||
logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
|
||||
all_vehicle_urls.extend(vehicle_urls)
|
||||
|
||||
deduped = _filter_vehicle_urls(all_vehicle_urls)
|
||||
if not deduped:
|
||||
raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
|
||||
logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
|
||||
return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)
|
||||
1
dubizzle_scraper/parsing/__init__.py
Normal file
1
dubizzle_scraper/parsing/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
__all__: list[str] = []
|
||||
622
dubizzle_scraper/parsing/mapper.py
Normal file
622
dubizzle_scraper/parsing/mapper.py
Normal file
@@ -0,0 +1,622 @@
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from string import ascii_letters, digits
|
||||
from typing import Any
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from ..core.utils import first_non_empty
|
||||
from ..storage.enums import (
|
||||
BODY_TYPE_ENUM_VALUES,
|
||||
COUNTRY_ENUM_VALUES,
|
||||
CURRENCY_ENUM_VALUES,
|
||||
DRIVE_ENUM_VALUES,
|
||||
GEARBOX_ENUM_VALUES,
|
||||
ORIGIN_ENUM_VALUES,
|
||||
SELLING_TYPE_ENUM_VALUES,
|
||||
STEERING_WHEEL_ENUM_VALUES,
|
||||
)
|
||||
from ..storage.schemas import CarRecord, ImageRecord
|
||||
|
||||
|
||||
class CarMapper:
|
||||
# Маппер DUBIZZLE в CarRecord.
|
||||
|
||||
BODY_MAP = {
|
||||
"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
|
||||
"suv": "SUV", "wagon": "STATION_WAGON", "station wagon": "STATION_WAGON", "pickup": "PICKUP",
|
||||
"pickup truck": "PICKUP", "crew cab": "PICKUP", "extended cab": "PICKUP", "regular cab": "PICKUP",
|
||||
"quad cab": "PICKUP", "double cab": "PICKUP", "king cab": "PICKUP", "mega cab": "PICKUP",
|
||||
"supercab": "PICKUP", "supercrew": "PICKUP", "truck": "TRUCK", "van": "MINIVAN",
|
||||
"minivan": "MINIVAN", "convertible": "OPEN", "cabriolet": "OPEN", "rv": "RV", "crossover": "SUV",
|
||||
}
|
||||
DRIVE_MAP = {
|
||||
"front wheel drive": "FWD", "fwd": "FWD", "rear wheel drive": "RWD", "rwd": "RWD",
|
||||
"all wheel drive": "4WD", "awd": "4WD", "4x4": "4WD", "four wheel drive": "4WD",
|
||||
"4wd": "4WD", "2wd": "2WD", "two wheel drive": "2WD",
|
||||
}
|
||||
GEARBOX_MAP = {
|
||||
"automatic": "AT", "automatic transmission": "AT", "a/t": "AT", "aut": "AT",
|
||||
"manual": "MT", "manual transmission": "MT", "m/t": "MT", "cvt": "CVT",
|
||||
"continuously variable transmission": "CVT", "electric": "EV", "ev": "EV",
|
||||
}
|
||||
STEERING_MAP = {"left": "LEFT", "left hand drive": "LEFT", "right": "RIGHT", "right hand drive": "RIGHT"}
|
||||
COUNTRY_MAP = {
|
||||
"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
|
||||
"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
|
||||
"ae": "AE", "uae": "AE", "united arab emirates": "AE", "dubai": "AE", "abu dhabi": "AE",
|
||||
}
|
||||
NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
|
||||
|
||||
def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
|
||||
# Собираем DB-модель.
|
||||
vehicle_summary = vehicle_summary or {}
|
||||
payload_insights = payload_insights or {}
|
||||
core = payload_insights.get("vehicle_core", {})
|
||||
pricing = payload_insights.get("pricing", {})
|
||||
damage = payload_insights.get("damage", {})
|
||||
auction = payload_insights.get("auction", {})
|
||||
images = payload_insights.get("images", {})
|
||||
|
||||
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
|
||||
parser_id = self._generate_parser_id(origin_id)
|
||||
brand = self._resolve_make(core, vehicle_summary) or "UNKNOWN"
|
||||
model = self._resolve_model(core, vehicle_summary) or "UNKNOWN"
|
||||
year = self._to_year(first_non_empty([
|
||||
core.get("year"),
|
||||
vehicle_summary.get("year"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "year"),
|
||||
self._extract_detail_value(vehicle_summary, "Year"),
|
||||
]))
|
||||
price = self._first_parsed_int(
|
||||
[
|
||||
pricing.get("buy_now"),
|
||||
pricing.get("current_bid"),
|
||||
vehicle_summary.get("buy_now"),
|
||||
vehicle_summary.get("current_bid"),
|
||||
pricing.get("actual_cash_value"),
|
||||
vehicle_summary.get("price"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "price"),
|
||||
self._extract_detail_value(vehicle_summary, "Price"),
|
||||
],
|
||||
self._to_money_int,
|
||||
)
|
||||
mileage = self._parse_odometer(
|
||||
first_non_empty([
|
||||
core.get("odometer"),
|
||||
core.get("kilometers"),
|
||||
vehicle_summary.get("odometer"),
|
||||
vehicle_summary.get("kilometers"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "kilometers"),
|
||||
self._extract_detail_value(vehicle_summary, "Kilometers"),
|
||||
])
|
||||
)
|
||||
color = self._normalize_color(first_non_empty([
|
||||
core.get("color"),
|
||||
vehicle_summary.get("color"),
|
||||
vehicle_summary.get("exterior_color"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "exterior_color"),
|
||||
self._extract_detail_value(vehicle_summary, "Exterior Color"),
|
||||
"other",
|
||||
]))
|
||||
drive = self._normalize_drive(first_non_empty([
|
||||
core.get("drive"),
|
||||
vehicle_summary.get("drive"),
|
||||
vehicle_summary.get("drive_type"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "drive_system"),
|
||||
self._extract_detail_value(vehicle_summary, "Drive Type"),
|
||||
self._extract_detail_value(vehicle_summary, "Drive System"),
|
||||
]))
|
||||
# Пробуем взять привод из двигателя.
|
||||
if not drive or drive == "NA":
|
||||
engine_text = self._as_str(first_non_empty([
|
||||
core.get("engine"),
|
||||
vehicle_summary.get("engine"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
|
||||
self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
|
||||
]))
|
||||
if engine_text:
|
||||
inferred_drive = self._normalize_drive(engine_text)
|
||||
if inferred_drive and inferred_drive != "NA":
|
||||
drive = inferred_drive
|
||||
gearbox = self._normalize_gearbox(first_non_empty([
|
||||
core.get("gearbox"),
|
||||
vehicle_summary.get("gearbox"),
|
||||
vehicle_summary.get("transmission_type"),
|
||||
vehicle_summary.get("transmission"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "transmission_type"),
|
||||
self._extract_detail_value(vehicle_summary, "Transmission Type"),
|
||||
]))
|
||||
steering = self._normalize_steering(first_non_empty([
|
||||
core.get("steering_wheel"),
|
||||
vehicle_summary.get("steering_wheel"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "steering_side"),
|
||||
self._extract_detail_value(vehicle_summary, "Steering Side"),
|
||||
])) or "LEFT"
|
||||
body_type = self._normalize_body_type(first_non_empty([
|
||||
core.get("body_type"),
|
||||
vehicle_summary.get("body_type"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "body_type"),
|
||||
self._extract_detail_value(vehicle_summary, "Body Type"),
|
||||
]))
|
||||
engine_volume = self._to_engine_cc(first_non_empty([
|
||||
core.get("engine"),
|
||||
core.get("engine_volume"),
|
||||
vehicle_summary.get("engine"),
|
||||
vehicle_summary.get("engine_volume"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
|
||||
self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
|
||||
]))
|
||||
title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
|
||||
seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
|
||||
location = self._as_str(first_non_empty([
|
||||
core.get("location"),
|
||||
vehicle_summary.get("location"),
|
||||
vehicle_summary.get("location_name"),
|
||||
self._extract_nested_text(vehicle_summary.get("neighbourhood"), "en"),
|
||||
self._extract_location_country(vehicle_summary),
|
||||
auction.get("branch"),
|
||||
"",
|
||||
]))
|
||||
country = self._normalize_country(first_non_empty([core.get("country"), location, "AE"]))
|
||||
is_damaged = self._bool_damage(damage, vehicle_summary)
|
||||
is_sold = self._bool_sold(auction)
|
||||
one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
|
||||
new_car = self._boolish(first_non_empty([core.get("new_car"), vehicle_summary.get("new_car"), False]))
|
||||
rental = self._boolish(first_non_empty([core.get("rental"), vehicle_summary.get("rental"), False]))
|
||||
repair_history = self._boolish(first_non_empty([core.get("repair_history"), vehicle_summary.get("repair_history"), False]))
|
||||
non_smoking = self._boolish(first_non_empty([core.get("non_smoking"), vehicle_summary.get("non_smoking"), True]))
|
||||
evaluation = self._as_str(first_non_empty([core.get("grade"), core.get("evaluation"), vehicle_summary.get("evaluation")])) or None
|
||||
currency = self._normalize_currency(
|
||||
first_non_empty(
|
||||
[
|
||||
pricing.get("currency"),
|
||||
vehicle_summary.get("currency"),
|
||||
pricing.get("buy_now"),
|
||||
pricing.get("current_bid"),
|
||||
vehicle_summary.get("buy_now"),
|
||||
vehicle_summary.get("current_bid"),
|
||||
"USD",
|
||||
]
|
||||
)
|
||||
)
|
||||
slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
|
||||
images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
|
||||
origin = "DUBIZZLE"
|
||||
|
||||
return CarRecord(
|
||||
parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
|
||||
mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
|
||||
steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
|
||||
selling_type=self._normalize_selling_type(first_non_empty([core.get("selling_type"), "STOCK"])), one_owner=one_owner, new_car=new_car,
|
||||
is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
|
||||
evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
|
||||
slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _as_str(value: Any) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
def _resolve_make(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
|
||||
category_make, _ = self._extract_category_make_model(vehicle_summary)
|
||||
slug_make, _ = self._extract_slug_make_model(vehicle_summary)
|
||||
return self._first_text(
|
||||
[
|
||||
core.get("make"),
|
||||
vehicle_summary.get("make"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "make"),
|
||||
self._extract_detail_value(vehicle_summary, "Make"),
|
||||
category_make,
|
||||
slug_make,
|
||||
]
|
||||
)
|
||||
|
||||
def _resolve_model(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
|
||||
_, category_model = self._extract_category_make_model(vehicle_summary)
|
||||
_, slug_model = self._extract_slug_make_model(vehicle_summary)
|
||||
return self._first_text(
|
||||
[
|
||||
core.get("model"),
|
||||
vehicle_summary.get("model"),
|
||||
self._extract_detail_v2_value(vehicle_summary, "model"),
|
||||
self._extract_detail_value(vehicle_summary, "Model"),
|
||||
category_model,
|
||||
slug_model,
|
||||
]
|
||||
)
|
||||
|
||||
def _first_text(self, values: list[Any]) -> str | None:
|
||||
for value in values:
|
||||
text = self._coerce_text(value)
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
|
||||
def _coerce_text(self, value: Any) -> str | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, list):
|
||||
for item in value:
|
||||
text = self._coerce_text(item)
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
if isinstance(value, dict):
|
||||
for key in ("en", "value", "name", "text", "label"):
|
||||
if key in value:
|
||||
text = self._coerce_text(value.get(key))
|
||||
if text:
|
||||
return text
|
||||
for nested in value.values():
|
||||
text = self._coerce_text(nested)
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
text = self._as_str(value)
|
||||
return text or None
|
||||
|
||||
def _extract_nested_text(self, value: Any, preferred_key: str = "en") -> str | None:
|
||||
if not isinstance(value, dict):
|
||||
return self._coerce_text(value)
|
||||
return self._coerce_text(value.get(preferred_key)) or self._coerce_text(value)
|
||||
|
||||
def _extract_detail_v2_value(self, vehicle_summary: dict[str, Any], slug: str) -> str | None:
|
||||
details_v2 = vehicle_summary.get("details_v2")
|
||||
if not isinstance(details_v2, dict):
|
||||
return None
|
||||
target = slug.strip().lower()
|
||||
for section in details_v2.values():
|
||||
if not isinstance(section, list):
|
||||
continue
|
||||
for item in section:
|
||||
if not isinstance(item, dict):
|
||||
continue
|
||||
if self._as_str(item.get("slug")).lower() != target:
|
||||
continue
|
||||
text = self._coerce_text(item.get("value"))
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
|
||||
def _extract_detail_value(self, vehicle_summary: dict[str, Any], detail_key: str) -> str | None:
|
||||
details = vehicle_summary.get("details")
|
||||
if not isinstance(details, dict):
|
||||
return None
|
||||
target = detail_key.strip().lower()
|
||||
for key, value in details.items():
|
||||
if self._as_str(key).lower() != target:
|
||||
continue
|
||||
text = self._coerce_text(value)
|
||||
if text:
|
||||
return text
|
||||
return None
|
||||
|
||||
def _extract_category_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
|
||||
category_v2 = vehicle_summary.get("category_v2")
|
||||
if isinstance(category_v2, dict):
|
||||
names_en = category_v2.get("names_en")
|
||||
if isinstance(names_en, list) and len(names_en) >= 4:
|
||||
return self._coerce_text(names_en[2]), self._coerce_text(names_en[3])
|
||||
|
||||
category = vehicle_summary.get("category")
|
||||
if isinstance(category, dict):
|
||||
names_en = category.get("en")
|
||||
if isinstance(names_en, list) and len(names_en) >= 3:
|
||||
return self._coerce_text(names_en[1]), self._coerce_text(names_en[2])
|
||||
|
||||
return None, None
|
||||
|
||||
def _extract_slug_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
|
||||
category_v2 = vehicle_summary.get("category_v2")
|
||||
if not isinstance(category_v2, dict):
|
||||
return None, None
|
||||
slug_paths = category_v2.get("slug_paths")
|
||||
if not isinstance(slug_paths, list) or len(slug_paths) < 3:
|
||||
return None, None
|
||||
make = self._humanize_slug_path_part(slug_paths[2])
|
||||
model = self._humanize_slug_path_part(slug_paths[3]) if len(slug_paths) >= 4 else None
|
||||
return make, model
|
||||
|
||||
def _humanize_slug_path_part(self, value: Any) -> str | None:
|
||||
text = self._as_str(value)
|
||||
if not text:
|
||||
return None
|
||||
slug = text.split("/")[-1].strip().strip("-")
|
||||
if not slug:
|
||||
return None
|
||||
return slug.replace("-", " ").title()
|
||||
|
||||
def _extract_location_country(self, vehicle_summary: dict[str, Any]) -> str | None:
|
||||
site = vehicle_summary.get("site")
|
||||
if isinstance(site, dict):
|
||||
return self._coerce_text(site.get("en"))
|
||||
location_list = vehicle_summary.get("location_list")
|
||||
if isinstance(location_list, dict):
|
||||
en_values = location_list.get("en")
|
||||
if isinstance(en_values, list) and en_values:
|
||||
return self._coerce_text(en_values[0])
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _to_int(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return int(value)
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
digits = re.sub(r"[^\d]", "", str(value))
|
||||
return int(digits) if digits else None
|
||||
|
||||
@classmethod
|
||||
def _to_money_int(cls, value: Any) -> int | None:
|
||||
# Нормализация цены.
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, (int, float)):
|
||||
return int(value)
|
||||
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return None
|
||||
|
||||
lowered = text.lower()
|
||||
if any(token in lowered for token in ["n/a", "na", "tbd", "unknown", "call", "contact"]):
|
||||
return None
|
||||
|
||||
numbers = re.findall(r"\d[\d\s.,]*", text)
|
||||
if not numbers:
|
||||
return None
|
||||
|
||||
best: int | None = None
|
||||
for number in numbers:
|
||||
clean = number.replace(" ", "")
|
||||
if "," in clean and "." in clean:
|
||||
# Поддержка двух форматов.
|
||||
if clean.rfind(",") > clean.rfind("."):
|
||||
clean = clean.replace(".", "").replace(",", ".")
|
||||
else:
|
||||
clean = clean.replace(",", "")
|
||||
elif "," in clean:
|
||||
parts = clean.split(",")
|
||||
# Десятичный или тысячный разделитель.
|
||||
if len(parts[-1]) in {1, 2} and len(parts) == 2:
|
||||
clean = clean.replace(",", ".")
|
||||
else:
|
||||
clean = clean.replace(",", "")
|
||||
elif "." in clean:
|
||||
parts = clean.split(".")
|
||||
if not (len(parts[-1]) in {1, 2} and len(parts) == 2):
|
||||
clean = clean.replace(".", "")
|
||||
|
||||
try:
|
||||
parsed = int(float(clean))
|
||||
except ValueError:
|
||||
continue
|
||||
|
||||
if parsed > 0 and (best is None or parsed > best):
|
||||
best = parsed
|
||||
|
||||
return best
|
||||
|
||||
@staticmethod
|
||||
def _first_parsed_int(values: list[Any], parser) -> int | None:
|
||||
for value in values:
|
||||
parsed = parser(value)
|
||||
if parsed is not None:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _to_year(value: Any) -> int | None:
|
||||
parsed = CarMapper._to_int(value)
|
||||
if parsed is None:
|
||||
return None
|
||||
if 1900 <= parsed <= 2100:
|
||||
return parsed
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _parse_odometer(value: Any) -> int:
|
||||
# Разбор пробега.
|
||||
if value is None:
|
||||
return 0
|
||||
text = str(value).strip()
|
||||
if not text:
|
||||
return 0
|
||||
lowered = text.lower()
|
||||
if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
|
||||
return 0
|
||||
# Ищем число.
|
||||
numbers = re.findall(r"[\d,]+", text)
|
||||
for num_str in numbers:
|
||||
clean = num_str.replace(",", "")
|
||||
if clean.isdigit() and int(clean) > 0:
|
||||
return int(clean)
|
||||
return 0
|
||||
|
||||
def _to_engine_cc(self, value: Any) -> int | None:
|
||||
# Поддержка литров и cc.
|
||||
text = str(value).lower().strip() if value is not None else ""
|
||||
if not text:
|
||||
return None
|
||||
if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
|
||||
return int(float(liters_match.group(1)) * 1000)
|
||||
if cubic_match := re.search(r"(\d{3,5})(?:\+)?\s*(?:cc|cm3|cubic)", text):
|
||||
return int(cubic_match.group(1))
|
||||
return int(text) if re.match(r"^\d+$", text) else None
|
||||
|
||||
def _normalize_currency(self, value: Any) -> str:
|
||||
text = self._as_str(value)
|
||||
upper = text.upper() or "USD"
|
||||
if any(token in text for token in ["€", "EUR"]):
|
||||
return "EUR"
|
||||
if any(token in text for token in ["¥", "JPY"]):
|
||||
return "JPY"
|
||||
if any(token in text for token in ["₩", "KRW"]):
|
||||
return "KRW"
|
||||
if any(token in text for token in ["£", "GBP"]):
|
||||
return "GBP"
|
||||
if any(token in text for token in ["₽", "RUB"]):
|
||||
return "RUB"
|
||||
if any(token in text for token in ["AED", "د.إ"]):
|
||||
return "AED"
|
||||
if any(token in text for token in ["CA$", "CAD"]):
|
||||
return "CAD"
|
||||
|
||||
text = upper
|
||||
if text in CURRENCY_ENUM_VALUES:
|
||||
return text
|
||||
return "USD" if "$" in str(value) else "USD"
|
||||
|
||||
def _normalize_drive(self, value: Any) -> str | None:
|
||||
return self._map_value(value, self.DRIVE_MAP, DRIVE_ENUM_VALUES, empty_default=None, fallback="NA")
|
||||
|
||||
def _normalize_gearbox(self, value: Any) -> str | None:
|
||||
return self._map_value(value, self.GEARBOX_MAP, GEARBOX_ENUM_VALUES, empty_default=None, fallback="NA")
|
||||
|
||||
def _normalize_steering(self, value: Any) -> str | None:
|
||||
return self._map_value(value, self.STEERING_MAP, STEERING_WHEEL_ENUM_VALUES, empty_default=None, fallback=None)
|
||||
|
||||
def _normalize_body_type(self, value: Any) -> str:
|
||||
return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
|
||||
|
||||
def _normalize_country(self, value: Any) -> str:
|
||||
fallback = "AE" if "AE" in COUNTRY_ENUM_VALUES else "NA"
|
||||
return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="AE", fallback=fallback) or fallback
|
||||
|
||||
def _normalize_selling_type(self, value: Any) -> str:
|
||||
text = (self._as_str(value) or "STOCK").upper()
|
||||
if text in SELLING_TYPE_ENUM_VALUES:
|
||||
return text
|
||||
if text in {"DEALER", "PRIVATE", "CLASSIFIED"}:
|
||||
return "STOCK"
|
||||
return "STOCK"
|
||||
|
||||
def _map_value(
|
||||
self,
|
||||
value: Any,
|
||||
mapping: dict[str, str],
|
||||
allowed_values: tuple[str, ...],
|
||||
*,
|
||||
empty_default: str | None,
|
||||
fallback: str | None,
|
||||
) -> str | None:
|
||||
# Общая нормализация enum.
|
||||
text = self._as_str(value).lower()
|
||||
if not text:
|
||||
return empty_default
|
||||
if (mapped := mapping.get(text)) and mapped in allowed_values:
|
||||
return mapped
|
||||
for marker, mapped in mapping.items():
|
||||
if marker in text and mapped in allowed_values:
|
||||
return mapped
|
||||
return fallback
|
||||
|
||||
@staticmethod
|
||||
def _normalize_color(value: Any) -> str:
|
||||
text = str(value).strip() if value is not None else "other"
|
||||
if not text:
|
||||
return "other"
|
||||
if "/" in text:
|
||||
text = text.split("/")[0].strip()
|
||||
return text.lower() or "other"
|
||||
|
||||
@staticmethod
|
||||
def _boolish(value: Any) -> bool:
|
||||
return value if isinstance(value, bool) else str(value).strip().lower() in {"1", "true", "yes", "y", "owner", "one owner", "new"}
|
||||
|
||||
def _bool_damage(self, damage: dict[str, Any], vehicle_summary: dict[str, Any]) -> bool:
|
||||
for value in [damage.get("primary"), damage.get("secondary"), damage.get("description"), vehicle_summary.get("primary_damage")]:
|
||||
text = self._as_str(value).lower()
|
||||
if text and text not in self.NO_DAMAGE_MARKERS:
|
||||
return True
|
||||
return False
|
||||
|
||||
def _bool_sold(self, auction: dict[str, Any]) -> bool:
|
||||
return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
|
||||
|
||||
def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
|
||||
# Для imageKeys берём самый большой размер.
|
||||
best_by_key: dict[str, str] = {}
|
||||
key_order: list[str] = []
|
||||
non_keyed: list[str] = []
|
||||
for item in urls:
|
||||
if not isinstance(item, str):
|
||||
continue
|
||||
url = item.strip()
|
||||
if not url:
|
||||
continue
|
||||
if m := re.search(r'imageKeys=([^&]+)', url):
|
||||
img_key = m.group(1)
|
||||
w = int(re.search(r'width=(\d+)', url).group(1)) if re.search(r'width=(\d+)', url) else 0
|
||||
existing = best_by_key.get(img_key)
|
||||
if existing is None:
|
||||
best_by_key[img_key] = url
|
||||
key_order.append(img_key)
|
||||
else:
|
||||
existing_w = int(re.search(r'width=(\d+)', existing).group(1)) if re.search(r'width=(\d+)', existing) else 0
|
||||
if w > existing_w:
|
||||
best_by_key[img_key] = url
|
||||
elif url not in non_keyed:
|
||||
non_keyed.append(url)
|
||||
deduped = [best_by_key[k] for k in key_order] + non_keyed
|
||||
return [ImageRecord(fullres_image=self._make_fullres_url(url), preview_image=self._make_preview_url(url), order_index=index) for index, url in enumerate(deduped)]
|
||||
|
||||
@staticmethod
|
||||
def _make_fullres_url(url: str) -> str:
|
||||
if "vis.dubizzle.com" in url:
|
||||
return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
|
||||
return url
|
||||
|
||||
@staticmethod
|
||||
def _make_preview_url(url: str) -> str:
|
||||
if "vis.dubizzle.com" in url:
|
||||
preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
|
||||
if preview != url:
|
||||
return preview
|
||||
return url
|
||||
|
||||
# Формирование parser_id.
|
||||
|
||||
_PARSER_ID_ALPHABET = ascii_letters + digits
|
||||
|
||||
@classmethod
|
||||
def _generate_parser_id(cls, origin_id: str) -> str:
|
||||
# Стабильный parser_id.
|
||||
digest = hashlib.sha256(origin_id.encode()).digest()
|
||||
alphabet = cls._PARSER_ID_ALPHABET
|
||||
base = len(alphabet)
|
||||
num = int.from_bytes(digest[:17], "big") # Хватает на 22 символа.
|
||||
chars: list[str] = []
|
||||
for _ in range(22):
|
||||
num, idx = divmod(num, base)
|
||||
chars.append(alphabet[idx])
|
||||
return "car-" + "".join(chars)
|
||||
|
||||
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
|
||||
# Формат: dubizzle:{lot_number}.
|
||||
for value in [
|
||||
core.get("lot_number"),
|
||||
vehicle_summary.get("lot_number"),
|
||||
vehicle_summary.get("id"),
|
||||
vehicle_summary.get("objectID"),
|
||||
vehicle_summary.get("uuid"),
|
||||
]:
|
||||
text = self._as_str(value)
|
||||
if text:
|
||||
return f"dubizzle:{text}"
|
||||
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
|
||||
if "~" in tail:
|
||||
tail = tail.split("~")[0]
|
||||
raw = tail or self._slugify(vehicle_url)
|
||||
return f"dubizzle:{raw}"
|
||||
|
||||
@staticmethod
|
||||
def _slugify(value: str) -> str:
|
||||
return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
|
||||
|
||||
|
||||
408
dubizzle_scraper/parsing/parser.py
Normal file
408
dubizzle_scraper/parsing/parser.py
Normal file
@@ -0,0 +1,408 @@
|
||||
import html as html_module
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.parsers")
|
||||
|
||||
|
||||
class VehicleParser:
|
||||
# Парсер страницы авто.
|
||||
|
||||
# Регулярки парсинга и защиты.
|
||||
_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
|
||||
_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
|
||||
_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
|
||||
_CAPTCHA_RE = re.compile(r"captcha|recaptcha|robot|are you human|security check", re.IGNORECASE)
|
||||
_ANTIBOT_RE = re.compile(r"incapsula|imperva|ddos.guard|cloudflare|access denied|forbidden", re.IGNORECASE)
|
||||
|
||||
SUMMARY_KEY_MAP = {
|
||||
"lot_number": {"lotnumber", "stockno", "itemid", "lotid", "itemnumber"},
|
||||
"year": {"year"},
|
||||
"make": {"make", "manufacturer", "brand"},
|
||||
"model": {"model"},
|
||||
"trim": {"trim", "series"},
|
||||
"odometer": {"odometer", "odometermiles", "mileage", "actualcashvalueodometer"},
|
||||
"primary_damage": {"primarydamage", "damage", "damagetype", "loss"},
|
||||
"secondary_damage": {"secondarydamage"},
|
||||
"run_and_drive": {"runanddrive", "canrunanddrive", "rundrive"},
|
||||
"buy_now": {"buynowprice", "buyitnowprice", "instantpurchaseprice"},
|
||||
"current_bid": {"currentbid", "highbid", "bidamount", "currenthighbid"},
|
||||
"actual_cash_value": {"actualcashvalue", "acv"},
|
||||
"estimated_repair_cost": {"estimatedrepaircost", "repaircost"},
|
||||
"keys": {"keys", "keystatus"},
|
||||
"title": {"titletype", "title", "documenttype"},
|
||||
"seller": {"seller", "sellername"},
|
||||
"location": {"location", "branchname", "auctionlocation", "branch"},
|
||||
"auction_date": {"auctiondate", "saledate", "liveauctiondate"},
|
||||
"body_type": {"bodytype", "bodystyle", "vehicletype", "bodyclass"},
|
||||
"drive": {"driveline", "drive", "drivelinetype", "drivetype", "drivetrain"},
|
||||
"gearbox": {"transmission", "gearbox", "transmissiontype"},
|
||||
"engine": {"engine", "enginevolume", "enginetype", "enginedescription"},
|
||||
"fuel_type": {"fueltype", "fuel"},
|
||||
"cylinders": {"cylinders", "cylindercount"},
|
||||
"color": {"color", "primarycolor", "exteriorcolor"},
|
||||
}
|
||||
|
||||
DOM_LABEL_MAP: dict[str, str] = {
|
||||
"stock #": "lot_number",
|
||||
"stock": "lot_number",
|
||||
"primary damage": "primary_damage",
|
||||
"secondary damage": "secondary_damage",
|
||||
"odometer": "odometer",
|
||||
"odometer (miles)": "odometer",
|
||||
"mileage": "odometer",
|
||||
"body style": "body_type",
|
||||
"body type": "body_type",
|
||||
"vehicle type": "body_type",
|
||||
"engine": "engine",
|
||||
"engine type": "engine",
|
||||
"transmission": "gearbox",
|
||||
"drive line type": "drive",
|
||||
"driveline type": "drive",
|
||||
"drive line": "drive",
|
||||
"driveline": "drive",
|
||||
"drive type": "drive",
|
||||
"fuel type": "fuel_type",
|
||||
"fuel": "fuel_type",
|
||||
"cylinders": "cylinders",
|
||||
"exterior/interior": "color",
|
||||
"exterior color": "color",
|
||||
"color": "color",
|
||||
"model": "model",
|
||||
"series": "trim",
|
||||
"selling branch": "location",
|
||||
"vehicle location": "vehicle_location",
|
||||
"auction date and time": "auction_date",
|
||||
"sale date": "auction_date",
|
||||
"lane/run #": "lane",
|
||||
"actual cash value": "actual_cash_value",
|
||||
"estimated repair cost": "estimated_repair_cost",
|
||||
"seller": "seller",
|
||||
"title/sale doc": "title",
|
||||
"title/sale doc brand": "title_brand",
|
||||
"start code": "run_and_drive",
|
||||
"key": "keys",
|
||||
"keys": "keys",
|
||||
"manufactured in": "manufactured_in",
|
||||
"vehicle class": "vehicle_class",
|
||||
}
|
||||
|
||||
def _parse_dom_key_value_pairs(self, dom_text: str) -> dict[str, str]:
|
||||
result: dict[str, str] = {}
|
||||
if not dom_text:
|
||||
return result
|
||||
lines = [line.strip() for line in dom_text.split("\n") if line.strip()]
|
||||
known_labels = set(self.DOM_LABEL_MAP.keys())
|
||||
skip_values = {"more actions", "view", "print", "share", "back to results", "all images", "view all images"}
|
||||
max_fields = len(set(self.DOM_LABEL_MAP.values()))
|
||||
|
||||
for i, line in enumerate(lines):
|
||||
# Ранний выход.
|
||||
if len(result) >= max_fields:
|
||||
break
|
||||
|
||||
# Метка и значение в одной строке.
|
||||
colon_pos = line.find(":")
|
||||
if colon_pos > 0:
|
||||
label_part = line[:colon_pos].strip().lower()
|
||||
value_part = line[colon_pos + 1:].strip()
|
||||
if label_part in known_labels and value_part and value_part.lower() not in skip_values:
|
||||
field_name = self.DOM_LABEL_MAP[label_part]
|
||||
if field_name not in result or not result[field_name]:
|
||||
result[field_name] = value_part
|
||||
continue
|
||||
|
||||
# Метка и значение в соседних строках.
|
||||
clean = line.rstrip(":").strip().lower()
|
||||
clean_alt = clean.rstrip("#").strip()
|
||||
matched_label = None
|
||||
if clean in known_labels:
|
||||
matched_label = clean
|
||||
elif clean_alt in known_labels:
|
||||
matched_label = clean_alt
|
||||
|
||||
if matched_label and i + 1 < len(lines):
|
||||
value = lines[i + 1].strip()
|
||||
if value.rstrip(":").lower().strip() in known_labels:
|
||||
continue
|
||||
if value.lower() in skip_values:
|
||||
continue
|
||||
field_name = self.DOM_LABEL_MAP[matched_label]
|
||||
if field_name not in result or not result[field_name]:
|
||||
result[field_name] = value
|
||||
return result
|
||||
|
||||
def _parse_title_for_year_make_model(self, page_title: str, dom_text: str) -> dict[str, str | None]:
|
||||
result: dict[str, str | None] = {"year": None, "make": None, "model": None}
|
||||
title_match = re.match(r"(\d{4})\s+(\S+)\s+(.+?)(?:\s+for\s+)", page_title or "")
|
||||
if title_match:
|
||||
result["year"] = title_match.group(1)
|
||||
result["make"] = title_match.group(2)
|
||||
result["model"] = title_match.group(3)
|
||||
return result
|
||||
dom_match = re.search(r"(?:Search|Log In)\s*\n\s*(\d{4})\s+(\S+)\s+(.+?)(?:\n|$)", dom_text or "")
|
||||
if dom_match:
|
||||
result["year"] = dom_match.group(1)
|
||||
result["make"] = dom_match.group(2)
|
||||
result["model"] = dom_match.group(3).strip()
|
||||
return result
|
||||
|
||||
def normalize(self, vehicle_url: str, page_html: str, dom_text: str, network_dump: dict[str, Any]) -> dict[str, Any]:
|
||||
page_html = page_html or ""
|
||||
dom_text = dom_text or ""
|
||||
network_dump = network_dump or {}
|
||||
responses = network_dump.get("json_responses", [])
|
||||
payloads = [item.get("payload") for item in responses if isinstance(item.get("payload"), (dict, list))]
|
||||
dom_kv = self._parse_dom_key_value_pairs(dom_text)
|
||||
page_title = ""
|
||||
title_match = re.search(r"<title[^>]*>(.*?)</title>", page_html or "", re.IGNORECASE | re.DOTALL)
|
||||
if title_match:
|
||||
page_title = title_match.group(1).strip()
|
||||
title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
|
||||
|
||||
# Один проход по payload.
|
||||
all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
|
||||
|
||||
summary: dict[str, Any] = {"source_url": vehicle_url}
|
||||
for field, values in all_found.items():
|
||||
if field in dom_kv:
|
||||
values.append(dom_kv[field])
|
||||
summary[field] = first_non_empty(values)
|
||||
|
||||
summary["year"] = summary.get("year") or title_parsed.get("year")
|
||||
summary["make"] = summary.get("make") or title_parsed.get("make")
|
||||
summary["model"] = summary.get("model") or title_parsed.get("model")
|
||||
summary["trim"] = summary.get("trim") or dom_kv.get("trim")
|
||||
summary["lot_number"] = summary.get("lot_number") or self._extract_lot_number(dom_text)
|
||||
summary["image_urls"] = self._extract_image_urls(payloads, page_html, vehicle_url)
|
||||
for dom_field, dom_value in dom_kv.items():
|
||||
if dom_field not in summary or not summary[dom_field]:
|
||||
summary[dom_field] = dom_value
|
||||
prices = self._extract_prices_from_text(dom_text)
|
||||
if not summary.get("actual_cash_value") and prices:
|
||||
summary["actual_cash_value"] = prices[0]
|
||||
if not summary.get("buy_now"):
|
||||
buy_now_match = self._BUY_NOW_RE.search(dom_text or "")
|
||||
if buy_now_match:
|
||||
summary["buy_now"] = buy_now_match.group(1)
|
||||
elif prices:
|
||||
summary["buy_now"] = prices[0]
|
||||
if not summary.get("current_bid") and len(prices) > 1:
|
||||
summary["current_bid"] = prices[1]
|
||||
|
||||
embedded = self._extract_embedded_json(page_html)
|
||||
for item in embedded:
|
||||
p = item.get("payload")
|
||||
if isinstance(p, (dict, list)):
|
||||
payloads.append(p)
|
||||
# Доп. проход по JSON.
|
||||
extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
|
||||
for field, vals in extra.items():
|
||||
if not summary.get(field):
|
||||
v = first_non_empty(vals)
|
||||
if v:
|
||||
summary[field] = v
|
||||
|
||||
# Передаём готовые image_urls.
|
||||
image_urls = summary.get("image_urls") or []
|
||||
return {
|
||||
"vehicle_summary": summary,
|
||||
"payload_insights": self._build_payload_insights(summary, responses, payloads, vehicle_url, image_urls=image_urls),
|
||||
"embedded_json": embedded,
|
||||
"dom_hints": self._dom_hints(dom_text),
|
||||
"access_notes": self._build_access_notes(summary, responses),
|
||||
}
|
||||
|
||||
def _build_payload_insights(self, summary: dict[str, Any], responses: list[dict[str, Any]], payloads: list[Any], vehicle_url: str = "", image_urls: list[str] | None = None) -> dict[str, Any]:
|
||||
if image_urls is None:
|
||||
image_urls = self._extract_image_urls(payloads, "", vehicle_url)
|
||||
return {
|
||||
"vehicle_core": {
|
||||
"lot_number": summary.get("lot_number"), "year": summary.get("year"),
|
||||
"make": summary.get("make"), "model": summary.get("model"), "trim": summary.get("trim"),
|
||||
"odometer": summary.get("odometer"), "run_and_drive": summary.get("run_and_drive"),
|
||||
"seller": summary.get("seller"), "location": summary.get("location"), "title": summary.get("title"),
|
||||
"body_type": summary.get("body_type"), "drive": summary.get("drive"), "gearbox": summary.get("gearbox"),
|
||||
"engine": summary.get("engine"), "fuel_type": summary.get("fuel_type"), "cylinders": summary.get("cylinders"),
|
||||
"color": summary.get("color"), "keys": summary.get("keys"),
|
||||
},
|
||||
"pricing": {
|
||||
"buy_now": summary.get("buy_now"), "current_bid": summary.get("current_bid"),
|
||||
"actual_cash_value": summary.get("actual_cash_value"), "estimated_repair_cost": summary.get("estimated_repair_cost"),
|
||||
"currency": self._guess_currency(summary),
|
||||
},
|
||||
"bids": self._build_bid_insights(summary, payloads),
|
||||
"damage": {
|
||||
"primary": summary.get("primary_damage"),
|
||||
"secondary": summary.get("secondary_damage"),
|
||||
"description": first_non_empty(self._find_in_payloads(payloads, {"damageDescription", "damageDetails"})),
|
||||
},
|
||||
"auction": {
|
||||
"auction_date": summary.get("auction_date"),
|
||||
"lane": first_non_empty(self._find_in_payloads(payloads, {"lane", "lanename"})),
|
||||
"branch": first_non_empty([summary.get("location"), *self._find_in_payloads(payloads, {"branch", "branchname"})]),
|
||||
"sale_status": first_non_empty(self._find_in_payloads(payloads, {"salestatus", "auctionstatus", "status"})),
|
||||
"item_number": first_non_empty([summary.get("lot_number"), *self._find_in_payloads(payloads, {"itemnumber", "lotnumber", "lotid"})]),
|
||||
},
|
||||
"images": {"count": len(image_urls), "urls": image_urls},
|
||||
"source_endpoints": self._build_source_endpoints(responses),
|
||||
}
|
||||
|
||||
def _build_bid_insights(self, summary: dict[str, Any], payloads: list[Any]) -> dict[str, Any]:
|
||||
return {
|
||||
"amount": summary.get("current_bid"),
|
||||
"currency": self._guess_currency(summary),
|
||||
"bid_count": first_non_empty(self._find_in_payloads(payloads, {"bidcount", "numberofbids"})),
|
||||
"status": first_non_empty(self._find_in_payloads(payloads, {"bidstatus", "biddingstatus"})),
|
||||
}
|
||||
|
||||
def _build_source_endpoints(self, responses: list[dict[str, Any]]) -> dict[str, list[str]]:
|
||||
mapping = {"vehicle": [], "pricing": [], "bids": [], "damage": [], "auction": [], "images": []}
|
||||
for item in responses:
|
||||
url = item.get("url", "")
|
||||
category = item.get("category", "other")
|
||||
if category == "vehicle":
|
||||
mapping["vehicle"].append(url)
|
||||
lowered = url.lower()
|
||||
if any(token in lowered for token in ["bid", "offer"]):
|
||||
mapping["bids"].append(url)
|
||||
if any(token in lowered for token in ["damage", "report"]):
|
||||
mapping["damage"].append(url)
|
||||
if any(token in lowered for token in ["auction", "sale", "lane", "branch"]):
|
||||
mapping["auction"].append(url)
|
||||
elif category in mapping:
|
||||
mapping[category].append(url)
|
||||
return {key: list(dict.fromkeys(urls)) for key, urls in mapping.items()}
|
||||
|
||||
def _build_access_notes(self, summary: dict[str, Any], responses: list[dict[str, Any]]) -> dict[str, Any]:
|
||||
endpoints = [item.get("url", "") for item in responses]
|
||||
dom_hints = self._dom_hints(" ".join(str(value) for value in summary.values() if value is not None))
|
||||
return {
|
||||
"images_visible": bool(summary.get("image_urls")),
|
||||
"network_json_count": len(responses),
|
||||
"possible_captcha": bool(dom_hints.get("has_captcha_text")),
|
||||
"possible_antibot": bool(dom_hints.get("has_antibot_text")),
|
||||
"observed_endpoints": endpoints[:20],
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def _find_in_payloads(payloads: list[Any], keys: set[str]) -> list[Any]:
|
||||
lowered = {key.lower() for key in keys}
|
||||
values: list[Any] = []
|
||||
for payload in payloads:
|
||||
values.extend(deep_find_key(payload, lowered))
|
||||
return values
|
||||
|
||||
@staticmethod
|
||||
def _guess_currency(summary: dict[str, Any]) -> str:
|
||||
for key in ["buy_now", "current_bid", "actual_cash_value", "estimated_repair_cost"]:
|
||||
value = str(summary.get(key) or "")
|
||||
if "$" in value:
|
||||
return "USD"
|
||||
if "€" in value:
|
||||
return "EUR"
|
||||
if "¥" in value:
|
||||
return "JPY"
|
||||
return "USD"
|
||||
|
||||
@staticmethod
|
||||
def _extract_lot_number(text: str) -> str | None:
|
||||
match = LOT_RE.search(text or "")
|
||||
return match.group(1) if match else None
|
||||
|
||||
@staticmethod
|
||||
def _extract_prices_from_text(text: str) -> list[str]:
|
||||
return [match.group(1) for match in PRICE_RE.finditer(text or "")]
|
||||
|
||||
@staticmethod
|
||||
def _extract_embedded_json(html: str) -> list[dict[str, Any]]:
|
||||
scripts = re.findall(r"<script[^>]*>(.*?)</script>", html or "", flags=re.DOTALL | re.IGNORECASE)
|
||||
extracted: list[dict[str, Any]] = []
|
||||
for script_text in scripts:
|
||||
if "{" not in script_text and "[" not in script_text:
|
||||
continue
|
||||
# Пропускаем большие блоки.
|
||||
if len(script_text) > 51_200:
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(script_text.strip())
|
||||
except Exception:
|
||||
continue
|
||||
extracted.append({"type": "inline_json", "payload": parsed})
|
||||
return extracted
|
||||
|
||||
@staticmethod
|
||||
def _extract_image_urls(payloads: list[Any], html: str, vehicle_url: str = "") -> list[str]:
|
||||
vehicle_key = ""
|
||||
key_match = re.search(r"VehicleDetail/(\d+)", vehicle_url or "")
|
||||
if key_match:
|
||||
vehicle_key = key_match.group(1)
|
||||
found: list[str] = []
|
||||
for payload in payloads:
|
||||
found.extend(deep_find_key(payload, {"imageurl", "imageurls", "url", "fullsizeurl", "thumbnailurl", "originalurl"}))
|
||||
flat: list[str] = []
|
||||
seen_flat: set[str] = set()
|
||||
for item in found:
|
||||
if isinstance(item, str) and item.startswith("http"):
|
||||
cleaned = html_module.unescape(item)
|
||||
lowered = cleaned.lower()
|
||||
if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
|
||||
continue
|
||||
if cleaned not in seen_flat:
|
||||
seen_flat.add(cleaned)
|
||||
flat.append(cleaned)
|
||||
elif isinstance(item, list):
|
||||
for child in item:
|
||||
if isinstance(child, str) and child.startswith("http"):
|
||||
cleaned = html_module.unescape(child)
|
||||
lowered = cleaned.lower()
|
||||
if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
|
||||
continue
|
||||
if cleaned not in seen_flat:
|
||||
seen_flat.add(cleaned)
|
||||
flat.append(cleaned)
|
||||
for pattern in [r'<img[^>]+(?:src|data-src)\s*=\s*["\']([^"\']+)["\']', r'data-src\s*=\s*["\']([^"\']+)["\']']:
|
||||
for match in re.finditer(pattern, html or "", re.IGNORECASE):
|
||||
url = html_module.unescape(match.group(1).strip())
|
||||
if not url.startswith("http") or url in seen_flat:
|
||||
continue
|
||||
lowered = url.lower()
|
||||
if vehicle_key and vehicle_key in url:
|
||||
seen_flat.add(url)
|
||||
flat.append(url)
|
||||
elif any(token in lowered for token in ["vis.dubizzle.com", "anvis", "vehicleimage"]):
|
||||
if vehicle_key and vehicle_key not in url:
|
||||
continue
|
||||
seen_flat.add(url)
|
||||
flat.append(url)
|
||||
if vehicle_key:
|
||||
for url in re.findall(r'https?://vis\.dubizzle\.com[^\s"\'<>]+', html or ""):
|
||||
cleaned = html_module.unescape(url)
|
||||
if cleaned not in seen_flat and vehicle_key in cleaned:
|
||||
seen_flat.add(cleaned)
|
||||
flat.append(cleaned)
|
||||
filtered: list[str] = []
|
||||
for url in flat:
|
||||
lowered = url.lower()
|
||||
if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
|
||||
continue
|
||||
if "vis.dubizzle.com" in lowered and "/resizer" not in lowered:
|
||||
continue
|
||||
filtered.append(url)
|
||||
return filtered
|
||||
|
||||
@staticmethod
|
||||
def _dom_hints(text: str) -> dict[str, Any]:
|
||||
lowered = (text or "").lower()
|
||||
return {
|
||||
"has_buy_now_text": "buy now" in lowered,
|
||||
"has_run_and_drive_text": "run & drive" in lowered or "run and drive" in lowered,
|
||||
"has_damage_text": "damage" in lowered,
|
||||
"has_title_text": "title" in lowered,
|
||||
"has_captcha_text": any(token in lowered for token in VehicleParser._CAPTCHA_TOKENS),
|
||||
"has_antibot_text": any(token in lowered for token in VehicleParser._ANTIBOT_TOKENS),
|
||||
}
|
||||
317
dubizzle_scraper/proxy_bridge.py
Normal file
317
dubizzle_scraper/proxy_bridge.py
Normal file
@@ -0,0 +1,317 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
import select
|
||||
import socket
|
||||
import socketserver
|
||||
import struct
|
||||
import threading
|
||||
from urllib.parse import urlsplit
|
||||
|
||||
BUFFER_SIZE = 65536
|
||||
CRLF = b"\r\n"
|
||||
DEFAULT_LISTEN_HOST = os.getenv("PROXY_BRIDGE_HOST", "127.0.0.1")
|
||||
DEFAULT_LISTEN_PORT = int(os.getenv("PROXY_BRIDGE_PORT", "8899"))
|
||||
SOCKS5_HOST = os.getenv("SOCKS5_PROXY_HOST", "")
|
||||
SOCKS5_PORT = int(os.getenv("SOCKS5_PROXY_PORT", "1002"))
|
||||
SOCKS5_USER = os.getenv("SOCKS5_PROXY_USER", "")
|
||||
SOCKS5_PASS = os.getenv("SOCKS5_PROXY_PASS", "")
|
||||
RELAY_IDLE_TIMEOUT_SECONDS = int(os.getenv("PROXY_BRIDGE_RELAY_IDLE_TIMEOUT_SECONDS", "60"))
|
||||
MAX_WORKERS = int(os.getenv("PROXY_BRIDGE_MAX_WORKERS", "64"))
|
||||
|
||||
logger = logging.getLogger("proxy_bridge")
|
||||
|
||||
|
||||
class ThreadingTCPServer(socketserver.ThreadingMixIn, socketserver.TCPServer):
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True
|
||||
|
||||
def __init__(self, server_address, request_handler_class):
|
||||
super().__init__(server_address, request_handler_class)
|
||||
self._worker_semaphore = threading.BoundedSemaphore(MAX_WORKERS)
|
||||
|
||||
def process_request_thread(self, request, client_address):
|
||||
with self._worker_semaphore:
|
||||
super().process_request_thread(request, client_address)
|
||||
|
||||
|
||||
def _recv_exact(sock: socket.socket, size: int) -> bytes:
|
||||
data = b""
|
||||
while len(data) < size:
|
||||
chunk = sock.recv(size - len(data))
|
||||
if not chunk:
|
||||
raise ConnectionError("Unexpected EOF from SOCKS5 server")
|
||||
data += chunk
|
||||
return data
|
||||
|
||||
|
||||
def _socks5_connect(host: str, port: int) -> socket.socket:
|
||||
if not SOCKS5_HOST:
|
||||
raise RuntimeError("SOCKS5_PROXY_HOST is not configured")
|
||||
|
||||
upstream = socket.create_connection((SOCKS5_HOST, SOCKS5_PORT), timeout=30)
|
||||
upstream.settimeout(30)
|
||||
|
||||
methods = [0x00]
|
||||
if SOCKS5_USER or SOCKS5_PASS:
|
||||
methods = [0x02]
|
||||
upstream.sendall(bytes([0x05, len(methods), *methods]))
|
||||
version, method = _recv_exact(upstream, 2)
|
||||
if version != 0x05 or method == 0xFF:
|
||||
upstream.close()
|
||||
raise ConnectionError("SOCKS5 authentication negotiation failed")
|
||||
|
||||
if method == 0x02:
|
||||
username = SOCKS5_USER.encode("utf-8")
|
||||
password = SOCKS5_PASS.encode("utf-8")
|
||||
if len(username) > 255 or len(password) > 255:
|
||||
upstream.close()
|
||||
raise ValueError("SOCKS5 username/password too long")
|
||||
upstream.sendall(bytes([0x01, len(username)]) + username + bytes([len(password)]) + password)
|
||||
auth_version, auth_status = _recv_exact(upstream, 2)
|
||||
if auth_version != 0x01 or auth_status != 0x00:
|
||||
upstream.close()
|
||||
raise ConnectionError("SOCKS5 username/password authentication failed")
|
||||
|
||||
try:
|
||||
socket.inet_aton(host)
|
||||
addr_type = 0x01
|
||||
addr_payload = socket.inet_aton(host)
|
||||
except OSError:
|
||||
host_bytes = host.encode("idna")
|
||||
if len(host_bytes) > 255:
|
||||
upstream.close()
|
||||
raise ValueError("Target host is too long for SOCKS5 domain format")
|
||||
addr_type = 0x03
|
||||
addr_payload = bytes([len(host_bytes)]) + host_bytes
|
||||
|
||||
request = bytes([0x05, 0x01, 0x00, addr_type]) + addr_payload + struct.pack("!H", port)
|
||||
upstream.sendall(request)
|
||||
|
||||
response_head = _recv_exact(upstream, 4)
|
||||
version, reply, _reserved, reply_addr_type = response_head
|
||||
if version != 0x05 or reply != 0x00:
|
||||
upstream.close()
|
||||
raise ConnectionError(f"SOCKS5 connect failed with code {reply}")
|
||||
|
||||
if reply_addr_type == 0x01:
|
||||
_recv_exact(upstream, 4)
|
||||
elif reply_addr_type == 0x03:
|
||||
domain_len = _recv_exact(upstream, 1)[0]
|
||||
_recv_exact(upstream, domain_len)
|
||||
elif reply_addr_type == 0x04:
|
||||
_recv_exact(upstream, 16)
|
||||
_recv_exact(upstream, 2)
|
||||
|
||||
upstream.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
|
||||
return upstream
|
||||
|
||||
|
||||
def _relay_bidirectional(left: socket.socket, right: socket.socket) -> None:
|
||||
sockets = [left, right]
|
||||
left.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
|
||||
right.settimeout(RELAY_IDLE_TIMEOUT_SECONDS)
|
||||
try:
|
||||
while True:
|
||||
readable, _, exceptional = select.select(sockets, [], sockets, RELAY_IDLE_TIMEOUT_SECONDS)
|
||||
if exceptional:
|
||||
break
|
||||
if not readable:
|
||||
logger.debug("Relay idle timeout reached; closing sockets")
|
||||
return
|
||||
for current in readable:
|
||||
other = right if current is left else left
|
||||
data = current.recv(BUFFER_SIZE)
|
||||
if not data:
|
||||
return
|
||||
other.sendall(data)
|
||||
finally:
|
||||
for sock in sockets:
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
try:
|
||||
sock.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
class ProxyHandler(socketserver.StreamRequestHandler):
|
||||
def handle(self) -> None:
|
||||
try:
|
||||
request_line = self.rfile.readline(BUFFER_SIZE).decode("iso-8859-1").strip()
|
||||
if not request_line:
|
||||
return
|
||||
|
||||
method, target, version = request_line.split()
|
||||
headers = self._read_headers()
|
||||
logger.info("%s %s", method, target)
|
||||
|
||||
if method.upper() == "CONNECT":
|
||||
host, port = self._parse_connect_target(target)
|
||||
logger.info("CONNECT %s:%s", host, port)
|
||||
upstream = _socks5_connect(host, port)
|
||||
self.wfile.write(f"{version} 200 Connection Established".encode("ascii") + CRLF + CRLF)
|
||||
self.wfile.flush()
|
||||
_relay_bidirectional(self.connection, upstream)
|
||||
return
|
||||
|
||||
host, port, path = self._parse_forward_target(target, headers)
|
||||
upstream = _socks5_connect(host, port)
|
||||
self._send_forward_request(upstream, method, path, version, headers)
|
||||
body = self._read_request_body(headers)
|
||||
if body:
|
||||
upstream.sendall(body)
|
||||
_relay_bidirectional(self.connection, upstream)
|
||||
except Exception as exc:
|
||||
logger.exception("Proxy bridge request failed: %s", exc)
|
||||
try:
|
||||
self.wfile.write(
|
||||
b"HTTP/1.1 502 Bad Gateway" + CRLF
|
||||
+ b"Connection: close" + CRLF
|
||||
+ b"Content-Type: text/plain; charset=utf-8" + CRLF + CRLF
|
||||
+ b"Bad Gateway"
|
||||
)
|
||||
self.wfile.flush()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _read_headers(self) -> list[tuple[str, str]]:
|
||||
headers: list[tuple[str, str]] = []
|
||||
while True:
|
||||
line = self.rfile.readline(BUFFER_SIZE)
|
||||
if line in {CRLF, b"\n", b""}:
|
||||
break
|
||||
decoded = line.decode("iso-8859-1")
|
||||
if ":" not in decoded:
|
||||
continue
|
||||
name, value = decoded.split(":", 1)
|
||||
headers.append((name.strip(), value.strip()))
|
||||
return headers
|
||||
|
||||
@staticmethod
|
||||
def _parse_connect_target(target: str) -> tuple[str, int]:
|
||||
if target.startswith("["):
|
||||
end = target.find("]")
|
||||
if end == -1 or len(target) <= end + 2 or target[end + 1] != ":":
|
||||
raise ValueError("Invalid CONNECT target")
|
||||
host = target[1:end]
|
||||
port_text = target[end + 2 :]
|
||||
return host, int(port_text)
|
||||
|
||||
host, port_text = target.rsplit(":", 1)
|
||||
return host, int(port_text)
|
||||
|
||||
@staticmethod
|
||||
def _parse_forward_target(target: str, headers: list[tuple[str, str]]) -> tuple[str, int, str]:
|
||||
if target.startswith("http://"):
|
||||
parts = urlsplit(target)
|
||||
port = parts.port or 80
|
||||
path = parts.path or "/"
|
||||
if parts.query:
|
||||
path += f"?{parts.query}"
|
||||
return parts.hostname or "", port, path
|
||||
|
||||
if target.startswith("https://"):
|
||||
raise ValueError("HTTPS absolute-form request must use CONNECT")
|
||||
|
||||
host_header = next((value for name, value in headers if name.lower() == "host"), "")
|
||||
if not host_header:
|
||||
raise ValueError("Missing Host header")
|
||||
if ":" in host_header:
|
||||
host, port_text = host_header.rsplit(":", 1)
|
||||
return host, int(port_text), target
|
||||
return host_header, 80, target
|
||||
|
||||
def _send_forward_request(
|
||||
self,
|
||||
upstream: socket.socket,
|
||||
method: str,
|
||||
path: str,
|
||||
version: str,
|
||||
headers: list[tuple[str, str]],
|
||||
) -> None:
|
||||
filtered_headers: list[tuple[str, str]] = []
|
||||
hop_by_hop = {
|
||||
"proxy-connection",
|
||||
"proxy-authorization",
|
||||
"connection",
|
||||
"keep-alive",
|
||||
"te",
|
||||
"trailer",
|
||||
"transfer-encoding",
|
||||
"upgrade",
|
||||
}
|
||||
for name, value in headers:
|
||||
if name.lower() in hop_by_hop:
|
||||
continue
|
||||
filtered_headers.append((name, value))
|
||||
|
||||
request_head = [f"{method} {path} {version}\r\n"]
|
||||
request_head.extend(f"{name}: {value}\r\n" for name, value in filtered_headers)
|
||||
request_head.append("\r\n")
|
||||
upstream.sendall("".join(request_head).encode("iso-8859-1"))
|
||||
|
||||
def _read_request_body(self, headers: list[tuple[str, str]]) -> bytes:
|
||||
transfer_encoding = next((value for name, value in headers if name.lower() == "transfer-encoding"), "")
|
||||
if "chunked" in transfer_encoding.lower():
|
||||
return self._read_chunked_request_body()
|
||||
|
||||
content_length = next((value for name, value in headers if name.lower() == "content-length"), None)
|
||||
if not content_length:
|
||||
return b""
|
||||
return self.rfile.read(int(content_length))
|
||||
|
||||
def _read_chunked_request_body(self) -> bytes:
|
||||
chunks: list[bytes] = []
|
||||
while True:
|
||||
size_line = self.rfile.readline(BUFFER_SIZE)
|
||||
if not size_line:
|
||||
raise ConnectionError("Unexpected EOF in chunked request")
|
||||
size_text = size_line.strip().split(b";", 1)[0]
|
||||
chunk_size = int(size_text, 16)
|
||||
chunks.append(size_line)
|
||||
if chunk_size == 0:
|
||||
while True:
|
||||
trailer_line = self.rfile.readline(BUFFER_SIZE)
|
||||
if not trailer_line:
|
||||
raise ConnectionError("Unexpected EOF in chunked trailers")
|
||||
chunks.append(trailer_line)
|
||||
if trailer_line in {CRLF, b"\n"}:
|
||||
return b"".join(chunks)
|
||||
|
||||
chunk_data = self.rfile.read(chunk_size)
|
||||
if len(chunk_data) != chunk_size:
|
||||
raise ConnectionError("Unexpected EOF in chunk body")
|
||||
chunks.append(chunk_data)
|
||||
chunk_end = self.rfile.read(2)
|
||||
if chunk_end != CRLF:
|
||||
raise ConnectionError("Invalid chunk terminator")
|
||||
chunks.append(chunk_end)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not SOCKS5_HOST:
|
||||
raise SystemExit("SOCKS5_PROXY_HOST is required")
|
||||
|
||||
logging.basicConfig(
|
||||
level=os.getenv("PROXY_BRIDGE_LOG_LEVEL", "INFO").upper(),
|
||||
format="[%(asctime)s] [proxy_bridge] %(levelname)s: %(message)s",
|
||||
)
|
||||
|
||||
with ThreadingTCPServer((DEFAULT_LISTEN_HOST, DEFAULT_LISTEN_PORT), ProxyHandler) as server:
|
||||
logger.info(
|
||||
"Listening on %s:%s -> socks5://%s:%s (max_workers=%s)",
|
||||
DEFAULT_LISTEN_HOST,
|
||||
DEFAULT_LISTEN_PORT,
|
||||
SOCKS5_HOST,
|
||||
SOCKS5_PORT,
|
||||
MAX_WORKERS,
|
||||
)
|
||||
server.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
2609
dubizzle_scraper/scraper.py
Normal file
2609
dubizzle_scraper/scraper.py
Normal file
File diff suppressed because it is too large
Load Diff
1
dubizzle_scraper/storage/__init__.py
Normal file
1
dubizzle_scraper/storage/__init__.py
Normal file
@@ -0,0 +1 @@
|
||||
__all__: list[str] = []
|
||||
675
dubizzle_scraper/storage/db.py
Normal file
675
dubizzle_scraper/storage/db.py
Normal file
@@ -0,0 +1,675 @@
|
||||
import logging
|
||||
from contextlib import contextmanager
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any, Iterator
|
||||
|
||||
from sqlalchemy import create_engine, delete, or_, select, text, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.orm import Session, sessionmaker
|
||||
|
||||
from ..core.config import Settings
|
||||
from .models import Base, Car, Image, SyncRun
|
||||
from .schemas import CarRecord
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.db")
|
||||
|
||||
|
||||
CAR_DB_FIELDS = {
|
||||
col.key for col in Car.__table__.columns
|
||||
if col.key not in ("id",)
|
||||
}
|
||||
|
||||
_IN_CHUNK_SIZE = 5000
|
||||
|
||||
|
||||
class PersistenceService:
|
||||
|
||||
def __init__(self, settings: Settings) -> None:
|
||||
self.settings = settings
|
||||
engine_kwargs = {
|
||||
"echo": settings.database.echo,
|
||||
"future": True,
|
||||
}
|
||||
if "postgresql" in settings.database.url:
|
||||
engine_kwargs["pool_size"] = settings.database.pool_size
|
||||
engine_kwargs["max_overflow"] = settings.database.max_overflow
|
||||
engine_kwargs["pool_pre_ping"] = True
|
||||
engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
|
||||
engine_kwargs["pool_timeout"] = 30
|
||||
# Таймауты запросов и блокировок.
|
||||
engine_kwargs["connect_args"] = {
|
||||
"options": "-c statement_timeout=120000 -c lock_timeout=30000"
|
||||
}
|
||||
self.engine = create_engine(settings.database.url, **engine_kwargs)
|
||||
self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
|
||||
|
||||
def create_tables(self) -> None:
|
||||
# Для SQLite можно create_all.
|
||||
is_sqlite = self.settings.database.url.startswith("sqlite")
|
||||
if not is_sqlite and not self.settings.database.auto_create_tables:
|
||||
return
|
||||
try:
|
||||
Base.metadata.create_all(self.engine)
|
||||
except Exception:
|
||||
logger.debug("create_tables skipped (schema already exists)")
|
||||
|
||||
@contextmanager
|
||||
def session_scope(self) -> Iterator[Session]:
|
||||
session = self.session_factory()
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
finally:
|
||||
session.close()
|
||||
|
||||
def start_sync_run(self, lane: str) -> int:
|
||||
with self.session_scope() as session:
|
||||
now = datetime.now(timezone.utc)
|
||||
stale_runs = session.execute(select(SyncRun).where(SyncRun.status == "running")).scalars().all()
|
||||
for stale in stale_runs:
|
||||
stale.status = "failed"
|
||||
stale.finished_at = now
|
||||
if not stale.error_summary:
|
||||
stale.error_summary = "Recovered stale running sync run before starting a new run"
|
||||
|
||||
run = SyncRun(status="running", lane=lane, ids_fetched=0, cars_upserted=0, cars_failed=0, images_upserted=0)
|
||||
session.add(run)
|
||||
session.flush()
|
||||
return int(run.id)
|
||||
|
||||
def finish_sync_run(self, run_id: int, *, status: str, ids_fetched: int, cars_upserted: int, cars_failed: int, images_upserted: int, error_summary: str | None = None) -> None:
|
||||
with self.session_scope() as session:
|
||||
run = session.get(SyncRun, run_id)
|
||||
if run is None:
|
||||
return
|
||||
run.finished_at = datetime.now(timezone.utc)
|
||||
run.status = status
|
||||
run.ids_fetched = ids_fetched
|
||||
run.cars_upserted = cars_upserted
|
||||
run.cars_failed = cars_failed
|
||||
run.images_upserted = images_upserted
|
||||
run.error_summary = error_summary
|
||||
|
||||
def get_existing_origin_urls(self, origin_urls: list[str]) -> set[str]:
|
||||
if not origin_urls:
|
||||
return set()
|
||||
with self.session_scope() as session:
|
||||
rows = session.execute(select(Car.origin_url).where(Car.origin_url.in_(origin_urls))).all()
|
||||
return {str(row[0]) for row in rows if row and row[0]}
|
||||
|
||||
def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
|
||||
if not origin_ids:
|
||||
return set()
|
||||
with self.session_scope() as session:
|
||||
rows = session.execute(select(Car.origin_id).where(Car.origin_id.in_(origin_ids))).all()
|
||||
return {str(row[0]) for row in rows if row and row[0]}
|
||||
|
||||
def get_existing_urls_and_ids(
|
||||
self, origin_urls: list[str], origin_ids: list[str],
|
||||
) -> tuple[set[str], set[str]]:
|
||||
# Загрузка URL и origin_id.
|
||||
if not origin_urls and not origin_ids:
|
||||
return set(), set()
|
||||
urls: set[str] = set()
|
||||
ids: set[str] = set()
|
||||
with self.session_scope() as session:
|
||||
max_len = max(len(origin_urls), len(origin_ids), 1)
|
||||
for i in range(0, max_len, _IN_CHUNK_SIZE):
|
||||
url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
|
||||
id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
|
||||
conditions = []
|
||||
if url_chunk:
|
||||
conditions.append(Car.origin_url.in_(url_chunk))
|
||||
if id_chunk:
|
||||
conditions.append(Car.origin_id.in_(id_chunk))
|
||||
if not conditions:
|
||||
continue
|
||||
rows = session.execute(
|
||||
select(Car.origin_url, Car.origin_id).where(or_(*conditions))
|
||||
).all()
|
||||
for r in rows:
|
||||
if r[0]:
|
||||
urls.add(str(r[0]))
|
||||
if r[1]:
|
||||
ids.add(str(r[1]))
|
||||
return urls, ids
|
||||
|
||||
@staticmethod
|
||||
def _add_images(session: Session, car_id: int, images: list[dict[str, object]]) -> None:
|
||||
if not images:
|
||||
return
|
||||
session.add_all([
|
||||
Image(
|
||||
fullres_image=str(img["fullres_image"]),
|
||||
preview_image=str(img["preview_image"]),
|
||||
order_index=int(img.get("order_index", 0)),
|
||||
car_id=car_id,
|
||||
)
|
||||
for img in images
|
||||
])
|
||||
|
||||
@staticmethod
|
||||
def _car_payload(record: CarRecord) -> dict[str, object]:
|
||||
payload = record.model_dump(mode="python")
|
||||
return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
|
||||
|
||||
def _is_postgres(self) -> bool:
|
||||
return self.engine.dialect.name == "postgresql"
|
||||
|
||||
def _load_existing_cars(
|
||||
self,
|
||||
session: Session,
|
||||
origin_ids: list[str],
|
||||
origin_urls: list[str],
|
||||
) -> tuple[dict[str, Car], dict[str, Car]]:
|
||||
existing_by_id: dict[str, Car] = {}
|
||||
existing_by_url: dict[str, Car] = {}
|
||||
if not origin_ids and not origin_urls:
|
||||
return existing_by_id, existing_by_url
|
||||
|
||||
existing_cars: list[Car] = []
|
||||
max_len = max(len(origin_ids), len(origin_urls), 1)
|
||||
for i in range(0, max_len, _IN_CHUNK_SIZE):
|
||||
id_chunk = origin_ids[i:i + _IN_CHUNK_SIZE]
|
||||
url_chunk = origin_urls[i:i + _IN_CHUNK_SIZE]
|
||||
conditions = []
|
||||
if id_chunk:
|
||||
conditions.append(Car.origin_id.in_(id_chunk))
|
||||
if url_chunk:
|
||||
conditions.append(Car.origin_url.in_(url_chunk))
|
||||
if not conditions:
|
||||
continue
|
||||
rows = session.execute(
|
||||
select(Car).where(or_(*conditions))
|
||||
).scalars().all()
|
||||
existing_cars.extend(rows)
|
||||
|
||||
for car in existing_cars:
|
||||
if car.origin_id:
|
||||
existing_by_id[car.origin_id] = car
|
||||
if car.origin_url:
|
||||
existing_by_url[car.origin_url] = car
|
||||
return existing_by_id, existing_by_url
|
||||
|
||||
def _load_existing_image_urls(self, session: Session, car_ids: set[int]) -> dict[int, set[str]]:
|
||||
existing_images_map: dict[int, set[str]] = {}
|
||||
if not car_ids:
|
||||
return existing_images_map
|
||||
|
||||
car_id_list = list(car_ids)
|
||||
for i in range(0, len(car_id_list), _IN_CHUNK_SIZE):
|
||||
chunk = car_id_list[i:i + _IN_CHUNK_SIZE]
|
||||
img_rows = session.execute(
|
||||
select(Image.car_id, Image.fullres_image).where(Image.car_id.in_(chunk))
|
||||
).all()
|
||||
for cid, furl in img_rows:
|
||||
existing_images_map.setdefault(int(cid), set()).add(str(furl))
|
||||
return existing_images_map
|
||||
|
||||
@staticmethod
|
||||
def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
|
||||
return {
|
||||
key: getattr(insert_stmt.excluded, key)
|
||||
for key in CAR_DB_FIELDS
|
||||
}
|
||||
|
||||
def _replace_images_for_car(
|
||||
self,
|
||||
session: Session,
|
||||
car_id: int,
|
||||
images: list[dict[str, object]],
|
||||
origin_id: str,
|
||||
) -> int:
|
||||
nested = session.begin_nested()
|
||||
try:
|
||||
session.execute(delete(Image).where(Image.car_id == car_id))
|
||||
self._add_images(session, car_id, images)
|
||||
session.flush()
|
||||
nested.commit()
|
||||
return len(images)
|
||||
except Exception:
|
||||
nested.rollback()
|
||||
logger.warning("Image replacement failed for car %s, keeping old images", origin_id, exc_info=True)
|
||||
return 0
|
||||
|
||||
def _upsert_cars_batch_postgres(self, records: list[CarRecord]) -> dict[str, int]:
|
||||
inserted = 0
|
||||
updated = 0
|
||||
images_total = 0
|
||||
|
||||
with self.session_scope() as session:
|
||||
origin_ids = [r.origin_id for r in records if r.origin_id]
|
||||
origin_urls = [r.origin_url for r in records if r.origin_url]
|
||||
existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
|
||||
|
||||
entries: list[dict[str, object]] = []
|
||||
upsert_payloads: list[dict[str, object]] = []
|
||||
for record in records:
|
||||
payload = self._car_payload(record)
|
||||
images = [image.model_dump(mode="python") for image in record.images]
|
||||
car_by_id = existing_by_id.get(record.origin_id)
|
||||
car_by_url = existing_by_url.get(record.origin_url)
|
||||
entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
|
||||
|
||||
if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
|
||||
for key, value in payload.items():
|
||||
setattr(car_by_url, key, value)
|
||||
car_by_url.last_seen_at = record.last_seen_at
|
||||
entry["car_id"] = int(car_by_url.id)
|
||||
updated += 1
|
||||
else:
|
||||
upsert_payloads.append(payload)
|
||||
if car_by_id is not None:
|
||||
updated += 1
|
||||
else:
|
||||
inserted += 1
|
||||
entries.append(entry)
|
||||
|
||||
session.flush()
|
||||
|
||||
if upsert_payloads:
|
||||
insert_stmt = pg_insert(Car).values(upsert_payloads)
|
||||
upsert_stmt = insert_stmt.on_conflict_do_update(
|
||||
index_elements=[Car.origin_id],
|
||||
set_=self._postgres_upsert_set_map(insert_stmt),
|
||||
).returning(Car.id, Car.origin_id)
|
||||
rows = session.execute(upsert_stmt).all()
|
||||
car_ids_by_origin_id = {str(origin_id): int(car_id) for car_id, origin_id in rows}
|
||||
for entry in entries:
|
||||
if entry["car_id"] is not None:
|
||||
continue
|
||||
record = entry["record"]
|
||||
car_id = car_ids_by_origin_id.get(record.origin_id)
|
||||
if car_id is None:
|
||||
raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
|
||||
entry["car_id"] = car_id
|
||||
|
||||
car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
|
||||
existing_images_map = self._load_existing_image_urls(session, car_ids)
|
||||
images_by_car_id: dict[int, list[dict[str, object]]] = {}
|
||||
replace_ids: list[int] = []
|
||||
|
||||
for entry in entries:
|
||||
car_id = int(entry["car_id"])
|
||||
images = entry["images"]
|
||||
new_image_urls = {
|
||||
str(img.get("fullres_image", ""))
|
||||
for img in images
|
||||
if img.get("fullres_image")
|
||||
}
|
||||
old_image_urls = existing_images_map.get(car_id, set())
|
||||
if new_image_urls != old_image_urls:
|
||||
replace_ids.append(car_id)
|
||||
images_by_car_id[car_id] = images
|
||||
else:
|
||||
images_total += len(old_image_urls)
|
||||
|
||||
if replace_ids:
|
||||
for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
|
||||
chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
|
||||
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
|
||||
for car_id in replace_ids:
|
||||
images = images_by_car_id[car_id]
|
||||
self._add_images(session, car_id, images)
|
||||
images_total += len(images)
|
||||
|
||||
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
|
||||
|
||||
def upsert_car(self, record: CarRecord):
|
||||
# Вставка или обновление авто.
|
||||
payload = self._car_payload(record)
|
||||
images = [image.model_dump(mode="python") for image in record.images]
|
||||
with self.session_scope() as session:
|
||||
if self._is_postgres():
|
||||
car_by_url = session.execute(
|
||||
select(Car).where(Car.origin_url == record.origin_url)
|
||||
).scalar_one_or_none()
|
||||
if car_by_url is not None and car_by_url.origin_id != record.origin_id:
|
||||
for key, value in payload.items():
|
||||
setattr(car_by_url, key, value)
|
||||
car_by_url.last_seen_at = record.last_seen_at
|
||||
session.flush()
|
||||
car_id = int(car_by_url.id)
|
||||
action = "updated"
|
||||
else:
|
||||
existed = session.execute(
|
||||
select(Car.id).where(Car.origin_id == record.origin_id)
|
||||
).scalar_one_or_none() is not None
|
||||
insert_stmt = pg_insert(Car).values(**payload)
|
||||
upsert_stmt = insert_stmt.on_conflict_do_update(
|
||||
index_elements=[Car.origin_id],
|
||||
set_=self._postgres_upsert_set_map(insert_stmt),
|
||||
).returning(Car.id)
|
||||
car_id = int(session.execute(upsert_stmt).scalar_one())
|
||||
action = "updated" if existed or car_by_url is not None else "inserted"
|
||||
|
||||
images_upserted = self._replace_images_for_car(
|
||||
session, car_id, images, record.origin_id,
|
||||
)
|
||||
return {"car_id": car_id, "images_upserted": images_upserted, "action": action}
|
||||
|
||||
car = session.execute(
|
||||
select(Car).where(or_(Car.origin_id == record.origin_id, Car.origin_url == record.origin_url))
|
||||
).scalar_one_or_none()
|
||||
action = "inserted"
|
||||
if car is None:
|
||||
car = Car(**payload)
|
||||
session.add(car)
|
||||
session.flush()
|
||||
else:
|
||||
action = "updated"
|
||||
for key, value in payload.items():
|
||||
setattr(car, key, value)
|
||||
car.last_seen_at = record.last_seen_at
|
||||
session.flush()
|
||||
images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
|
||||
return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
|
||||
self._add_images(session, int(car.id), images)
|
||||
session.flush()
|
||||
return {"car_id": int(car.id), "images_upserted": len(images), "action": action}
|
||||
def upsert_cars_batch(self, records: list[CarRecord]) -> dict[str, int]:
|
||||
"""Пакетный upsert нескольких автомобилей в одной транзакции.
|
||||
|
||||
Оптимизации:
|
||||
- Дедупликация записей по origin_id перед вставкой.
|
||||
- Chunked IN-queries для больших списков (обход лимита PG параметров).
|
||||
- Пропуск перезаписи изображений, если набор URL не изменился.
|
||||
- Один DELETE по car_id IN (...) вместо удаления по одному.
|
||||
- Fallback на по-одному upsert если batch commit упал.
|
||||
"""
|
||||
# Дедупликация батча.
|
||||
seen_ids: dict[str, int] = {}
|
||||
unique_records: list[CarRecord] = []
|
||||
for idx, r in enumerate(records):
|
||||
key = r.origin_id or r.origin_url
|
||||
if key in seen_ids:
|
||||
logger.debug("Dedup: skipping duplicate record %s (idx %d vs %d)", key, idx, seen_ids[key])
|
||||
continue
|
||||
seen_ids[key] = idx
|
||||
unique_records.append(r)
|
||||
|
||||
if len(unique_records) < len(records):
|
||||
logger.info("Deduped batch: %d → %d records", len(records), len(unique_records))
|
||||
records = unique_records
|
||||
|
||||
try:
|
||||
return self._upsert_cars_batch_inner(records)
|
||||
except Exception as exc:
|
||||
logger.warning("Batch upsert failed (%s), falling back to individual upserts", exc)
|
||||
return self._upsert_cars_individually(records)
|
||||
|
||||
def _upsert_cars_batch_inner(self, records: list[CarRecord]) -> dict[str, int]:
|
||||
"""Внутренняя реализация batched upsert (одна транзакция)."""
|
||||
if self._is_postgres():
|
||||
return self._upsert_cars_batch_postgres(records)
|
||||
|
||||
inserted = 0
|
||||
updated = 0
|
||||
images_total = 0
|
||||
|
||||
with self.session_scope() as session:
|
||||
# Загружаем существующие записи.
|
||||
origin_ids = [r.origin_id for r in records if r.origin_id]
|
||||
origin_urls = [r.origin_url for r in records if r.origin_url]
|
||||
|
||||
existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
|
||||
|
||||
# Предзагружаем изображения.
|
||||
existing_car_ids = set()
|
||||
for record in records:
|
||||
car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
|
||||
if car is not None:
|
||||
existing_car_ids.add(int(car.id))
|
||||
|
||||
# Готовим map car_id -> image_urls.
|
||||
existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
|
||||
|
||||
new_cars: list[tuple[Car, list[dict]]] = []
|
||||
update_cars_needing_images: list[tuple[Car, list[dict]]] = []
|
||||
|
||||
for record in records:
|
||||
payload = self._car_payload(record)
|
||||
images = [image.model_dump(mode="python") for image in record.images]
|
||||
|
||||
car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
|
||||
if car is None:
|
||||
car = Car(**payload)
|
||||
session.add(car)
|
||||
inserted += 1
|
||||
new_cars.append((car, images))
|
||||
else:
|
||||
for key, value in payload.items():
|
||||
setattr(car, key, value)
|
||||
car.last_seen_at = record.last_seen_at
|
||||
updated += 1
|
||||
|
||||
# Проверяем изменения картинок.
|
||||
new_image_urls = {img.get("fullres_image", "") for img in images}
|
||||
old_image_urls = existing_images_map.get(int(car.id), set())
|
||||
if new_image_urls != old_image_urls:
|
||||
update_cars_needing_images.append((car, images))
|
||||
else:
|
||||
images_total += len(old_image_urls)
|
||||
|
||||
# Один flush.
|
||||
session.flush()
|
||||
|
||||
# Добавляем картинки новым авто.
|
||||
for car, images in new_cars:
|
||||
self._add_images(session, int(car.id), images)
|
||||
images_total += len(images)
|
||||
|
||||
# Обновляем только изменённые картинки.
|
||||
if update_cars_needing_images:
|
||||
update_ids = [int(car.id) for car, _ in update_cars_needing_images]
|
||||
for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
|
||||
chunk = update_ids[i:i + _IN_CHUNK_SIZE]
|
||||
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
|
||||
for car, images in update_cars_needing_images:
|
||||
self._add_images(session, int(car.id), images)
|
||||
images_total += len(images)
|
||||
|
||||
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
|
||||
|
||||
def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
|
||||
# Запасной поштучный upsert.
|
||||
inserted = 0
|
||||
updated = 0
|
||||
images_total = 0
|
||||
for record in records:
|
||||
try:
|
||||
result = self.upsert_car(record)
|
||||
action = result.get("action", "inserted")
|
||||
if action == "inserted":
|
||||
inserted += 1
|
||||
else:
|
||||
updated += 1
|
||||
images_total += int(result.get("images_upserted", 0))
|
||||
except Exception as exc:
|
||||
logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
|
||||
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
|
||||
|
||||
def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "dubizzle") -> int:
|
||||
"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
|
||||
if not active_origin_ids:
|
||||
return 0
|
||||
with self.session_scope() as session:
|
||||
stmt = (
|
||||
update(Car)
|
||||
.where(Car.origin_id.notin_(active_origin_ids))
|
||||
.where(Car.is_sold == False) # noqa: E712
|
||||
.where(Car.origin_id.like("dubizzle:%"))
|
||||
.values(is_sold=True)
|
||||
)
|
||||
result = session.execute(stmt)
|
||||
count = result.rowcount or 0
|
||||
if count:
|
||||
logger.info("Marked %d cars as sold (no longer in listing)", count)
|
||||
return count
|
||||
|
||||
def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "dubizzle") -> int:
|
||||
"""Помечает авто как проданные, если их URL нет в активном листинге.
|
||||
|
||||
Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
|
||||
что кардинально быстрее при больших объёмах (100K+ URLs).
|
||||
Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент.
|
||||
"""
|
||||
if not active_origin_urls:
|
||||
return 0
|
||||
|
||||
# Нормализация URL.
|
||||
def _norm(url: str) -> str:
|
||||
return url.replace("~", "-")
|
||||
|
||||
normalized_urls = {_norm(u) for u in active_origin_urls}
|
||||
|
||||
is_postgres = "postgresql" in self.settings.database.url
|
||||
|
||||
with self.session_scope() as session:
|
||||
if is_postgres:
|
||||
# Считаем кандидатов на sold.
|
||||
total_active = session.execute(
|
||||
text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%%'")
|
||||
).scalar() or 0
|
||||
|
||||
if total_active == 0:
|
||||
return 0
|
||||
|
||||
# Временная таблица URL.
|
||||
session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
|
||||
session.execute(text("TRUNCATE _active_urls"))
|
||||
|
||||
# Вставляем URL чанками.
|
||||
url_list = list(normalized_urls)
|
||||
for i in range(0, len(url_list), _IN_CHUNK_SIZE):
|
||||
chunk = url_list[i:i + _IN_CHUNK_SIZE]
|
||||
values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
|
||||
params = {f"u{j}": url for j, url in enumerate(chunk)}
|
||||
session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
|
||||
|
||||
# Индекс для JOIN.
|
||||
session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
|
||||
|
||||
# Считаем будущие sold.
|
||||
would_mark = session.execute(text("""
|
||||
SELECT count(*)
|
||||
FROM cars c
|
||||
LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
|
||||
WHERE a.url IS NULL
|
||||
AND c.is_sold = FALSE
|
||||
AND c.origin_id LIKE 'dubizzle:%%'
|
||||
""")).scalar() or 0
|
||||
|
||||
# Защита от аномалии.
|
||||
if total_active > 100 and would_mark > total_active * 0.8:
|
||||
logger.error(
|
||||
"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
|
||||
would_mark, total_active, would_mark / total_active * 100,
|
||||
)
|
||||
return 0
|
||||
|
||||
# Массовая пометка sold.
|
||||
result = session.execute(text("""
|
||||
UPDATE cars
|
||||
SET is_sold = TRUE
|
||||
FROM (
|
||||
SELECT c.id
|
||||
FROM cars c
|
||||
LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
|
||||
WHERE a.url IS NULL
|
||||
AND c.is_sold = FALSE
|
||||
AND c.origin_id LIKE 'dubizzle:%%'
|
||||
) sub
|
||||
WHERE cars.id = sub.id
|
||||
"""))
|
||||
count = result.rowcount or 0
|
||||
else:
|
||||
# Упрощённый путь для SQLite.
|
||||
stmt = (
|
||||
update(Car)
|
||||
.where(Car.is_sold == False) # noqa: E712
|
||||
.where(Car.origin_id.like("dubizzle:%"))
|
||||
.values(is_sold=True)
|
||||
)
|
||||
# Загружаем active URL.
|
||||
all_active = session.execute(
|
||||
select(Car.id, Car.origin_url).where(
|
||||
Car.is_sold == False, Car.origin_id.like("dubizzle:%") # noqa: E712
|
||||
)
|
||||
).all()
|
||||
mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
|
||||
|
||||
if not mark_ids:
|
||||
return 0
|
||||
total_active = len(all_active)
|
||||
if total_active > 100 and len(mark_ids) > total_active * 0.8:
|
||||
logger.error(
|
||||
"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
|
||||
len(mark_ids), total_active, len(mark_ids) / total_active * 100,
|
||||
)
|
||||
return 0
|
||||
|
||||
for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
|
||||
chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
|
||||
session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
|
||||
count = len(mark_ids)
|
||||
|
||||
if count:
|
||||
logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
|
||||
return count
|
||||
|
||||
def get_all_origin_ids_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
|
||||
"""Возвращает все известные origin_id для заданного префикса.
|
||||
|
||||
Использует yield_per для потоковой загрузки при большом количестве записей.
|
||||
"""
|
||||
with self.session_scope() as session:
|
||||
result = session.execute(
|
||||
select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
|
||||
)
|
||||
return {str(row[0]) for row in result if row and row[0]}
|
||||
|
||||
def get_all_active_origin_urls_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
|
||||
"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
|
||||
with self.session_scope() as session:
|
||||
result = session.execute(
|
||||
select(Car.origin_url).where(
|
||||
Car.origin_id.like(f"{prefix}%"),
|
||||
Car.is_sold == False, # noqa: E712
|
||||
).execution_options(yield_per=10000)
|
||||
)
|
||||
return {str(row[0]) for row in result if row and row[0]}
|
||||
|
||||
def count_active_cars_for_lane(self, prefix: str = "dubizzle:") -> int:
|
||||
"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
|
||||
from sqlalchemy import func as sa_func
|
||||
with self.session_scope() as session:
|
||||
result = session.execute(
|
||||
select(sa_func.count()).select_from(Car).where(
|
||||
Car.origin_id.like(f"{prefix}%"),
|
||||
Car.is_sold == False, # noqa: E712
|
||||
)
|
||||
)
|
||||
return int(result.scalar() or 0)
|
||||
|
||||
def get_active_origin_urls_batch_for_refresh(
|
||||
self,
|
||||
prefix: str = "dubizzle:",
|
||||
offset: int = 0,
|
||||
limit: int = 500,
|
||||
) -> list[str]:
|
||||
"""Возвращает батч origin_url активных авто для rolling refresh.
|
||||
|
||||
Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
|
||||
"""
|
||||
with self.session_scope() as session:
|
||||
result = session.execute(
|
||||
select(Car.origin_url).where(
|
||||
Car.origin_id.like(f"{prefix}%"),
|
||||
Car.is_sold == False, # noqa: E712
|
||||
).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
|
||||
)
|
||||
return [str(row[0]) for row in result if row and row[0]]
|
||||
24
dubizzle_scraper/storage/enums.py
Normal file
24
dubizzle_scraper/storage/enums.py
Normal file
@@ -0,0 +1,24 @@
|
||||
CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
|
||||
DRIVE_ENUM_VALUES = ("FWD", "RWD", "2WD", "4WD", "NA")
|
||||
GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
|
||||
STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "NA")
|
||||
BODY_TYPE_ENUM_VALUES = (
|
||||
"COUPE",
|
||||
"SUV",
|
||||
"HATCHBACK",
|
||||
"MINIVAN",
|
||||
"SEDAN",
|
||||
"STATION_WAGON",
|
||||
"PICKUP",
|
||||
"TRUCK",
|
||||
"OPEN",
|
||||
"RV",
|
||||
"OTHER",
|
||||
"NA",
|
||||
)
|
||||
COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "AE", "NA")
|
||||
ORIGIN_ENUM_VALUES = (
|
||||
"DUBIZZLE",
|
||||
"NA",
|
||||
)
|
||||
SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")
|
||||
82
dubizzle_scraper/storage/models.py
Normal file
82
dubizzle_scraper/storage/models.py
Normal file
@@ -0,0 +1,82 @@
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import BigInteger, Boolean, DateTime, Enum, ForeignKey, Index, Integer, String, Text, func
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
|
||||
|
||||
from .enums import (
|
||||
BODY_TYPE_ENUM_VALUES,
|
||||
COUNTRY_ENUM_VALUES,
|
||||
CURRENCY_ENUM_VALUES,
|
||||
DRIVE_ENUM_VALUES,
|
||||
GEARBOX_ENUM_VALUES,
|
||||
ORIGIN_ENUM_VALUES,
|
||||
SELLING_TYPE_ENUM_VALUES,
|
||||
STEERING_WHEEL_ENUM_VALUES,
|
||||
)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class Car(Base):
|
||||
__tablename__ = "cars"
|
||||
__table_args__ = (
|
||||
Index("ix_cars_brand_model", "brand", "model"),
|
||||
Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
|
||||
)
|
||||
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
|
||||
parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
|
||||
brand: Mapped[str] = mapped_column(String(50), nullable=False, index=True)
|
||||
model: Mapped[str] = mapped_column(String(50), nullable=False)
|
||||
year: Mapped[int | None] = mapped_column(Integer, nullable=True, index=True)
|
||||
price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
|
||||
currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
|
||||
mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
|
||||
is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
|
||||
color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
|
||||
drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
|
||||
gearbox: Mapped[str | None] = mapped_column(Enum(*GEARBOX_ENUM_VALUES, name="gearboxenum", native_enum=False, create_constraint=False), nullable=True)
|
||||
steering_wheel: Mapped[str | None] = mapped_column(Enum(*STEERING_WHEEL_ENUM_VALUES, name="steeringwheelenum", native_enum=False, create_constraint=False), nullable=True)
|
||||
body_type: Mapped[str] = mapped_column(Enum(*BODY_TYPE_ENUM_VALUES, name="bodytypeenum", native_enum=False, create_constraint=False), nullable=False, default="OTHER")
|
||||
engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
|
||||
selling_type: Mapped[str] = mapped_column(Enum(*SELLING_TYPE_ENUM_VALUES, name="sellingtypeenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
|
||||
one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
origin: Mapped[str] = mapped_column(Enum(*ORIGIN_ENUM_VALUES, name="originenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
|
||||
origin_url: Mapped[str] = mapped_column(String(), nullable=False, index=True)
|
||||
origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
|
||||
is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
|
||||
non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
|
||||
rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
|
||||
slug: Mapped[str] = mapped_column(String(), nullable=False)
|
||||
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
|
||||
images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")
|
||||
|
||||
|
||||
class Image(Base):
|
||||
__tablename__ = "images"
|
||||
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
|
||||
fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
|
||||
preview_image: Mapped[str] = mapped_column(String(), nullable=False)
|
||||
order_index: Mapped[int] = mapped_column(Integer, nullable=False)
|
||||
car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
|
||||
car: Mapped[Car] = relationship("Car", back_populates="images")
|
||||
|
||||
|
||||
class SyncRun(Base):
|
||||
__tablename__ = "sync_runs"
|
||||
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
|
||||
status: Mapped[str] = mapped_column(Text, nullable=False, index=True)
|
||||
lane: Mapped[str] = mapped_column(Text, nullable=False)
|
||||
ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
|
||||
error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
|
||||
87
dubizzle_scraper/storage/schemas.py
Normal file
87
dubizzle_scraper/storage/schemas.py
Normal file
@@ -0,0 +1,87 @@
|
||||
from datetime import datetime, timezone
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
|
||||
class ImageRecord(BaseModel):
|
||||
fullres_image: str
|
||||
preview_image: str
|
||||
order_index: int = 0
|
||||
|
||||
|
||||
class CarRecord(BaseModel):
|
||||
parser_id: str
|
||||
brand: str
|
||||
model: str
|
||||
year: int | None = None
|
||||
price: int | None = None
|
||||
currency: str = "USD"
|
||||
mileage: int = 0
|
||||
country: str = "US"
|
||||
is_sold: bool = False
|
||||
color: str = "other"
|
||||
drive: str | None = None
|
||||
gearbox: str | None = None
|
||||
steering_wheel: str | None = None
|
||||
body_type: str = "OTHER"
|
||||
engine_volume: int | None = None
|
||||
selling_type: str = "AUCTION"
|
||||
one_owner: bool = False
|
||||
new_car: bool = False
|
||||
is_hidden: bool = False
|
||||
origin: str = "NA"
|
||||
origin_url: str
|
||||
origin_id: str
|
||||
is_damaged: bool = False
|
||||
evaluation: str | None = None
|
||||
non_smoking: bool = True
|
||||
rental: bool = False
|
||||
repair_history: bool = False
|
||||
slug: str
|
||||
last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
|
||||
images: list[ImageRecord] = Field(default_factory=list)
|
||||
|
||||
|
||||
class ImageRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: int
|
||||
fullres_image: str
|
||||
preview_image: str
|
||||
order_index: int = 0
|
||||
|
||||
|
||||
class CarRead(BaseModel):
|
||||
model_config = ConfigDict(from_attributes=True)
|
||||
|
||||
id: int
|
||||
parser_id: str
|
||||
brand: str
|
||||
model: str
|
||||
year: int | None = None
|
||||
price: int | None = None
|
||||
currency: str = "USD"
|
||||
mileage: int = 0
|
||||
country: str = "US"
|
||||
is_sold: bool = False
|
||||
color: str = "other"
|
||||
drive: str | None = None
|
||||
gearbox: str | None = None
|
||||
steering_wheel: str | None = None
|
||||
body_type: str = "OTHER"
|
||||
engine_volume: int | None = None
|
||||
selling_type: str = "AUCTION"
|
||||
one_owner: bool = False
|
||||
new_car: bool = False
|
||||
is_hidden: bool = False
|
||||
origin: str = "NA"
|
||||
origin_url: str = ""
|
||||
origin_id: str = ""
|
||||
is_damaged: bool = False
|
||||
evaluation: str | None = None
|
||||
non_smoking: bool = True
|
||||
rental: bool = False
|
||||
repair_history: bool = False
|
||||
slug: str = ""
|
||||
last_seen_at: datetime | None = None
|
||||
images: list[ImageRead] = Field(default_factory=list)
|
||||
|
||||
3
dubizzle_scraper/worker/__init__.py
Normal file
3
dubizzle_scraper/worker/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from .celery_app import celery_app
|
||||
|
||||
__all__ = ["celery_app"]
|
||||
148
dubizzle_scraper/worker/celery_app.py
Normal file
148
dubizzle_scraper/worker/celery_app.py
Normal file
@@ -0,0 +1,148 @@
|
||||
# Инициализация Celery-приложения и периодических задач.
|
||||
|
||||
import logging
|
||||
|
||||
from celery import Celery
|
||||
from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
|
||||
from redis import Redis
|
||||
|
||||
from ..core.config import settings
|
||||
from ..core.logs import setup_logging
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.worker.celery_app")
|
||||
|
||||
|
||||
@celery_setup_logging.connect
|
||||
def _configure_logging(loglevel=None, **kwargs):
|
||||
# Перехватываем логирование Celery и пишем только в stderr (Docker logs).
|
||||
level = settings.log_level if settings.log_level else "INFO"
|
||||
setup_logging(level, None)
|
||||
|
||||
|
||||
@worker_process_init.connect
|
||||
def _on_worker_process_init(**kwargs):
|
||||
# Повторно настраиваем логирование в каждом дочернем prefork-процессе,
|
||||
# чтобы StreamHandler(stderr) корректно работал после fork.
|
||||
level = settings.log_level if settings.log_level else "INFO"
|
||||
setup_logging(level, None)
|
||||
|
||||
|
||||
def _broker_url() -> str:
|
||||
return settings.celery.broker_url or settings.redis.url
|
||||
|
||||
|
||||
def _result_backend() -> str:
|
||||
return settings.celery.result_backend or settings.redis.url
|
||||
|
||||
|
||||
celery_app = Celery(
|
||||
"dubizzle_scraper",
|
||||
broker=_broker_url(),
|
||||
backend=_result_backend(),
|
||||
)
|
||||
|
||||
# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
|
||||
# ограничиваем, чтобы зависший worker не жил вечно.
|
||||
_soft = settings.celery.task_soft_time_limit
|
||||
_hard = settings.celery.task_time_limit
|
||||
_max_hard = _soft + 120 if _soft else _hard
|
||||
if _hard > _max_hard:
|
||||
logger.warning(
|
||||
"CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
|
||||
"clamping hard limit to %d",
|
||||
_hard, _soft, _max_hard,
|
||||
)
|
||||
_hard = _max_hard
|
||||
|
||||
celery_app.conf.update(
|
||||
task_serializer="json",
|
||||
accept_content=["json"],
|
||||
result_serializer="json",
|
||||
timezone="UTC",
|
||||
enable_utc=True,
|
||||
task_soft_time_limit=_soft,
|
||||
task_time_limit=_hard,
|
||||
task_acks_late=True,
|
||||
task_reject_on_worker_lost=True,
|
||||
task_track_started=True,
|
||||
worker_concurrency=settings.celery.worker_concurrency,
|
||||
worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
|
||||
worker_pool="solo",
|
||||
worker_prefetch_multiplier=1,
|
||||
broker_connection_retry_on_startup=True,
|
||||
broker_transport_options={
|
||||
"visibility_timeout": settings.celery.broker_visibility_timeout,
|
||||
},
|
||||
result_expires=86400,
|
||||
worker_redirect_stdouts=False,
|
||||
worker_hijack_root_logger=False,
|
||||
beat_schedule={
|
||||
"periodic-sync-listing": {
|
||||
"task": "dubizzle_scraper.worker.tasks.sync_listing_task",
|
||||
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
|
||||
"args": (),
|
||||
"kwargs": {
|
||||
"limit": settings.celery.beat_sync_limit,
|
||||
"only_new": False if settings.discovery.always_full_scan else True,
|
||||
},
|
||||
"options": {
|
||||
"queue": "scraping",
|
||||
"expires": settings.celery.beat_sync_interval_minutes * 60.0,
|
||||
},
|
||||
}
|
||||
},
|
||||
task_routes={
|
||||
"dubizzle_scraper.worker.tasks.*": {"queue": "scraping"}
|
||||
},
|
||||
)
|
||||
|
||||
celery_app.autodiscover_tasks(["dubizzle_scraper.worker"])
|
||||
|
||||
|
||||
@worker_ready.connect
|
||||
def _on_worker_ready(**kwargs):
|
||||
"""При старте worker отправляем первый sync_listing, если очередь пуста."""
|
||||
redis_client = None
|
||||
try:
|
||||
redis_client = Redis.from_url(
|
||||
settings.redis.url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=settings.redis.socket_connect_timeout_seconds,
|
||||
socket_timeout=settings.redis.socket_timeout_seconds,
|
||||
health_check_interval=settings.redis.health_check_interval_seconds,
|
||||
retry_on_timeout=True,
|
||||
)
|
||||
|
||||
for stale_key in ("dubizzle:locks:sync_listing",):
|
||||
try:
|
||||
ttl = redis_client.ttl(stale_key)
|
||||
if ttl is not None and ttl != -2:
|
||||
redis_client.delete(stale_key)
|
||||
logger.warning("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
|
||||
except Exception:
|
||||
logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
|
||||
|
||||
try:
|
||||
queue_len = int(redis_client.llen("scraping") or 0)
|
||||
except Exception:
|
||||
queue_len = 0
|
||||
if queue_len > 0:
|
||||
logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len)
|
||||
return
|
||||
except Exception:
|
||||
logger.warning("Worker ready startup sync check failed; skipping immediate dispatch", exc_info=True)
|
||||
return
|
||||
finally:
|
||||
if redis_client is not None:
|
||||
try:
|
||||
redis_client.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
logger.info("Worker ready — dispatching initial sync_listing task")
|
||||
celery_app.send_task(
|
||||
"dubizzle_scraper.worker.tasks.sync_listing_task",
|
||||
kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
|
||||
queue="scraping",
|
||||
expires=settings.celery.beat_sync_interval_minutes * 60.0,
|
||||
)
|
||||
209
dubizzle_scraper/worker/self_heal.py
Normal file
209
dubizzle_scraper/worker/self_heal.py
Normal file
@@ -0,0 +1,209 @@
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import signal
|
||||
import time
|
||||
|
||||
from redis import Redis
|
||||
|
||||
|
||||
logger = logging.getLogger("dubizzle_scraper.worker.self_heal")
|
||||
|
||||
GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts"
|
||||
SELF_HEAL_RESTART_LOCK_KEY = "dubizzle:state:self_heal_restart_in_progress"
|
||||
|
||||
|
||||
def _env_str(name: str, default: str) -> str:
|
||||
value = os.getenv(name)
|
||||
if value is None and name.startswith("DUBIZZLE_"):
|
||||
value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):])
|
||||
return value if value is not None else default
|
||||
|
||||
|
||||
def _env_bool(name: str, default: bool) -> bool:
|
||||
value = _env_str(name, "true" if default else "false")
|
||||
if value is None:
|
||||
return default
|
||||
return value.strip().lower() in {"1", "true", "yes", "on"}
|
||||
|
||||
|
||||
def _env_int(name: str, default: int) -> int:
|
||||
value = _env_str(name, str(default))
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
return int(value.strip())
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _get_redis() -> Redis:
|
||||
url = _env_str("DUBIZZLE_REDIS_URL", "redis://redis:6379/0")
|
||||
return Redis.from_url(
|
||||
url,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=5.0,
|
||||
socket_timeout=10.0,
|
||||
health_check_interval=30,
|
||||
retry_on_timeout=True,
|
||||
)
|
||||
|
||||
|
||||
def _safe_int(value: str | None, default: int = 0) -> int:
|
||||
if value is None:
|
||||
return default
|
||||
try:
|
||||
return int(str(value).strip())
|
||||
except Exception:
|
||||
return default
|
||||
|
||||
|
||||
def _read_last_progress_ts(redis_client: Redis) -> int | None:
|
||||
raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY)
|
||||
if raw:
|
||||
ts = _safe_int(raw)
|
||||
if ts > 0:
|
||||
return ts
|
||||
|
||||
# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
|
||||
# Это дороже, но выполняется только при отсутствии основного маркера.
|
||||
max_ts = 0
|
||||
for key in redis_client.scan_iter(match="dubizzle:state:task_progress:*"):
|
||||
try:
|
||||
payload = redis_client.get(key)
|
||||
if not payload:
|
||||
continue
|
||||
data = json.loads(payload)
|
||||
ts = _safe_int(data.get("ts"), 0)
|
||||
if ts > max_ts:
|
||||
max_ts = ts
|
||||
except Exception:
|
||||
continue
|
||||
return max_ts or None
|
||||
|
||||
|
||||
def _kill_worker_process() -> None:
|
||||
pid_file = "/tmp/celery-worker.pid"
|
||||
pid: int | None = None
|
||||
try:
|
||||
with open(pid_file, "r", encoding="utf-8") as f:
|
||||
pid = int(f.read().strip())
|
||||
except Exception:
|
||||
pid = None
|
||||
|
||||
if not pid:
|
||||
logger.error("Self-heal: failed to read worker pid from %s", pid_file)
|
||||
return
|
||||
|
||||
logger.error("Self-heal: terminating stuck worker process pid=%s", pid)
|
||||
try:
|
||||
os.kill(pid, signal.SIGTERM)
|
||||
except Exception:
|
||||
logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid)
|
||||
return
|
||||
|
||||
time.sleep(20)
|
||||
try:
|
||||
# Если процесс ещё жив — принудительно убиваем.
|
||||
os.kill(pid, 0)
|
||||
logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
|
||||
kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM)
|
||||
os.kill(pid, kill_signal)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
except Exception:
|
||||
logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
if not _env_bool("DUBIZZLE_SELF_HEAL_ENABLED", True):
|
||||
logger.info("Self-heal watchdog disabled via DUBIZZLE_SELF_HEAL_ENABLED/DUBIZZLE_SELF_HEAL_ENABLED")
|
||||
return
|
||||
|
||||
queue_name = _env_str("DUBIZZLE_CELERY_QUEUE", "scraping")
|
||||
check_interval = max(5, _env_int("DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
|
||||
stall_seconds = max(180, _env_int("DUBIZZLE_SELF_HEAL_STALL_SECONDS", 720))
|
||||
startup_grace = max(30, _env_int("DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
|
||||
restart_cooldown = max(60, _env_int("DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
|
||||
|
||||
logger.warning(
|
||||
"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
|
||||
queue_name,
|
||||
check_interval,
|
||||
stall_seconds,
|
||||
startup_grace,
|
||||
restart_cooldown,
|
||||
)
|
||||
|
||||
started_at = time.time()
|
||||
redis_client: Redis | None = None
|
||||
|
||||
while True:
|
||||
try:
|
||||
if redis_client is None:
|
||||
redis_client = _get_redis()
|
||||
redis_client.ping()
|
||||
|
||||
queue_len = _safe_int(redis_client.llen(queue_name), 0)
|
||||
if queue_len <= 0:
|
||||
time.sleep(check_interval)
|
||||
continue
|
||||
|
||||
last_progress_ts = _read_last_progress_ts(redis_client)
|
||||
now_ts = int(time.time())
|
||||
age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts))
|
||||
|
||||
if age is None:
|
||||
if now_ts - int(started_at) < startup_grace:
|
||||
time.sleep(check_interval)
|
||||
continue
|
||||
logger.warning(
|
||||
"Self-heal: queue=%d but no progress timestamp found after startup grace",
|
||||
queue_len,
|
||||
)
|
||||
age = stall_seconds + 1
|
||||
|
||||
if age <= stall_seconds:
|
||||
time.sleep(check_interval)
|
||||
continue
|
||||
|
||||
# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
|
||||
acquired = bool(
|
||||
redis_client.set(
|
||||
SELF_HEAL_RESTART_LOCK_KEY,
|
||||
str(now_ts),
|
||||
nx=True,
|
||||
ex=restart_cooldown,
|
||||
)
|
||||
)
|
||||
if not acquired:
|
||||
time.sleep(check_interval)
|
||||
continue
|
||||
|
||||
logger.error(
|
||||
"Self-heal: detected global stall (queue=%d, progress_age=%ss > %ss). Restarting worker process...",
|
||||
queue_len,
|
||||
age,
|
||||
stall_seconds,
|
||||
)
|
||||
# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
|
||||
# перезапуск был не строго одновременно.
|
||||
time.sleep(random.uniform(0.3, 2.0))
|
||||
_kill_worker_process()
|
||||
# После kill pid1 контейнер будет перезапущен Docker restart-policy.
|
||||
# На случай неуспеха не молотим цикл.
|
||||
time.sleep(check_interval)
|
||||
|
||||
except Exception:
|
||||
logger.exception("Self-heal watchdog iteration failed")
|
||||
redis_client = None
|
||||
time.sleep(check_interval)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
logging.basicConfig(
|
||||
level=_env_str("DUBIZZLE_LOG_LEVEL", "INFO"),
|
||||
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
|
||||
)
|
||||
main()
|
||||
1848
dubizzle_scraper/worker/tasks.py
Normal file
1848
dubizzle_scraper/worker/tasks.py
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user