Prepare mobile de parser preview
This commit is contained in:
qananasikq
3
iaai_scraper/mobile_de/__init__.py
Normal file
3
iaai_scraper/mobile_de/__init__.py
Normal file
@@ -0,0 +1,3 @@
|
||||
from .client import MobileDeClient
|
||||
from .scraper import MobileDeScraper
|
||||
|
||||
250
iaai_scraper/mobile_de/client.py
Normal file
250
iaai_scraper/mobile_de/client.py
Normal file
@@ -0,0 +1,250 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from collections.abc import Callable, Iterable
|
||||
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
|
||||
|
||||
import requests
|
||||
|
||||
from .flight import extract_detail_listing, extract_search_results
|
||||
from .models import MobileDeListing, MobileDeSearchPage
|
||||
|
||||
logger = logging.getLogger("mobile_de.client")
|
||||
|
||||
BASE_URL = "https://www.mobile.de"
|
||||
SEARCH_PATH = "/ru/транспортные-средства/поиск.html"
|
||||
DETAIL_PATH = "/ru/транспортные-средства/подробности.html"
|
||||
DEFAULT_HEADERS = {
|
||||
"user-agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/124.0.0.0 Safari/537.36"
|
||||
),
|
||||
"accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
|
||||
"accept-language": "ru,en;q=0.9,de;q=0.8",
|
||||
}
|
||||
|
||||
|
||||
class MobileDeClient:
|
||||
"""HTTP client for mobile.de search/detail pages."""
|
||||
|
||||
def __init__(self, session: requests.Session | None = None, *, delay_seconds: float = 0.7) -> None:
|
||||
self.session = session or requests.Session()
|
||||
self.session.headers.update(DEFAULT_HEADERS)
|
||||
self.delay_seconds = max(0.0, delay_seconds)
|
||||
|
||||
@classmethod
|
||||
def for_worker(cls, *, delay_seconds: float = 0.0) -> "MobileDeClient":
|
||||
session = requests.Session()
|
||||
adapter = requests.adapters.HTTPAdapter(pool_connections=100, pool_maxsize=100, max_retries=0)
|
||||
session.mount("https://", adapter)
|
||||
session.mount("http://", adapter)
|
||||
return cls(session=session, delay_seconds=delay_seconds)
|
||||
|
||||
@staticmethod
|
||||
def build_make_model_param(make_id: str | int, model_id: str | int | None = None) -> str:
|
||||
make = str(make_id).strip()
|
||||
model = str(model_id).strip() if model_id is not None else ""
|
||||
return f"{make};{model};;"
|
||||
|
||||
@staticmethod
|
||||
def build_search_url(page_number: int = 1, **params: str | int | None) -> str:
|
||||
query = {
|
||||
"sb": "rel",
|
||||
"od": "up",
|
||||
"vc": "Car",
|
||||
"s": "Car",
|
||||
"pageNumber": page_number,
|
||||
}
|
||||
query.update({key: value for key, value in params.items() if value is not None})
|
||||
return f"{BASE_URL}{SEARCH_PATH}?{urlencode(query)}"
|
||||
|
||||
@staticmethod
|
||||
def build_search_url_from_existing(
|
||||
search_url: str,
|
||||
*,
|
||||
page_number: int | None = None,
|
||||
**params: str | int | None,
|
||||
) -> str:
|
||||
parts = urlsplit(search_url)
|
||||
query_items = [
|
||||
(key, value)
|
||||
for key, value in parse_qsl(parts.query, keep_blank_values=True)
|
||||
if key != "pageNumber" and key not in params
|
||||
]
|
||||
if page_number is not None:
|
||||
query_items.append(("pageNumber", str(page_number)))
|
||||
query_items.extend((key, str(value)) for key, value in params.items() if value is not None)
|
||||
scheme = parts.scheme or "https"
|
||||
netloc = parts.netloc or urlsplit(BASE_URL).netloc
|
||||
path = parts.path or SEARCH_PATH
|
||||
return urlunsplit((scheme, netloc, path, urlencode(query_items), ""))
|
||||
|
||||
@staticmethod
|
||||
def build_detail_url(listing_id: str | int) -> str:
|
||||
query = urlencode({"id": listing_id, "vc": "Car", "s": "Car"})
|
||||
return f"{BASE_URL}{DETAIL_PATH}?{query}"
|
||||
|
||||
def fetch_html(self, url: str, *, timeout: int = 30) -> str:
|
||||
response = self.session.get(url, timeout=timeout)
|
||||
response.raise_for_status()
|
||||
return response.text
|
||||
|
||||
def fetch_search_page(
|
||||
self,
|
||||
page_number: int = 1,
|
||||
*,
|
||||
search_url: str | None = None,
|
||||
**params: str | int | None,
|
||||
) -> MobileDeSearchPage:
|
||||
url = (
|
||||
self.build_search_url_from_existing(search_url, page_number=page_number, **params)
|
||||
if search_url
|
||||
else self.build_search_url(page_number=page_number, **params)
|
||||
)
|
||||
html = self.fetch_html(url)
|
||||
raw = extract_search_results(html)
|
||||
listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
|
||||
return MobileDeSearchPage(
|
||||
url=url,
|
||||
page_number=page_number,
|
||||
total_results=raw.get("numResultsTotal"),
|
||||
listings=listings,
|
||||
raw_search_results=raw,
|
||||
)
|
||||
|
||||
def iter_search_pages(
|
||||
self,
|
||||
*,
|
||||
start_page: int = 1,
|
||||
max_pages: int | None = None,
|
||||
search_url: str | None = None,
|
||||
stop_after_empty: bool = True,
|
||||
progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
|
||||
**params: str | int | None,
|
||||
) -> Iterable[MobileDeSearchPage]:
|
||||
page_number = start_page
|
||||
pages_seen = 0
|
||||
logger.debug(
|
||||
"mobile.de search window started: start_page=%s max_pages=%s params=%s",
|
||||
start_page,
|
||||
max_pages,
|
||||
{key: value for key, value in params.items() if value is not None},
|
||||
)
|
||||
while max_pages is None or pages_seen < max_pages:
|
||||
logger.debug("mobile.de fetching search page=%s", page_number)
|
||||
page = self.fetch_search_page(page_number=page_number, search_url=search_url, **params)
|
||||
page_meta = {
|
||||
"page_number": page.page_number,
|
||||
"pages_seen": pages_seen + 1,
|
||||
"max_pages": max_pages,
|
||||
"listing_count": len(page.listings),
|
||||
"total_results": page.total_results,
|
||||
}
|
||||
logger.debug(
|
||||
"mobile.de fetched search page=%s listings=%s total_results=%s",
|
||||
page.page_number,
|
||||
len(page.listings),
|
||||
page.total_results,
|
||||
)
|
||||
if progress_callback is not None:
|
||||
progress_callback(page, page_meta)
|
||||
if stop_after_empty and not page.listings:
|
||||
logger.debug("mobile.de stopping search window on empty page=%s", page.page_number)
|
||||
break
|
||||
yield page
|
||||
pages_seen += 1
|
||||
page_number += 1
|
||||
if self.delay_seconds:
|
||||
time.sleep(self.delay_seconds)
|
||||
logger.debug(
|
||||
"mobile.de search window finished: pages_seen=%s next_page=%s",
|
||||
pages_seen,
|
||||
page_number,
|
||||
)
|
||||
|
||||
def fetch_search_pages_concurrent(
|
||||
self,
|
||||
*,
|
||||
start_page: int = 1,
|
||||
max_pages: int = 1,
|
||||
workers: int = 8,
|
||||
search_url: str | None = None,
|
||||
stop_after_empty: bool = True,
|
||||
progress_callback: Callable[[MobileDeSearchPage, dict[str, int | None]], None] | None = None,
|
||||
**params: str | int | None,
|
||||
) -> list[MobileDeSearchPage]:
|
||||
max_pages = max(1, int(max_pages))
|
||||
workers = max(1, min(int(workers), max_pages))
|
||||
page_numbers = list(range(max(1, int(start_page)), max(1, int(start_page)) + max_pages))
|
||||
pages_by_number: dict[int, MobileDeSearchPage] = {}
|
||||
thread_local = threading.local()
|
||||
|
||||
def _fetch_page(page_number: int) -> MobileDeSearchPage:
|
||||
client = getattr(thread_local, "client", None)
|
||||
if client is None:
|
||||
client = MobileDeClient.for_worker(delay_seconds=0)
|
||||
thread_local.client = client
|
||||
return client.fetch_search_page(page_number=page_number, search_url=search_url, **params)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=workers) as executor:
|
||||
futures = {
|
||||
executor.submit(_fetch_page, page_number): page_number
|
||||
for page_number in page_numbers
|
||||
}
|
||||
for future in as_completed(futures):
|
||||
page_number = futures[future]
|
||||
page = future.result()
|
||||
pages_by_number[page_number] = page
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
page,
|
||||
{
|
||||
"page_number": page.page_number,
|
||||
"pages_seen": len(pages_by_number),
|
||||
"max_pages": max_pages,
|
||||
"listing_count": len(page.listings),
|
||||
"total_results": page.total_results,
|
||||
},
|
||||
)
|
||||
ordered_pages = [pages_by_number[page_number] for page_number in page_numbers if page_number in pages_by_number]
|
||||
if stop_after_empty:
|
||||
non_empty_pages: list[MobileDeSearchPage] = []
|
||||
for page in ordered_pages:
|
||||
if not page.listings:
|
||||
break
|
||||
non_empty_pages.append(page)
|
||||
return non_empty_pages
|
||||
return ordered_pages
|
||||
|
||||
def fetch_detail(self, listing_id: str | int) -> dict:
|
||||
html = self.fetch_html(self.build_detail_url(listing_id))
|
||||
return extract_detail_listing(html)
|
||||
|
||||
@staticmethod
|
||||
def _map_listing(item: dict) -> MobileDeListing:
|
||||
listing_id = str(item.get("id") or item.get("adId") or "")
|
||||
attr = item.get("attr") if isinstance(item.get("attr"), dict) else {}
|
||||
contact = item.get("contact") if isinstance(item.get("contact"), dict) else {}
|
||||
location = ", ".join(
|
||||
part for part in [attr.get("z"), attr.get("loc")] if isinstance(part, str) and part
|
||||
) or None
|
||||
return MobileDeListing(
|
||||
id=listing_id,
|
||||
url=MobileDeClient.build_detail_url(listing_id) if listing_id else "",
|
||||
title=item.get("shortTitle"),
|
||||
subtitle=item.get("subTitle"),
|
||||
price=item.get("p"),
|
||||
seller_name=contact.get("name"),
|
||||
seller_type=contact.get("type") or item.get("st"),
|
||||
location=location,
|
||||
first_registration=attr.get("fr"),
|
||||
mileage=attr.get("ml"),
|
||||
power=attr.get("pw"),
|
||||
fuel=attr.get("ft"),
|
||||
transmission=attr.get("tr"),
|
||||
raw=item,
|
||||
)
|
||||
79
iaai_scraper/mobile_de/flight.py
Normal file
79
iaai_scraper/mobile_de/flight.py
Normal file
@@ -0,0 +1,79 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
NEXT_FLIGHT_RE = re.compile(r"self\.__next_f\.push\(\[1,\"(.*?)\"\]\)", re.DOTALL)
|
||||
|
||||
|
||||
def extract_next_flight_strings(html: str) -> list[str]:
|
||||
"""Extract decoded Next.js Flight chunks from mobile.de HTML."""
|
||||
chunks: list[str] = []
|
||||
for match in NEXT_FLIGHT_RE.finditer(html):
|
||||
raw = match.group(1)
|
||||
try:
|
||||
chunks.append(json.loads(f'"{raw}"'))
|
||||
except json.JSONDecodeError:
|
||||
# Fallback keeps parser useful if one chunk has non-standard escaping.
|
||||
chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore"))
|
||||
return chunks
|
||||
|
||||
|
||||
def extract_json_object_after(text: str, marker: str) -> dict[str, Any] | None:
|
||||
"""Return JSON object that starts immediately after a marker in a decoded Flight chunk."""
|
||||
marker_index = text.find(marker)
|
||||
if marker_index < 0:
|
||||
return None
|
||||
start = text.find("{", marker_index + len(marker))
|
||||
if start < 0:
|
||||
return None
|
||||
|
||||
depth = 0
|
||||
in_string = False
|
||||
escaped = False
|
||||
for index in range(start, len(text)):
|
||||
char = text[index]
|
||||
if in_string:
|
||||
if escaped:
|
||||
escaped = False
|
||||
elif char == "\\":
|
||||
escaped = True
|
||||
elif char == '"':
|
||||
in_string = False
|
||||
continue
|
||||
if char == '"':
|
||||
in_string = True
|
||||
elif char == "{":
|
||||
depth += 1
|
||||
elif char == "}":
|
||||
depth -= 1
|
||||
if depth == 0:
|
||||
candidate = text[start : index + 1]
|
||||
try:
|
||||
return json.loads(candidate)
|
||||
except json.JSONDecodeError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def extract_search_results(html: str) -> dict[str, Any]:
|
||||
"""Extract searchResults from mobile.de SRP HTML."""
|
||||
for chunk in extract_next_flight_strings(html):
|
||||
if '"eventScope":"page-srp"' not in chunk or '"searchResults"' not in chunk:
|
||||
continue
|
||||
results = extract_json_object_after(chunk, '"searchResults":')
|
||||
if isinstance(results, dict):
|
||||
return results
|
||||
return {}
|
||||
|
||||
|
||||
def extract_detail_listing(html: str) -> dict[str, Any]:
|
||||
"""Extract listing object from mobile.de VIP/detail HTML."""
|
||||
for chunk in extract_next_flight_strings(html):
|
||||
if '"eventScope":"page-vip"' not in chunk or '"listing"' not in chunk:
|
||||
continue
|
||||
listing = extract_json_object_after(chunk, '"listing":')
|
||||
if isinstance(listing, dict):
|
||||
return listing
|
||||
return {}
|
||||
220
iaai_scraper/mobile_de/mapper.py
Normal file
220
iaai_scraper/mobile_de/mapper.py
Normal file
@@ -0,0 +1,220 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import re
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
from ..storage.schemas import CarRecord, ImageRecord
|
||||
from .client import MobileDeClient
|
||||
from .models import MobileDeListing
|
||||
|
||||
_BODY_MAP = {
|
||||
"cabrio": "OPEN",
|
||||
"кабриолет": "OPEN",
|
||||
"limousine": "SEDAN",
|
||||
"седан": "SEDAN",
|
||||
"suv": "SUV",
|
||||
"внедорожник": "SUV",
|
||||
"kombi": "STATION_WAGON",
|
||||
"универсал": "STATION_WAGON",
|
||||
"van": "MINIVAN",
|
||||
"фургон": "MINIVAN",
|
||||
"coupe": "COUPE",
|
||||
"купе": "COUPE",
|
||||
"kleinwagen": "HATCHBACK",
|
||||
"маленький": "HATCHBACK",
|
||||
}
|
||||
|
||||
_GEARBOX_MAP = {
|
||||
"автомат": "AT",
|
||||
"automatic": "AT",
|
||||
"механ": "MT",
|
||||
"manual": "MT",
|
||||
"cvt": "CVT",
|
||||
}
|
||||
|
||||
_COLOR_MAP = {
|
||||
"schwarz": "black",
|
||||
"черный": "black",
|
||||
"weiß": "white",
|
||||
"weiss": "white",
|
||||
"белый": "white",
|
||||
"серый": "gray",
|
||||
"grau": "gray",
|
||||
"silber": "silver",
|
||||
"сереб": "silver",
|
||||
"rot": "red",
|
||||
"красный": "red",
|
||||
"blau": "blue",
|
||||
"синий": "blue",
|
||||
"grün": "green",
|
||||
"gruen": "green",
|
||||
"зеленый": "green",
|
||||
}
|
||||
|
||||
|
||||
class MobileDeMapper:
|
||||
"""Map mobile.de search/detail payloads into the existing CarRecord schema."""
|
||||
|
||||
def listing_to_car_record(self, listing: MobileDeListing) -> CarRecord:
|
||||
raw = listing.raw or {}
|
||||
attr = raw.get("attr") if isinstance(raw.get("attr"), dict) else {}
|
||||
make = raw.get("make") if isinstance(raw.get("make"), dict) else {}
|
||||
model_payload = raw.get("model") if isinstance(raw.get("model"), dict) else {}
|
||||
|
||||
brand = self._text(make.get("localized") or self._brand_from_title(listing.title) or listing.title or "UNKNOWN")
|
||||
model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
|
||||
origin_id = self.origin_id(str(listing.id))
|
||||
title = " ".join(part for part in [listing.title, listing.subtitle] if part)
|
||||
|
||||
return CarRecord(
|
||||
parser_id=self._parser_id(origin_id),
|
||||
brand=brand[:50] or "UNKNOWN",
|
||||
model=model[:50] or "UNKNOWN",
|
||||
year=self._year_from_first_registration(listing.first_registration or attr.get("fr")),
|
||||
price=self._money_to_int(listing.price or raw.get("p")),
|
||||
currency="EUR",
|
||||
mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
|
||||
country="NA",
|
||||
is_sold=False,
|
||||
color=self._normalize_color(attr.get("ecol")),
|
||||
drive=None,
|
||||
gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")),
|
||||
steering_wheel="LEFT",
|
||||
body_type=self._normalize_body(attr.get("c")),
|
||||
engine_volume=self._int_from_text(attr.get("cc")),
|
||||
selling_type="CLASSIFIED",
|
||||
one_owner=(str(attr.get("pvo") or "").strip() == "1"),
|
||||
new_car=False,
|
||||
is_hidden=False,
|
||||
origin="MOBILE_DE",
|
||||
origin_url=listing.url,
|
||||
origin_id=origin_id,
|
||||
is_damaged=bool(raw.get("hasDamage")),
|
||||
evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None,
|
||||
non_smoking=True,
|
||||
rental=False,
|
||||
repair_history=bool(raw.get("hasDamage")),
|
||||
slug=self._slugify(title or f"{brand} {model}"),
|
||||
last_seen_at=datetime.now(timezone.utc),
|
||||
images=self._images_from_listing(raw),
|
||||
)
|
||||
|
||||
def detail_to_car_record(self, listing_id: str, detail: dict[str, Any]) -> CarRecord:
|
||||
title = self._text(detail.get("shortTitle") or detail.get("make") or "UNKNOWN")
|
||||
subtitle = self._text(detail.get("subTitle"))
|
||||
fake_listing = MobileDeListing(
|
||||
id=str(listing_id),
|
||||
url=MobileDeClient.build_detail_url(listing_id),
|
||||
title=title,
|
||||
subtitle=subtitle,
|
||||
price=self._text(detail.get("price") or detail.get("p")),
|
||||
raw=detail,
|
||||
)
|
||||
return self.listing_to_car_record(fake_listing)
|
||||
|
||||
@staticmethod
|
||||
def origin_id(listing_id: str) -> str:
|
||||
return f"mobile.de:{listing_id}"
|
||||
|
||||
@staticmethod
|
||||
def _parser_id(origin_id: str) -> str:
|
||||
digest = hashlib.sha1(origin_id.encode("utf-8")).hexdigest()[:16]
|
||||
return f"mobilede-{digest}"
|
||||
|
||||
@staticmethod
|
||||
def _text(value: Any) -> str:
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
@classmethod
|
||||
def _money_to_int(cls, value: Any) -> int | None:
|
||||
return cls._int_from_text(value)
|
||||
|
||||
@staticmethod
|
||||
def _int_from_text(value: Any) -> int | None:
|
||||
if value is None:
|
||||
return None
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
return int(value)
|
||||
digits = re.sub(r"[^0-9]", "", str(value))
|
||||
return int(digits) if digits else None
|
||||
|
||||
@staticmethod
|
||||
def _year_from_first_registration(value: Any) -> int | None:
|
||||
text = "" if value is None else str(value)
|
||||
match = re.search(r"(19|20)\d{2}", text)
|
||||
return int(match.group(0)) if match else None
|
||||
|
||||
@staticmethod
|
||||
def _brand_from_title(title: str | None) -> str | None:
|
||||
if not title:
|
||||
return None
|
||||
return title.split()[0]
|
||||
|
||||
@staticmethod
|
||||
def _model_from_title(title: str | None, brand: str) -> str | None:
|
||||
if not title:
|
||||
return None
|
||||
rest = title.replace(brand, "", 1).strip()
|
||||
return rest or None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_gearbox(value: Any) -> str | None:
|
||||
text = "" if value is None else str(value).lower()
|
||||
for marker, mapped in _GEARBOX_MAP.items():
|
||||
if marker in text:
|
||||
return mapped
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def _normalize_body(value: Any) -> str:
|
||||
text = "" if value is None else str(value).lower()
|
||||
for marker, mapped in _BODY_MAP.items():
|
||||
if marker in text:
|
||||
return mapped
|
||||
return "OTHER"
|
||||
|
||||
@staticmethod
|
||||
def _normalize_color(value: Any) -> str:
|
||||
text = "" if value is None else str(value).lower().strip()
|
||||
for marker, mapped in _COLOR_MAP.items():
|
||||
if marker in text:
|
||||
return mapped
|
||||
return text[:50] if text else "other"
|
||||
|
||||
@staticmethod
|
||||
def _slugify(value: str) -> str:
|
||||
slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
|
||||
return slug[:180] or "mobilede-car"
|
||||
|
||||
@staticmethod
|
||||
def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
|
||||
urls: list[str] = []
|
||||
image = raw.get("image")
|
||||
if isinstance(image, str):
|
||||
urls.append(MobileDeMapper._normalize_image_url(image))
|
||||
images = raw.get("images")
|
||||
if isinstance(images, list):
|
||||
for item in images:
|
||||
if isinstance(item, str):
|
||||
urls.append(MobileDeMapper._normalize_image_url(item))
|
||||
elif isinstance(item, dict):
|
||||
src = item.get("src") or item.get("url") or item.get("uri")
|
||||
if src:
|
||||
urls.append(MobileDeMapper._normalize_image_url(str(src)))
|
||||
return [
|
||||
ImageRecord(fullres_image=url, preview_image=url, order_index=index)
|
||||
for index, url in enumerate(dict.fromkeys(url for url in urls if url))
|
||||
]
|
||||
|
||||
@staticmethod
|
||||
def _normalize_image_url(value: str) -> str:
|
||||
url = str(value).strip()
|
||||
if not url:
|
||||
return ""
|
||||
if url.startswith("//"):
|
||||
return f"https:{url}"
|
||||
if url.startswith("http://") or url.startswith("https://"):
|
||||
return url
|
||||
return f"https://{url.lstrip('/')}"
|
||||
35
iaai_scraper/mobile_de/models.py
Normal file
35
iaai_scraper/mobile_de/models.py
Normal file
@@ -0,0 +1,35 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MobileDeListing:
|
||||
"""One listing extracted from mobile.de search results."""
|
||||
|
||||
id: str
|
||||
url: str
|
||||
title: str | None = None
|
||||
subtitle: str | None = None
|
||||
price: str | None = None
|
||||
seller_name: str | None = None
|
||||
seller_type: str | None = None
|
||||
location: str | None = None
|
||||
first_registration: str | None = None
|
||||
mileage: str | None = None
|
||||
power: str | None = None
|
||||
fuel: str | None = None
|
||||
transmission: str | None = None
|
||||
raw: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MobileDeSearchPage:
|
||||
"""Parsed mobile.de search page."""
|
||||
|
||||
url: str
|
||||
page_number: int
|
||||
total_results: int | None
|
||||
listings: list[MobileDeListing]
|
||||
raw_search_results: dict[str, Any] = field(default_factory=dict)
|
||||
325
iaai_scraper/mobile_de/scraper.py
Normal file
325
iaai_scraper/mobile_de/scraper.py
Normal file
@@ -0,0 +1,325 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
from ..core.config import Settings, settings
|
||||
from ..storage.db import PersistenceService
|
||||
from .client import MobileDeClient
|
||||
from .mapper import MobileDeMapper
|
||||
from .models import MobileDeListing
|
||||
|
||||
logger = logging.getLogger("mobile_de.scraper")
|
||||
|
||||
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
|
||||
MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
|
||||
|
||||
|
||||
class MobileDeScraper:
|
||||
"""High-level mobile.de scraper facade."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
client: MobileDeClient | None = None,
|
||||
persistence: PersistenceService | None = None,
|
||||
mapper: MobileDeMapper | None = None,
|
||||
runtime_settings: Settings | None = None,
|
||||
) -> None:
|
||||
self.settings = runtime_settings or settings
|
||||
self.client = client or MobileDeClient()
|
||||
self.persistence = persistence or PersistenceService(self.settings)
|
||||
self.mapper = mapper or MobileDeMapper()
|
||||
|
||||
def collect_search(
|
||||
self,
|
||||
*,
|
||||
start_page: int = 1,
|
||||
max_pages: int = 1,
|
||||
search_url: str | None = None,
|
||||
make_id: str | None = None,
|
||||
model_id: str | None = None,
|
||||
price_min: str | None = None,
|
||||
price_max: str | None = None,
|
||||
year_min: str | None = None,
|
||||
year_max: str | None = None,
|
||||
mileage_min: str | None = None,
|
||||
mileage_max: str | None = None,
|
||||
sort_by: str | None = None,
|
||||
sort_order: str | None = None,
|
||||
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
params: dict[str, str | None] = {}
|
||||
if not search_url:
|
||||
params = {
|
||||
"p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
|
||||
"fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
|
||||
"ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
|
||||
}
|
||||
if make_id:
|
||||
params["ms"] = self.client.build_make_model_param(make_id, model_id)
|
||||
if sort_by:
|
||||
params["sb"] = sort_by
|
||||
if sort_order:
|
||||
params["od"] = sort_order
|
||||
|
||||
logger.debug(
|
||||
"mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
|
||||
start_page,
|
||||
max_pages,
|
||||
bool(search_url),
|
||||
make_id,
|
||||
model_id,
|
||||
year_min,
|
||||
year_max,
|
||||
price_min,
|
||||
price_max,
|
||||
mileage_min,
|
||||
mileage_max,
|
||||
)
|
||||
pages = []
|
||||
|
||||
def _on_page(page, meta: dict[str, int | None]) -> None:
|
||||
payload = {
|
||||
**meta,
|
||||
"page_url": page.url,
|
||||
"unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
|
||||
+ len({listing.id for listing in page.listings if listing.id}),
|
||||
}
|
||||
if progress_callback is not None:
|
||||
progress_callback("page_collected", payload)
|
||||
|
||||
concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
|
||||
if concurrent_pages > 1 and max_pages and max_pages > 1:
|
||||
pages.extend(self.client.fetch_search_pages_concurrent(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
workers=concurrent_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
))
|
||||
else:
|
||||
for page in self.client.iter_search_pages(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
):
|
||||
pages.append(page)
|
||||
|
||||
unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
|
||||
result = {
|
||||
"source": "mobile.de",
|
||||
"strategy_note": (
|
||||
"mobile.de UI shows 50 pages, but pageNumber works beyond 50; "
|
||||
"for full coverage split by make/model/year/price and deduplicate by id."
|
||||
),
|
||||
"search_url": search_url,
|
||||
"pages": [asdict(page) for page in pages],
|
||||
"listing_count": sum(len(page.listings) for page in pages),
|
||||
"unique_listing_count": len(unique_ids),
|
||||
"unique_listing_ids": unique_ids,
|
||||
}
|
||||
logger.debug(
|
||||
"mobile.de collect_search finished: pages=%s listings=%s unique=%s",
|
||||
len(pages),
|
||||
result["listing_count"],
|
||||
result["unique_listing_count"],
|
||||
)
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"search_collection_done",
|
||||
{
|
||||
"pages_collected": len(pages),
|
||||
"listing_count": result["listing_count"],
|
||||
"unique_listing_count": result["unique_listing_count"],
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
def collect_detail(self, listing_id: str) -> dict[str, Any]:
|
||||
return {
|
||||
"source": "mobile.de",
|
||||
"listing_id": str(listing_id),
|
||||
"url": self.client.build_detail_url(listing_id),
|
||||
"listing": self.client.fetch_detail(listing_id),
|
||||
}
|
||||
|
||||
def init_db(self) -> dict[str, Any]:
|
||||
self.persistence.create_tables()
|
||||
return {"status": "ok", "source": "mobile.de"}
|
||||
|
||||
def sync_search(
|
||||
self,
|
||||
*,
|
||||
start_page: int = 1,
|
||||
max_pages: int = 1,
|
||||
lane: str = "mobile_de_cars",
|
||||
only_new: bool | None = None,
|
||||
search_url: str | None = None,
|
||||
make_id: str | None = None,
|
||||
model_id: str | None = None,
|
||||
price_min: str | None = None,
|
||||
price_max: str | None = None,
|
||||
year_min: str | None = None,
|
||||
year_max: str | None = None,
|
||||
mileage_min: str | None = None,
|
||||
mileage_max: str | None = None,
|
||||
sort_by: str | None = None,
|
||||
sort_order: str | None = None,
|
||||
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
self.persistence.create_tables()
|
||||
run_id = self.persistence.start_sync_run(lane)
|
||||
failed = 0
|
||||
logger.debug(
|
||||
"mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
|
||||
run_id,
|
||||
lane,
|
||||
start_page,
|
||||
max_pages,
|
||||
)
|
||||
try:
|
||||
data = self.collect_search(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
search_url=search_url,
|
||||
make_id=make_id,
|
||||
model_id=model_id,
|
||||
price_min=price_min,
|
||||
price_max=price_max,
|
||||
year_min=year_min,
|
||||
year_max=year_max,
|
||||
mileage_min=mileage_min,
|
||||
mileage_max=mileage_max,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
# Map serialized listings back to records.
|
||||
records = []
|
||||
for page in data["pages"]:
|
||||
for item in page["listings"]:
|
||||
records.append(self.mapper.listing_to_car_record(MobileDeListing(**item)))
|
||||
|
||||
skipped_existing = 0
|
||||
if only_new and records:
|
||||
existing_origin_ids = self.persistence.get_existing_origin_ids(
|
||||
[record.origin_id for record in records if record.origin_id]
|
||||
)
|
||||
before_filter_count = len(records)
|
||||
|
||||
# For newest-first, cut tail after existing streak.
|
||||
should_cut_tail = (
|
||||
str(sort_by or "").lower() == "doc"
|
||||
and str(sort_order or "").lower() == "down"
|
||||
and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
|
||||
)
|
||||
if should_cut_tail:
|
||||
filtered_records = []
|
||||
existing_streak = 0
|
||||
considered = 0
|
||||
for record in records:
|
||||
considered += 1
|
||||
is_existing = record.origin_id in existing_origin_ids
|
||||
if is_existing:
|
||||
skipped_existing += 1
|
||||
existing_streak += 1
|
||||
if (
|
||||
existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
|
||||
and len(filtered_records) >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
|
||||
):
|
||||
break
|
||||
continue
|
||||
existing_streak = 0
|
||||
filtered_records.append(record)
|
||||
records = filtered_records
|
||||
data["listing_count"] = considered
|
||||
data["unique_listing_count"] = min(int(data.get("unique_listing_count", considered)), considered)
|
||||
logger.info(
|
||||
"mobile.de only_new head-cut applied: considered=%s kept=%s skipped_existing=%s streak=%s",
|
||||
considered,
|
||||
len(records),
|
||||
skipped_existing,
|
||||
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
|
||||
)
|
||||
else:
|
||||
records = [record for record in records if record.origin_id not in existing_origin_ids]
|
||||
skipped_existing = before_filter_count - len(records)
|
||||
logger.info(
|
||||
"mobile.de only_new filter applied: kept=%s skipped_existing=%s",
|
||||
len(records),
|
||||
skipped_existing,
|
||||
)
|
||||
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"records_mapped",
|
||||
{
|
||||
"record_count": len(records),
|
||||
"skipped_existing": skipped_existing,
|
||||
"only_new": bool(only_new),
|
||||
"run_id": run_id,
|
||||
},
|
||||
)
|
||||
|
||||
upsert = self.persistence.upsert_cars_batch(records) if records else {
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"images_upserted": 0,
|
||||
}
|
||||
logger.debug(
|
||||
"mobile.de sync_search upsert finished: run_id=%s inserted=%s updated=%s images=%s",
|
||||
run_id,
|
||||
upsert.get("inserted", 0),
|
||||
upsert.get("updated", 0),
|
||||
upsert.get("images_upserted", 0),
|
||||
)
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"db_upsert_done",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"inserted": int(upsert.get("inserted", 0)),
|
||||
"updated": int(upsert.get("updated", 0)),
|
||||
"images_upserted": int(upsert.get("images_upserted", 0)),
|
||||
},
|
||||
)
|
||||
self.persistence.finish_sync_run(
|
||||
run_id,
|
||||
status="success",
|
||||
ids_fetched=len(records),
|
||||
cars_upserted=int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0)),
|
||||
cars_failed=failed,
|
||||
images_upserted=int(upsert.get("images_upserted", 0)),
|
||||
)
|
||||
logger.debug(
|
||||
"mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
|
||||
run_id,
|
||||
data.get("listing_count", 0),
|
||||
data.get("unique_listing_count", 0),
|
||||
)
|
||||
return {"run_id": run_id, "source": "mobile.de", "upsert": upsert, "skipped_existing": skipped_existing, **data}
|
||||
except Exception as exc:
|
||||
self.persistence.finish_sync_run(
|
||||
run_id,
|
||||
status="failed",
|
||||
ids_fetched=0,
|
||||
cars_upserted=0,
|
||||
cars_failed=failed,
|
||||
images_upserted=0,
|
||||
error_summary=str(exc),
|
||||
)
|
||||
logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
|
||||
raise
|
||||
|
||||
def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
|
||||
self.persistence.create_tables()
|
||||
detail = self.client.fetch_detail(listing_id)
|
||||
record = self.mapper.detail_to_car_record(str(listing_id), detail)
|
||||
result = self.persistence.upsert_car(record)
|
||||
return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}
|
||||
Reference in New Issue
Block a user