add mobilede scraper
This commit is contained in:
qananasikq
454
mobilede_scraper/mobile_de/scraper.py
Normal file
454
mobilede_scraper/mobile_de/scraper.py
Normal file
@@ -0,0 +1,454 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import os
|
||||
from collections.abc import Callable
|
||||
from dataclasses import asdict
|
||||
from typing import Any
|
||||
|
||||
import requests
|
||||
|
||||
from ..core.config import Settings, settings
|
||||
from ..storage.db import PersistenceService
|
||||
from ..storage.schemas import CarRecord
|
||||
from .client import MobileDeClient
|
||||
from .mapper import MobileDeMapper
|
||||
from .models import MobileDeListing
|
||||
|
||||
logger = logging.getLogger("mobile_de.scraper")
|
||||
|
||||
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
|
||||
MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
|
||||
MOBILEDE_SEARCH_STRATEGY_NOTE = (
|
||||
"mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
|
||||
"for full coverage split into narrower segments and deduplicate by id."
|
||||
)
|
||||
|
||||
|
||||
class MobileDeScraper:
|
||||
"""High-level mobile.de scraper facade."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
client: MobileDeClient | None = None,
|
||||
persistence: PersistenceService | None = None,
|
||||
mapper: MobileDeMapper | None = None,
|
||||
runtime_settings: Settings | None = None,
|
||||
) -> None:
|
||||
self.settings = runtime_settings or settings
|
||||
self.client = client or MobileDeClient()
|
||||
self.persistence = persistence or PersistenceService(self.settings)
|
||||
self.mapper = mapper or MobileDeMapper()
|
||||
|
||||
@staticmethod
|
||||
def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool:
|
||||
return (
|
||||
str(sort_by or "").lower() == "doc"
|
||||
and str(sort_order or "").lower() == "down"
|
||||
and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
|
||||
)
|
||||
|
||||
def _build_search_params(
|
||||
self,
|
||||
*,
|
||||
search_url: str | None,
|
||||
make_id: str | None,
|
||||
model_id: str | None,
|
||||
price_min: str | None,
|
||||
price_max: str | None,
|
||||
year_min: str | None,
|
||||
year_max: str | None,
|
||||
mileage_min: str | None,
|
||||
mileage_max: str | None,
|
||||
sort_by: str | None,
|
||||
sort_order: str | None,
|
||||
) -> dict[str, str | None]:
|
||||
params: dict[str, str | None] = {}
|
||||
if not search_url:
|
||||
params = {
|
||||
"p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
|
||||
"fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
|
||||
"ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
|
||||
}
|
||||
if make_id:
|
||||
params["ms"] = self.client.build_make_model_param(make_id, model_id)
|
||||
if sort_by:
|
||||
params["sb"] = sort_by
|
||||
if sort_order:
|
||||
params["od"] = sort_order
|
||||
return params
|
||||
|
||||
@staticmethod
|
||||
def _dedupe_page_records(
|
||||
page_records: list[CarRecord],
|
||||
seen_record_keys: set[str],
|
||||
) -> list[CarRecord]:
|
||||
deduped_page_records: list[CarRecord] = []
|
||||
for record in page_records:
|
||||
record_key = record.origin_id or record.origin_url
|
||||
if not record_key or record_key in seen_record_keys:
|
||||
continue
|
||||
seen_record_keys.add(record_key)
|
||||
deduped_page_records.append(record)
|
||||
return deduped_page_records
|
||||
|
||||
def _apply_only_new_page_policy(
|
||||
self,
|
||||
*,
|
||||
page_records: list[CarRecord],
|
||||
only_new: bool | None,
|
||||
sort_by: str | None,
|
||||
sort_order: str | None,
|
||||
skipped_existing: int,
|
||||
existing_streak: int,
|
||||
new_records_kept: int,
|
||||
) -> tuple[list[CarRecord], int, int, int, bool]:
|
||||
if not only_new or not page_records:
|
||||
return page_records, skipped_existing, existing_streak, new_records_kept, False
|
||||
|
||||
existing_origin_ids = self.persistence.get_existing_origin_ids(
|
||||
[record.origin_id for record in page_records if record.origin_id]
|
||||
)
|
||||
head_cut_triggered = False
|
||||
should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
|
||||
|
||||
if should_cut_tail:
|
||||
filtered_records: list[CarRecord] = []
|
||||
for record_index, record in enumerate(page_records):
|
||||
is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
|
||||
if is_existing:
|
||||
existing_streak += 1
|
||||
if (
|
||||
existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
|
||||
and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
|
||||
):
|
||||
head_cut_triggered = True
|
||||
skipped_existing += max(0, len(page_records) - record_index - 1)
|
||||
break
|
||||
else:
|
||||
existing_streak = 0
|
||||
new_records_kept += 1
|
||||
filtered_records.append(record)
|
||||
return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered
|
||||
|
||||
for record in page_records:
|
||||
if not record.origin_id or record.origin_id not in existing_origin_ids:
|
||||
new_records_kept += 1
|
||||
return page_records, skipped_existing, existing_streak, new_records_kept, False
|
||||
|
||||
def collect_search(
|
||||
self,
|
||||
*,
|
||||
start_page: int = 1,
|
||||
max_pages: int = 1,
|
||||
search_url: str | None = None,
|
||||
make_id: str | None = None,
|
||||
model_id: str | None = None,
|
||||
price_min: str | None = None,
|
||||
price_max: str | None = None,
|
||||
year_min: str | None = None,
|
||||
year_max: str | None = None,
|
||||
mileage_min: str | None = None,
|
||||
mileage_max: str | None = None,
|
||||
sort_by: str | None = None,
|
||||
sort_order: str | None = None,
|
||||
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
params = self._build_search_params(
|
||||
search_url=search_url,
|
||||
make_id=make_id,
|
||||
model_id=model_id,
|
||||
price_min=price_min,
|
||||
price_max=price_max,
|
||||
year_min=year_min,
|
||||
year_max=year_max,
|
||||
mileage_min=mileage_min,
|
||||
mileage_max=mileage_max,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
)
|
||||
|
||||
logger.debug(
|
||||
"mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
|
||||
start_page,
|
||||
max_pages,
|
||||
bool(search_url),
|
||||
make_id,
|
||||
model_id,
|
||||
year_min,
|
||||
year_max,
|
||||
price_min,
|
||||
price_max,
|
||||
mileage_min,
|
||||
mileage_max,
|
||||
)
|
||||
pages = []
|
||||
|
||||
def _on_page(page, meta: dict[str, int | None]) -> None:
|
||||
payload = {
|
||||
**meta,
|
||||
"page_url": page.url,
|
||||
"unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
|
||||
+ len({listing.id for listing in page.listings if listing.id}),
|
||||
}
|
||||
if progress_callback is not None:
|
||||
progress_callback("page_collected", payload)
|
||||
|
||||
concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
|
||||
if concurrent_pages > 1 and max_pages and max_pages > 1:
|
||||
try:
|
||||
pages.extend(self.client.fetch_search_pages_concurrent(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
workers=concurrent_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
))
|
||||
except requests.RequestException as exc:
|
||||
logger.warning(
|
||||
"mobile.de concurrent fetch failed, fallback to sequential: workers=%s start_page=%s max_pages=%s error=%s",
|
||||
concurrent_pages,
|
||||
start_page,
|
||||
max_pages,
|
||||
exc,
|
||||
)
|
||||
pages.clear()
|
||||
for page in self.client.iter_search_pages(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
):
|
||||
pages.append(page)
|
||||
else:
|
||||
for page in self.client.iter_search_pages(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
):
|
||||
pages.append(page)
|
||||
|
||||
unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
|
||||
result = {
|
||||
"source": "mobile.de",
|
||||
"strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
|
||||
"search_url": search_url,
|
||||
"pages": [asdict(page) for page in pages],
|
||||
"listing_count": sum(len(page.listings) for page in pages),
|
||||
"unique_listing_count": len(unique_ids),
|
||||
"unique_listing_ids": unique_ids,
|
||||
}
|
||||
logger.debug(
|
||||
"mobile.de collect_search finished: pages=%s listings=%s unique=%s",
|
||||
len(pages),
|
||||
result["listing_count"],
|
||||
result["unique_listing_count"],
|
||||
)
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"search_collection_done",
|
||||
{
|
||||
"pages_collected": len(pages),
|
||||
"listing_count": result["listing_count"],
|
||||
"unique_listing_count": result["unique_listing_count"],
|
||||
},
|
||||
)
|
||||
return result
|
||||
|
||||
def collect_detail(self, listing_id: str) -> dict[str, Any]:
|
||||
return {
|
||||
"source": "mobile.de",
|
||||
"listing_id": str(listing_id),
|
||||
"url": self.client.build_detail_url(listing_id),
|
||||
"listing": self.client.fetch_detail(listing_id),
|
||||
}
|
||||
|
||||
def init_db(self) -> dict[str, Any]:
|
||||
self.persistence.create_tables()
|
||||
return {"status": "ok", "source": "mobile.de"}
|
||||
|
||||
def sync_search(
|
||||
self,
|
||||
*,
|
||||
start_page: int = 1,
|
||||
max_pages: int = 1,
|
||||
lane: str = "mobile_de_cars",
|
||||
only_new: bool | None = None,
|
||||
search_url: str | None = None,
|
||||
make_id: str | None = None,
|
||||
model_id: str | None = None,
|
||||
price_min: str | None = None,
|
||||
price_max: str | None = None,
|
||||
year_min: str | None = None,
|
||||
year_max: str | None = None,
|
||||
mileage_min: str | None = None,
|
||||
mileage_max: str | None = None,
|
||||
sort_by: str | None = None,
|
||||
sort_order: str | None = None,
|
||||
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
self.persistence.create_tables()
|
||||
run_id = self.persistence.start_sync_run(lane)
|
||||
pages_payload: list[dict[str, Any]] = []
|
||||
unique_ids: set[str] = set()
|
||||
listing_count = 0
|
||||
skipped_existing = 0
|
||||
ids_fetched = 0
|
||||
cars_upserted = 0
|
||||
inserted_total = 0
|
||||
updated_total = 0
|
||||
images_upserted = 0
|
||||
existing_streak = 0
|
||||
new_records_kept = 0
|
||||
head_cut_triggered = False
|
||||
seen_record_keys: set[str] = set()
|
||||
logger.debug(
|
||||
"mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
|
||||
run_id,
|
||||
lane,
|
||||
start_page,
|
||||
max_pages,
|
||||
)
|
||||
try:
|
||||
data = self.collect_search(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
search_url=search_url,
|
||||
make_id=make_id,
|
||||
model_id=model_id,
|
||||
price_min=price_min,
|
||||
price_max=price_max,
|
||||
year_min=year_min,
|
||||
year_max=year_max,
|
||||
mileage_min=mileage_min,
|
||||
mileage_max=mileage_max,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
pages_payload = list(data.get("pages", []))
|
||||
listing_count = int(data.get("listing_count", 0) or 0)
|
||||
unique_ids = set(data.get("unique_listing_ids", []))
|
||||
pages_collected = len(pages_payload)
|
||||
|
||||
records: list[CarRecord] = []
|
||||
for page in pages_payload:
|
||||
page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])]
|
||||
records.extend(self._dedupe_page_records(page_records, seen_record_keys))
|
||||
|
||||
if only_new and records:
|
||||
existing_origin_ids = self.persistence.get_existing_origin_ids(
|
||||
[record.origin_id for record in records if record.origin_id]
|
||||
)
|
||||
should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
|
||||
if should_cut_tail:
|
||||
filtered_records: list[CarRecord] = []
|
||||
for record_index, record in enumerate(records):
|
||||
is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
|
||||
filtered_records.append(record)
|
||||
if is_existing:
|
||||
existing_streak += 1
|
||||
if (
|
||||
existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
|
||||
and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
|
||||
):
|
||||
head_cut_triggered = True
|
||||
skipped_existing += max(0, len(records) - record_index - 1)
|
||||
break
|
||||
else:
|
||||
existing_streak = 0
|
||||
new_records_kept += 1
|
||||
records = filtered_records
|
||||
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"records_mapped",
|
||||
{
|
||||
"record_count": len(records),
|
||||
"skipped_existing": skipped_existing,
|
||||
"only_new": bool(only_new),
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
},
|
||||
)
|
||||
|
||||
upsert = self.persistence.upsert_cars_batch(records) if records else {
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"images_upserted": 0,
|
||||
}
|
||||
ids_fetched = len(records)
|
||||
inserted_total = int(upsert.get("inserted", 0))
|
||||
updated_total = int(upsert.get("updated", 0))
|
||||
images_upserted = int(upsert.get("images_upserted", 0))
|
||||
cars_upserted = inserted_total + updated_total
|
||||
|
||||
logger.debug(
|
||||
"mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s",
|
||||
run_id,
|
||||
pages_collected,
|
||||
inserted_total,
|
||||
updated_total,
|
||||
images_upserted,
|
||||
)
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"db_upsert_done",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
"pages_in_batch": pages_collected,
|
||||
"inserted": inserted_total,
|
||||
"updated": updated_total,
|
||||
"images_upserted": images_upserted,
|
||||
},
|
||||
)
|
||||
|
||||
if head_cut_triggered:
|
||||
logger.info(
|
||||
"mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s",
|
||||
len(records),
|
||||
skipped_existing,
|
||||
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
|
||||
)
|
||||
|
||||
data["early_stopped"] = head_cut_triggered
|
||||
self.persistence.finish_sync_run(
|
||||
run_id,
|
||||
status="success",
|
||||
ids_fetched=ids_fetched,
|
||||
cars_upserted=cars_upserted,
|
||||
cars_failed=0,
|
||||
images_upserted=images_upserted,
|
||||
)
|
||||
logger.debug(
|
||||
"mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
|
||||
run_id,
|
||||
data.get("listing_count", 0),
|
||||
data.get("unique_listing_count", 0),
|
||||
)
|
||||
return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data}
|
||||
except Exception as exc:
|
||||
self.persistence.finish_sync_run(
|
||||
run_id,
|
||||
status="failed",
|
||||
ids_fetched=ids_fetched,
|
||||
cars_upserted=cars_upserted,
|
||||
cars_failed=0,
|
||||
images_upserted=images_upserted,
|
||||
error_summary=str(exc),
|
||||
)
|
||||
logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
|
||||
raise
|
||||
|
||||
def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
|
||||
self.persistence.create_tables()
|
||||
detail = self.client.fetch_detail(listing_id)
|
||||
record = self.mapper.detail_to_car_record(str(listing_id), detail)
|
||||
result = self.persistence.upsert_car(record)
|
||||
return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}
|
||||
Reference in New Issue
Block a user