326 lines
13 KiB
Python
326 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from collections.abc import Callable
|
|
from dataclasses import asdict
|
|
from typing import Any
|
|
|
|
from ..core.config import Settings, settings
|
|
from ..storage.db import PersistenceService
|
|
from .client import MobileDeClient
|
|
from .mapper import MobileDeMapper
|
|
from .models import MobileDeListing
|
|
|
|
logger = logging.getLogger("mobile_de.scraper")
|
|
|
|
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
|
|
MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
|
|
|
|
|
|
class MobileDeScraper:
|
|
"""High-level mobile.de scraper facade."""
|
|
|
|
def __init__(
|
|
self,
|
|
client: MobileDeClient | None = None,
|
|
persistence: PersistenceService | None = None,
|
|
mapper: MobileDeMapper | None = None,
|
|
runtime_settings: Settings | None = None,
|
|
) -> None:
|
|
self.settings = runtime_settings or settings
|
|
self.client = client or MobileDeClient()
|
|
self.persistence = persistence or PersistenceService(self.settings)
|
|
self.mapper = mapper or MobileDeMapper()
|
|
|
|
def collect_search(
|
|
self,
|
|
*,
|
|
start_page: int = 1,
|
|
max_pages: int = 1,
|
|
search_url: str | None = None,
|
|
make_id: str | None = None,
|
|
model_id: str | None = None,
|
|
price_min: str | None = None,
|
|
price_max: str | None = None,
|
|
year_min: str | None = None,
|
|
year_max: str | None = None,
|
|
mileage_min: str | None = None,
|
|
mileage_max: str | None = None,
|
|
sort_by: str | None = None,
|
|
sort_order: str | None = None,
|
|
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
|
) -> dict[str, Any]:
|
|
params: dict[str, str | None] = {}
|
|
if not search_url:
|
|
params = {
|
|
"p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
|
|
"fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
|
|
"ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
|
|
}
|
|
if make_id:
|
|
params["ms"] = self.client.build_make_model_param(make_id, model_id)
|
|
if sort_by:
|
|
params["sb"] = sort_by
|
|
if sort_order:
|
|
params["od"] = sort_order
|
|
|
|
logger.debug(
|
|
"mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
|
|
start_page,
|
|
max_pages,
|
|
bool(search_url),
|
|
make_id,
|
|
model_id,
|
|
year_min,
|
|
year_max,
|
|
price_min,
|
|
price_max,
|
|
mileage_min,
|
|
mileage_max,
|
|
)
|
|
pages = []
|
|
|
|
def _on_page(page, meta: dict[str, int | None]) -> None:
|
|
payload = {
|
|
**meta,
|
|
"page_url": page.url,
|
|
"unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
|
|
+ len({listing.id for listing in page.listings if listing.id}),
|
|
}
|
|
if progress_callback is not None:
|
|
progress_callback("page_collected", payload)
|
|
|
|
concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
|
|
if concurrent_pages > 1 and max_pages and max_pages > 1:
|
|
pages.extend(self.client.fetch_search_pages_concurrent(
|
|
start_page=start_page,
|
|
max_pages=max_pages,
|
|
workers=concurrent_pages,
|
|
search_url=search_url,
|
|
progress_callback=_on_page,
|
|
**params,
|
|
))
|
|
else:
|
|
for page in self.client.iter_search_pages(
|
|
start_page=start_page,
|
|
max_pages=max_pages,
|
|
search_url=search_url,
|
|
progress_callback=_on_page,
|
|
**params,
|
|
):
|
|
pages.append(page)
|
|
|
|
unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
|
|
result = {
|
|
"source": "mobile.de",
|
|
"strategy_note": (
|
|
"mobile.de UI shows 50 pages, but pageNumber works beyond 50; "
|
|
"for full coverage split by make/model/year/price and deduplicate by id."
|
|
),
|
|
"search_url": search_url,
|
|
"pages": [asdict(page) for page in pages],
|
|
"listing_count": sum(len(page.listings) for page in pages),
|
|
"unique_listing_count": len(unique_ids),
|
|
"unique_listing_ids": unique_ids,
|
|
}
|
|
logger.debug(
|
|
"mobile.de collect_search finished: pages=%s listings=%s unique=%s",
|
|
len(pages),
|
|
result["listing_count"],
|
|
result["unique_listing_count"],
|
|
)
|
|
if progress_callback is not None:
|
|
progress_callback(
|
|
"search_collection_done",
|
|
{
|
|
"pages_collected": len(pages),
|
|
"listing_count": result["listing_count"],
|
|
"unique_listing_count": result["unique_listing_count"],
|
|
},
|
|
)
|
|
return result
|
|
|
|
def collect_detail(self, listing_id: str) -> dict[str, Any]:
|
|
return {
|
|
"source": "mobile.de",
|
|
"listing_id": str(listing_id),
|
|
"url": self.client.build_detail_url(listing_id),
|
|
"listing": self.client.fetch_detail(listing_id),
|
|
}
|
|
|
|
def init_db(self) -> dict[str, Any]:
|
|
self.persistence.create_tables()
|
|
return {"status": "ok", "source": "mobile.de"}
|
|
|
|
def sync_search(
|
|
self,
|
|
*,
|
|
start_page: int = 1,
|
|
max_pages: int = 1,
|
|
lane: str = "mobile_de_cars",
|
|
only_new: bool | None = None,
|
|
search_url: str | None = None,
|
|
make_id: str | None = None,
|
|
model_id: str | None = None,
|
|
price_min: str | None = None,
|
|
price_max: str | None = None,
|
|
year_min: str | None = None,
|
|
year_max: str | None = None,
|
|
mileage_min: str | None = None,
|
|
mileage_max: str | None = None,
|
|
sort_by: str | None = None,
|
|
sort_order: str | None = None,
|
|
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
|
) -> dict[str, Any]:
|
|
self.persistence.create_tables()
|
|
run_id = self.persistence.start_sync_run(lane)
|
|
failed = 0
|
|
logger.debug(
|
|
"mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
|
|
run_id,
|
|
lane,
|
|
start_page,
|
|
max_pages,
|
|
)
|
|
try:
|
|
data = self.collect_search(
|
|
start_page=start_page,
|
|
max_pages=max_pages,
|
|
search_url=search_url,
|
|
make_id=make_id,
|
|
model_id=model_id,
|
|
price_min=price_min,
|
|
price_max=price_max,
|
|
year_min=year_min,
|
|
year_max=year_max,
|
|
mileage_min=mileage_min,
|
|
mileage_max=mileage_max,
|
|
sort_by=sort_by,
|
|
sort_order=sort_order,
|
|
progress_callback=progress_callback,
|
|
)
|
|
# Map serialized listings back to records.
|
|
records = []
|
|
for page in data["pages"]:
|
|
for item in page["listings"]:
|
|
records.append(self.mapper.listing_to_car_record(MobileDeListing(**item)))
|
|
|
|
skipped_existing = 0
|
|
if only_new and records:
|
|
existing_origin_ids = self.persistence.get_existing_origin_ids(
|
|
[record.origin_id for record in records if record.origin_id]
|
|
)
|
|
before_filter_count = len(records)
|
|
|
|
# For newest-first, cut tail after existing streak.
|
|
should_cut_tail = (
|
|
str(sort_by or "").lower() == "doc"
|
|
and str(sort_order or "").lower() == "down"
|
|
and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
|
|
)
|
|
if should_cut_tail:
|
|
filtered_records = []
|
|
existing_streak = 0
|
|
considered = 0
|
|
for record in records:
|
|
considered += 1
|
|
is_existing = record.origin_id in existing_origin_ids
|
|
if is_existing:
|
|
skipped_existing += 1
|
|
existing_streak += 1
|
|
if (
|
|
existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
|
|
and len(filtered_records) >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
|
|
):
|
|
break
|
|
continue
|
|
existing_streak = 0
|
|
filtered_records.append(record)
|
|
records = filtered_records
|
|
data["listing_count"] = considered
|
|
data["unique_listing_count"] = min(int(data.get("unique_listing_count", considered)), considered)
|
|
logger.info(
|
|
"mobile.de only_new head-cut applied: considered=%s kept=%s skipped_existing=%s streak=%s",
|
|
considered,
|
|
len(records),
|
|
skipped_existing,
|
|
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
|
|
)
|
|
else:
|
|
records = [record for record in records if record.origin_id not in existing_origin_ids]
|
|
skipped_existing = before_filter_count - len(records)
|
|
logger.info(
|
|
"mobile.de only_new filter applied: kept=%s skipped_existing=%s",
|
|
len(records),
|
|
skipped_existing,
|
|
)
|
|
|
|
if progress_callback is not None:
|
|
progress_callback(
|
|
"records_mapped",
|
|
{
|
|
"record_count": len(records),
|
|
"skipped_existing": skipped_existing,
|
|
"only_new": bool(only_new),
|
|
"run_id": run_id,
|
|
},
|
|
)
|
|
|
|
upsert = self.persistence.upsert_cars_batch(records) if records else {
|
|
"inserted": 0,
|
|
"updated": 0,
|
|
"images_upserted": 0,
|
|
}
|
|
logger.debug(
|
|
"mobile.de sync_search upsert finished: run_id=%s inserted=%s updated=%s images=%s",
|
|
run_id,
|
|
upsert.get("inserted", 0),
|
|
upsert.get("updated", 0),
|
|
upsert.get("images_upserted", 0),
|
|
)
|
|
if progress_callback is not None:
|
|
progress_callback(
|
|
"db_upsert_done",
|
|
{
|
|
"run_id": run_id,
|
|
"inserted": int(upsert.get("inserted", 0)),
|
|
"updated": int(upsert.get("updated", 0)),
|
|
"images_upserted": int(upsert.get("images_upserted", 0)),
|
|
},
|
|
)
|
|
self.persistence.finish_sync_run(
|
|
run_id,
|
|
status="success",
|
|
ids_fetched=len(records),
|
|
cars_upserted=int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0)),
|
|
cars_failed=failed,
|
|
images_upserted=int(upsert.get("images_upserted", 0)),
|
|
)
|
|
logger.debug(
|
|
"mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
|
|
run_id,
|
|
data.get("listing_count", 0),
|
|
data.get("unique_listing_count", 0),
|
|
)
|
|
return {"run_id": run_id, "source": "mobile.de", "upsert": upsert, "skipped_existing": skipped_existing, **data}
|
|
except Exception as exc:
|
|
self.persistence.finish_sync_run(
|
|
run_id,
|
|
status="failed",
|
|
ids_fetched=0,
|
|
cars_upserted=0,
|
|
cars_failed=failed,
|
|
images_upserted=0,
|
|
error_summary=str(exc),
|
|
)
|
|
logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
|
|
raise
|
|
|
|
def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
|
|
self.persistence.create_tables()
|
|
detail = self.client.fetch_detail(listing_id)
|
|
record = self.mapper.detail_to_car_record(str(listing_id), detail)
|
|
result = self.persistence.upsert_car(record)
|
|
return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}
|