653 lines
27 KiB
Python
653 lines
27 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import os
|
|
from collections.abc import Callable
|
|
from dataclasses import asdict
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
|
|
import requests
|
|
|
|
from ..core.config import Settings, settings
|
|
from ..core.runtime_config import RuntimeFiltersConfig
|
|
from ..storage.db import PersistenceService
|
|
from ..storage.schemas import CarRecord
|
|
from .client import MobileDeClient
|
|
from .mapper import MobileDeMapper
|
|
from .models import MobileDeListing
|
|
|
|
logger = logging.getLogger("mobile_de.scraper")
|
|
|
|
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
|
|
MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
|
|
MOBILEDE_LIGHT_REFRESH_EXISTING = os.getenv("MOBILEDE_LIGHT_REFRESH_EXISTING", "false").strip().lower() in {"1", "true", "yes", "on"}
|
|
MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED = os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
|
|
MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED = os.getenv("MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
|
|
MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", "20")))
|
|
MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", "2")))
|
|
MOBILEDE_SEARCH_STRATEGY_NOTE = (
|
|
"mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
|
|
"for full coverage split into narrower segments and deduplicate by id."
|
|
)
|
|
|
|
|
|
class MobileDeScraper:
|
|
"""High-level mobile.de scraper facade."""
|
|
|
|
def __init__(
|
|
self,
|
|
client: MobileDeClient | None = None,
|
|
persistence: PersistenceService | None = None,
|
|
mapper: MobileDeMapper | None = None,
|
|
runtime_settings: Settings | None = None,
|
|
) -> None:
|
|
self.settings = runtime_settings or settings
|
|
self.client = client or MobileDeClient()
|
|
self.persistence = persistence or PersistenceService(self.settings)
|
|
self.mapper = mapper or MobileDeMapper()
|
|
|
|
@staticmethod
|
|
def _should_cut_only_new_tail(sort_by: str | None, sort_order: str | None) -> bool:
|
|
return (
|
|
str(sort_by or "").lower() == "doc"
|
|
and str(sort_order or "").lower() == "down"
|
|
and MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK > 0
|
|
)
|
|
|
|
def _build_search_params(
|
|
self,
|
|
*,
|
|
search_url: str | None,
|
|
make_id: str | None,
|
|
model_id: str | None,
|
|
price_min: str | None,
|
|
price_max: str | None,
|
|
year_min: str | None,
|
|
year_max: str | None,
|
|
mileage_min: str | None,
|
|
mileage_max: str | None,
|
|
sort_by: str | None,
|
|
sort_order: str | None,
|
|
) -> dict[str, str | None]:
|
|
params: dict[str, str | None] = {
|
|
"p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
|
|
"fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
|
|
"ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
|
|
}
|
|
if not search_url:
|
|
if make_id:
|
|
params["ms"] = self.client.build_make_model_param(make_id, model_id)
|
|
if sort_by:
|
|
params["sb"] = sort_by
|
|
if sort_order:
|
|
params["od"] = sort_order
|
|
return params
|
|
|
|
@staticmethod
|
|
def _dedupe_page_records(
|
|
page_records: list[CarRecord],
|
|
seen_record_keys: set[str],
|
|
) -> list[CarRecord]:
|
|
deduped_page_records: list[CarRecord] = []
|
|
for record in page_records:
|
|
record_key = record.origin_id or record.origin_url
|
|
if not record_key or record_key in seen_record_keys:
|
|
continue
|
|
seen_record_keys.add(record_key)
|
|
deduped_page_records.append(record)
|
|
return deduped_page_records
|
|
|
|
@staticmethod
|
|
def _record_key(record: CarRecord) -> str:
|
|
return record.origin_id or record.origin_url
|
|
|
|
@staticmethod
|
|
def _record_needs_detail_enrich(record: CarRecord) -> bool:
|
|
return any(
|
|
(
|
|
record.year is None,
|
|
record.mileage == 0,
|
|
record.engine_volume is None,
|
|
record.body_type == "OTHER",
|
|
record.color == "other",
|
|
)
|
|
)
|
|
|
|
@staticmethod
|
|
def _detail_enrich_priority(record: CarRecord) -> tuple[int, int, int, int, int]:
|
|
return (
|
|
int(record.year is None),
|
|
int(record.engine_volume is None),
|
|
int(record.mileage == 0),
|
|
int(record.body_type == "OTHER"),
|
|
int(record.color == "other"),
|
|
)
|
|
|
|
def _apply_only_new_page_policy(
|
|
self,
|
|
*,
|
|
page_records: list[CarRecord],
|
|
only_new: bool | None,
|
|
sort_by: str | None,
|
|
sort_order: str | None,
|
|
skipped_existing: int,
|
|
existing_streak: int,
|
|
new_records_kept: int,
|
|
existing_origin_ids: set[str] | None = None,
|
|
) -> tuple[list[CarRecord], int, int, int, bool]:
|
|
if not only_new or not page_records:
|
|
return page_records, skipped_existing, existing_streak, new_records_kept, False
|
|
|
|
if existing_origin_ids is None:
|
|
existing_origin_ids = self.persistence.get_existing_origin_ids(
|
|
[record.origin_id for record in page_records if record.origin_id]
|
|
)
|
|
head_cut_triggered = False
|
|
should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
|
|
|
|
if should_cut_tail:
|
|
filtered_records: list[CarRecord] = []
|
|
for record_index, record in enumerate(page_records):
|
|
is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
|
|
if is_existing:
|
|
existing_streak += 1
|
|
if (
|
|
existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
|
|
and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
|
|
):
|
|
head_cut_triggered = True
|
|
skipped_existing += max(0, len(page_records) - record_index - 1)
|
|
break
|
|
else:
|
|
existing_streak = 0
|
|
new_records_kept += 1
|
|
filtered_records.append(record)
|
|
return filtered_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered
|
|
|
|
for record in page_records:
|
|
if not record.origin_id or record.origin_id not in existing_origin_ids:
|
|
new_records_kept += 1
|
|
return page_records, skipped_existing, existing_streak, new_records_kept, False
|
|
|
|
def collect_search(
|
|
self,
|
|
*,
|
|
start_page: int = 1,
|
|
max_pages: int = 1,
|
|
search_url: str | None = None,
|
|
make_id: str | None = None,
|
|
model_id: str | None = None,
|
|
price_min: str | None = None,
|
|
price_max: str | None = None,
|
|
year_min: str | None = None,
|
|
year_max: str | None = None,
|
|
mileage_min: str | None = None,
|
|
mileage_max: str | None = None,
|
|
sort_by: str | None = None,
|
|
sort_order: str | None = None,
|
|
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
|
) -> dict[str, Any]:
|
|
params = self._build_search_params(
|
|
search_url=search_url,
|
|
make_id=make_id,
|
|
model_id=model_id,
|
|
price_min=price_min,
|
|
price_max=price_max,
|
|
year_min=year_min,
|
|
year_max=year_max,
|
|
mileage_min=mileage_min,
|
|
mileage_max=mileage_max,
|
|
sort_by=sort_by,
|
|
sort_order=sort_order,
|
|
)
|
|
|
|
logger.debug(
|
|
"mobile.de collect_search started: start_page=%s max_pages=%s search_url=%s make_id=%s model_id=%s year=%s-%s price=%s-%s mileage=%s-%s",
|
|
start_page,
|
|
max_pages,
|
|
bool(search_url),
|
|
make_id,
|
|
model_id,
|
|
year_min,
|
|
year_max,
|
|
price_min,
|
|
price_max,
|
|
mileage_min,
|
|
mileage_max,
|
|
)
|
|
pages = []
|
|
|
|
def _on_page(page, meta: dict[str, int | None]) -> None:
|
|
payload = {
|
|
**meta,
|
|
"page_url": page.url,
|
|
"unique_ids_seen": len({listing.id for item in pages for listing in item.listings if listing.id})
|
|
+ len({listing.id for listing in page.listings if listing.id}),
|
|
}
|
|
if progress_callback is not None:
|
|
progress_callback("page_collected", payload)
|
|
|
|
concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
|
|
if concurrent_pages > 1 and max_pages and max_pages > 1:
|
|
try:
|
|
pages.extend(self.client.fetch_search_pages_concurrent(
|
|
start_page=start_page,
|
|
max_pages=max_pages,
|
|
workers=concurrent_pages,
|
|
search_url=search_url,
|
|
progress_callback=_on_page,
|
|
**params,
|
|
))
|
|
except requests.RequestException as exc:
|
|
logger.warning(
|
|
"mobile.de concurrent fetch failed, fallback to sequential: workers=%s start_page=%s max_pages=%s error=%s",
|
|
concurrent_pages,
|
|
start_page,
|
|
max_pages,
|
|
exc,
|
|
)
|
|
pages.clear()
|
|
for page in self.client.iter_search_pages(
|
|
start_page=start_page,
|
|
max_pages=max_pages,
|
|
search_url=search_url,
|
|
progress_callback=_on_page,
|
|
**params,
|
|
):
|
|
pages.append(page)
|
|
else:
|
|
for page in self.client.iter_search_pages(
|
|
start_page=start_page,
|
|
max_pages=max_pages,
|
|
search_url=search_url,
|
|
progress_callback=_on_page,
|
|
**params,
|
|
):
|
|
pages.append(page)
|
|
|
|
unique_ids = sorted({listing.id for page in pages for listing in page.listings if listing.id})
|
|
result = {
|
|
"source": "mobile.de",
|
|
"strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
|
|
"search_url": search_url,
|
|
"pages": [asdict(page) for page in pages],
|
|
"listing_count": sum(len(page.listings) for page in pages),
|
|
"unique_listing_count": len(unique_ids),
|
|
"unique_listing_ids": unique_ids,
|
|
}
|
|
logger.debug(
|
|
"mobile.de collect_search finished: pages=%s listings=%s unique=%s",
|
|
len(pages),
|
|
result["listing_count"],
|
|
result["unique_listing_count"],
|
|
)
|
|
if progress_callback is not None:
|
|
progress_callback(
|
|
"search_collection_done",
|
|
{
|
|
"pages_collected": len(pages),
|
|
"listing_count": result["listing_count"],
|
|
"unique_listing_count": result["unique_listing_count"],
|
|
},
|
|
)
|
|
return result
|
|
|
|
def collect_detail(self, listing_id: str) -> dict[str, Any]:
|
|
return {
|
|
"source": "mobile.de",
|
|
"listing_id": str(listing_id),
|
|
"url": self.client.build_detail_url(listing_id),
|
|
"listing": self.client.fetch_detail(listing_id),
|
|
}
|
|
|
|
def init_db(self) -> dict[str, Any]:
|
|
self.persistence.create_tables()
|
|
return {"status": "ok", "source": "mobile.de"}
|
|
|
|
def sync_search(
|
|
self,
|
|
*,
|
|
start_page: int = 1,
|
|
max_pages: int = 1,
|
|
lane: str = "mobile_de_cars",
|
|
only_new: bool | None = None,
|
|
search_url: str | None = None,
|
|
make_id: str | None = None,
|
|
model_id: str | None = None,
|
|
price_min: str | None = None,
|
|
price_max: str | None = None,
|
|
year_min: str | None = None,
|
|
year_max: str | None = None,
|
|
mileage_min: str | None = None,
|
|
mileage_max: str | None = None,
|
|
sort_by: str | None = None,
|
|
sort_order: str | None = None,
|
|
seen_at: datetime | None = None,
|
|
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
|
|
runtime_filters: RuntimeFiltersConfig | None = None,
|
|
limit: int | None = None,
|
|
) -> dict[str, Any]:
|
|
if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None:
|
|
raise ValueError(
|
|
"mobile.de does not support the runtime filter flags.run_and_drive: "
|
|
"the source payload has no reliable vehicle-condition field"
|
|
)
|
|
self.persistence.create_tables()
|
|
run_id = self.persistence.start_sync_run(lane)
|
|
run_seen_at = seen_at or datetime.now(timezone.utc)
|
|
if run_seen_at.tzinfo is None:
|
|
run_seen_at = run_seen_at.replace(tzinfo=timezone.utc)
|
|
pages_payload: list[dict[str, Any]] = []
|
|
unique_ids: set[str] = set()
|
|
listing_count = 0
|
|
skipped_existing = 0
|
|
ids_fetched = 0
|
|
cars_upserted = 0
|
|
inserted_total = 0
|
|
updated_total = 0
|
|
images_upserted = 0
|
|
detail_enriched = 0
|
|
detail_enrich_failed = 0
|
|
existing_streak = 0
|
|
new_records_kept = 0
|
|
head_cut_triggered = False
|
|
seen_record_keys: set[str] = set()
|
|
accepted_records = 0
|
|
effective_limit = max(0, int(limit)) if limit is not None else None
|
|
logger.debug(
|
|
"mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
|
|
run_id,
|
|
lane,
|
|
start_page,
|
|
max_pages,
|
|
)
|
|
try:
|
|
params = self._build_search_params(
|
|
search_url=search_url,
|
|
make_id=make_id,
|
|
model_id=model_id,
|
|
price_min=price_min,
|
|
price_max=price_max,
|
|
year_min=year_min,
|
|
year_max=year_max,
|
|
mileage_min=mileage_min,
|
|
mileage_max=mileage_max,
|
|
sort_by=sort_by,
|
|
sort_order=sort_order,
|
|
)
|
|
|
|
def _on_page(page, meta: dict[str, int | None]) -> None:
|
|
if progress_callback is None:
|
|
return
|
|
payload = {
|
|
**meta,
|
|
"page_url": page.url,
|
|
"unique_ids_seen": len(unique_ids) + len({listing.id for listing in page.listings if listing.id}),
|
|
}
|
|
progress_callback("page_collected", payload)
|
|
|
|
pages_collected = 0
|
|
early_stopped = False
|
|
concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
|
|
if concurrent_pages > 1 and max_pages and max_pages > 1 and only_new is not True:
|
|
page_iterator = self.client.fetch_search_pages_concurrent(
|
|
start_page=start_page,
|
|
max_pages=max_pages,
|
|
workers=concurrent_pages,
|
|
search_url=search_url,
|
|
progress_callback=_on_page,
|
|
**params,
|
|
)
|
|
else:
|
|
page_iterator = self.client.iter_search_pages(
|
|
start_page=start_page,
|
|
max_pages=max_pages,
|
|
search_url=search_url,
|
|
progress_callback=_on_page,
|
|
**params,
|
|
)
|
|
for page in page_iterator:
|
|
pages_collected += 1
|
|
pages_payload.append(asdict(page))
|
|
listing_count += len(page.listings)
|
|
unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
|
|
|
|
page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
|
|
listing_by_record_key = {
|
|
self._record_key(record): listing
|
|
for listing, record in zip(page.listings, page_records, strict=False)
|
|
if self._record_key(record)
|
|
}
|
|
page_records = self._dedupe_page_records(page_records, seen_record_keys)
|
|
if runtime_filters is not None:
|
|
page_records = [
|
|
record for record in page_records
|
|
if runtime_filters.matches({
|
|
"brand": record.brand,
|
|
"model": record.model,
|
|
"year": record.year,
|
|
"price": record.price,
|
|
"mileage": record.mileage,
|
|
"body_type": record.body_type,
|
|
"color": record.color,
|
|
"drive": record.drive,
|
|
"gearbox": record.gearbox,
|
|
"location": record.country,
|
|
"is_damaged": record.is_damaged,
|
|
})
|
|
]
|
|
if effective_limit is not None:
|
|
page_records = page_records[:max(0, effective_limit - accepted_records)]
|
|
for record in page_records:
|
|
record.is_sold = False
|
|
record.first_seen_at = run_seen_at
|
|
record.last_seen_at = run_seen_at
|
|
record.sold_at = None
|
|
record.skip_image_sync = False
|
|
existing_origin_ids: set[str] = set()
|
|
if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING or MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED):
|
|
existing_origin_ids = self.persistence.get_existing_origin_ids(
|
|
[record.origin_id for record in page_records if record.origin_id]
|
|
)
|
|
page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
|
|
self._apply_only_new_page_policy(
|
|
page_records=page_records,
|
|
only_new=only_new,
|
|
sort_by=sort_by,
|
|
sort_order=sort_order,
|
|
skipped_existing=skipped_existing,
|
|
existing_streak=existing_streak,
|
|
new_records_kept=new_records_kept,
|
|
existing_origin_ids=existing_origin_ids,
|
|
)
|
|
)
|
|
if MOBILEDE_LIGHT_REFRESH_EXISTING and existing_origin_ids:
|
|
for record in page_records:
|
|
if record.origin_id and record.origin_id in existing_origin_ids:
|
|
record.skip_image_sync = True
|
|
if (
|
|
MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED
|
|
and detail_enriched < MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN
|
|
and page_records
|
|
):
|
|
remaining_budget = MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN - detail_enriched
|
|
page_budget = min(MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE, remaining_budget)
|
|
candidate_records = [
|
|
record
|
|
for record in page_records
|
|
if record.origin_id
|
|
and record.origin_id not in existing_origin_ids
|
|
and self._record_needs_detail_enrich(record)
|
|
]
|
|
candidate_records.sort(key=self._detail_enrich_priority, reverse=True)
|
|
selected_keys = {
|
|
self._record_key(record)
|
|
for record in candidate_records[:page_budget]
|
|
}
|
|
if selected_keys:
|
|
enriched_records: list[CarRecord] = []
|
|
for record in page_records:
|
|
record_key = self._record_key(record)
|
|
listing = listing_by_record_key.get(record_key)
|
|
if record_key not in selected_keys or listing is None:
|
|
enriched_records.append(record)
|
|
continue
|
|
try:
|
|
if progress_callback is not None:
|
|
progress_callback(
|
|
"detail_enriching",
|
|
{
|
|
"run_id": run_id,
|
|
"pages_collected": pages_collected,
|
|
"page_number": page.page_number,
|
|
"listing_id": str(listing.id),
|
|
"detail_enriched": detail_enriched,
|
|
"detail_enrich_failed": detail_enrich_failed,
|
|
},
|
|
)
|
|
enriched = self.mapper.detail_to_car_record(str(listing.id), self.client.fetch_detail(listing.id))
|
|
enriched.is_sold = False
|
|
enriched.first_seen_at = run_seen_at
|
|
enriched.last_seen_at = run_seen_at
|
|
enriched.sold_at = None
|
|
enriched.skip_image_sync = False
|
|
if not MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:
|
|
enriched.images = record.images
|
|
enriched_records.append(enriched)
|
|
detail_enriched += 1
|
|
except Exception:
|
|
logger.warning(
|
|
"mobile.de selective detail enrich failed: listing_id=%s title=%s",
|
|
getattr(listing, "id", None),
|
|
getattr(listing, "title", None),
|
|
exc_info=True,
|
|
)
|
|
detail_enrich_failed += 1
|
|
enriched_records.append(record)
|
|
page_records = enriched_records
|
|
|
|
if progress_callback is not None:
|
|
progress_callback(
|
|
"records_mapped",
|
|
{
|
|
"record_count": len(page_records),
|
|
"skipped_existing": skipped_existing,
|
|
"detail_enriched": detail_enriched,
|
|
"detail_enrich_failed": detail_enrich_failed,
|
|
"only_new": bool(only_new),
|
|
"run_id": run_id,
|
|
"pages_collected": pages_collected,
|
|
"page_number": page.page_number,
|
|
},
|
|
)
|
|
|
|
upsert = self.persistence.upsert_cars_batch(page_records) if page_records else {
|
|
"inserted": 0,
|
|
"updated": 0,
|
|
"images_upserted": 0,
|
|
}
|
|
page_inserted = int(upsert.get("inserted", 0))
|
|
page_updated = int(upsert.get("updated", 0))
|
|
page_images = int(upsert.get("images_upserted", 0))
|
|
|
|
ids_fetched += len(page_records)
|
|
inserted_total += page_inserted
|
|
updated_total += page_updated
|
|
images_upserted += page_images
|
|
cars_upserted = inserted_total + updated_total
|
|
accepted_records += len(page_records)
|
|
|
|
logger.debug(
|
|
"mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
|
|
run_id,
|
|
page.page_number,
|
|
page_inserted,
|
|
page_updated,
|
|
page_images,
|
|
)
|
|
if progress_callback is not None:
|
|
progress_callback(
|
|
"db_upsert_done",
|
|
{
|
|
"run_id": run_id,
|
|
"pages_collected": pages_collected,
|
|
"pages_in_batch": 1,
|
|
"page_number": page.page_number,
|
|
"inserted": page_inserted,
|
|
"updated": page_updated,
|
|
"images_upserted": page_images,
|
|
},
|
|
)
|
|
|
|
if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit):
|
|
early_stopped = True
|
|
break
|
|
|
|
data = {
|
|
"source": "mobile.de",
|
|
"strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
|
|
"search_url": search_url,
|
|
"pages": pages_payload,
|
|
"listing_count": listing_count,
|
|
"unique_listing_count": len(unique_ids),
|
|
"unique_listing_ids": sorted(unique_ids),
|
|
"early_stopped": early_stopped,
|
|
}
|
|
|
|
if progress_callback is not None:
|
|
progress_callback(
|
|
"search_collection_done",
|
|
{
|
|
"pages_collected": pages_collected,
|
|
"listing_count": listing_count,
|
|
"unique_listing_count": len(unique_ids),
|
|
},
|
|
)
|
|
|
|
self.persistence.finish_sync_run(
|
|
run_id,
|
|
status="success",
|
|
ids_fetched=ids_fetched,
|
|
cars_upserted=cars_upserted,
|
|
cars_failed=0,
|
|
images_upserted=images_upserted,
|
|
)
|
|
logger.debug(
|
|
"mobile.de sync_search completed: run_id=%s listings=%s unique=%s",
|
|
run_id,
|
|
data.get("listing_count", 0),
|
|
data.get("unique_listing_count", 0),
|
|
)
|
|
return {
|
|
"run_id": run_id,
|
|
"upsert": {
|
|
"inserted": inserted_total,
|
|
"updated": updated_total,
|
|
"images_upserted": images_upserted,
|
|
},
|
|
"detail_enriched": detail_enriched,
|
|
"detail_enrich_failed": detail_enrich_failed,
|
|
"skipped_existing": skipped_existing,
|
|
**data,
|
|
}
|
|
except Exception as exc:
|
|
self.persistence.finish_sync_run(
|
|
run_id,
|
|
status="failed",
|
|
ids_fetched=ids_fetched,
|
|
cars_upserted=cars_upserted,
|
|
cars_failed=0,
|
|
images_upserted=images_upserted,
|
|
error_summary=str(exc),
|
|
)
|
|
logger.error("mobile.de sync_search failed: run_id=%s error=%s", run_id, exc, exc_info=True)
|
|
raise
|
|
|
|
def sync_detail(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
|
|
self.persistence.create_tables()
|
|
detail = self.client.fetch_detail(listing_id)
|
|
record = self.mapper.detail_to_car_record(str(listing_id), detail)
|
|
result = self.persistence.upsert_car(record)
|
|
return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}
|