qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply runtime configuration

Browse Source
This commit is contained in:
qananasikq
2026-08-05 18:13:53 +03:00
parent fba7299747
commit 2d3260d8f9
4 changed files with 103 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
mobilede_scraper/mobile_de/scraper.py
View File

@@ -10,6 +10,7 @@ from typing import Any
import requests
from ..core.config import Settings, settings
from ..core.runtime_config import RuntimeFiltersConfig
from ..storage.db import PersistenceService
from ..storage.schemas import CarRecord
from .client import MobileDeClient
@@ -69,13 +70,12 @@ class MobileDeScraper:
sort_by: str | None,
sort_order: str | None,
) -> dict[str, str | None]:
params: dict[str, str | None] = {}
params: dict[str, str | None] = {
"p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
"fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
"ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
}
if not search_url:
params = {
"p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
"fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
"ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
}
if make_id:
params["ms"] = self.client.build_make_model_param(make_id, model_id)
if sort_by:
@@ -325,7 +325,14 @@ class MobileDeScraper:
sort_order: str | None = None,
seen_at: datetime | None = None,
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
runtime_filters: RuntimeFiltersConfig | None = None,
limit: int | None = None,
) -> dict[str, Any]:
if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None:
raise ValueError(
"mobile.de does not support the runtime filter flags.run_and_drive: "
"the source payload has no reliable vehicle-condition field"
)
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane)
run_seen_at = seen_at or datetime.now(timezone.utc)
@@ -346,6 +353,8 @@ class MobileDeScraper:
new_records_kept = 0
head_cut_triggered = False
seen_record_keys: set[str] = set()
accepted_records = 0
effective_limit = max(0, int(limit)) if limit is not None else None
logger.debug(
"mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
run_id,
@@ -411,6 +420,25 @@ class MobileDeScraper:
if self._record_key(record)
}
page_records = self._dedupe_page_records(page_records, seen_record_keys)
if runtime_filters is not None:
page_records = [
record for record in page_records
if runtime_filters.matches({
"brand": record.brand,
"model": record.model,
"year": record.year,
"price": record.price,
"mileage": record.mileage,
"body_type": record.body_type,
"color": record.color,
"drive": record.drive,
"gearbox": record.gearbox,
"location": record.country,
"is_damaged": record.is_damaged,
})
]
if effective_limit is not None:
page_records = page_records[:max(0, effective_limit - accepted_records)]
for record in page_records:
record.is_sold = False
record.first_seen_at = run_seen_at
@@ -528,6 +556,7 @@ class MobileDeScraper:
updated_total += page_updated
images_upserted += page_images
cars_upserted = inserted_total + updated_total
accepted_records += len(page_records)
logger.debug(
"mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
@@ -551,7 +580,7 @@ class MobileDeScraper:
},
)
if head_cut_triggered:
if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit):
early_stopped = True
break