qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Apply runtime configuration

Browse Source
This commit is contained in:
qananasikq
2026-08-05 18:13:53 +03:00
parent fba7299747
commit 2d3260d8f9
4 changed files with 103 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files

43
mobilede_scraper/mobile_de/scraper.py
View File

@@ -10,6 +10,7 @@ from typing import Any
import requests
from ..core.config import Settings, settings
from ..core.runtime_config import RuntimeFiltersConfig
from ..storage.db import PersistenceService
from ..storage.schemas import CarRecord
from .client import MobileDeClient
@@ -69,13 +70,12 @@ class MobileDeScraper:
sort_by: str | None,
sort_order: str | None,
) -> dict[str, str | None]:
params: dict[str, str | None] = {}
params: dict[str, str | None] = {
"p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
"fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
"ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
}
if not search_url:
params = {
"p": f"{price_min or ''}:{price_max or ''}" if price_min or price_max else None,
"fr": f"{year_min or ''}:{year_max or ''}" if year_min or year_max else None,
"ml": f"{mileage_min or ''}:{mileage_max or ''}" if mileage_min or mileage_max else None,
}
if make_id:
params["ms"] = self.client.build_make_model_param(make_id, model_id)
if sort_by:
@@ -325,7 +325,14 @@ class MobileDeScraper:
sort_order: str | None = None,
seen_at: datetime | None = None,
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
runtime_filters: RuntimeFiltersConfig | None = None,
limit: int | None = None,
) -> dict[str, Any]:
if runtime_filters is not None and runtime_filters.flags.run_and_drive is not None:
raise ValueError(
"mobile.de does not support the runtime filter flags.run_and_drive: "
"the source payload has no reliable vehicle-condition field"
)
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane)
run_seen_at = seen_at or datetime.now(timezone.utc)
@@ -346,6 +353,8 @@ class MobileDeScraper:
new_records_kept = 0
head_cut_triggered = False
seen_record_keys: set[str] = set()
accepted_records = 0
effective_limit = max(0, int(limit)) if limit is not None else None
logger.debug(
"mobile.de sync_search started: run_id=%s lane=%s start_page=%s max_pages=%s",
run_id,
@@ -411,6 +420,25 @@ class MobileDeScraper:
if self._record_key(record)
}
page_records = self._dedupe_page_records(page_records, seen_record_keys)
if runtime_filters is not None:
page_records = [
record for record in page_records
if runtime_filters.matches({
"brand": record.brand,
"model": record.model,
"year": record.year,
"price": record.price,
"mileage": record.mileage,
"body_type": record.body_type,
"color": record.color,
"drive": record.drive,
"gearbox": record.gearbox,
"location": record.country,
"is_damaged": record.is_damaged,
})
]
if effective_limit is not None:
page_records = page_records[:max(0, effective_limit - accepted_records)]
for record in page_records:
record.is_sold = False
record.first_seen_at = run_seen_at
@@ -528,6 +556,7 @@ class MobileDeScraper:
updated_total += page_updated
images_upserted += page_images
cars_upserted = inserted_total + updated_total
accepted_records += len(page_records)
logger.debug(
"mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
@@ -551,7 +580,7 @@ class MobileDeScraper:
},
)
if head_cut_triggered:
if head_cut_triggered or (effective_limit is not None and accepted_records >= effective_limit):
early_stopped = True
break

15
mobilede_scraper/worker/search_sync.py
View File

@@ -423,7 +423,20 @@ def run_mobilede_sync_search_task(
scraper = MobileDeScraper(
client=MobileDeClient.for_worker(delay_seconds=delay_seconds),
persistence=_get_persistence(),
runtime_settings=settings,
)
runtime_filters = runtime_config.filters
if runtime_filters.price.min is not None:
price_min = str(runtime_filters.price.min)
if runtime_filters.price.max is not None:
price_max = str(runtime_filters.price.max)
if runtime_filters.mileage.min is not None:
mileage_min = str(runtime_filters.mileage.min)
if runtime_filters.mileage.max is not None:
mileage_max = str(runtime_filters.mileage.max)
if runtime_filters.include.years:
year_min = str(min(runtime_filters.include.years))
year_max = str(max(runtime_filters.include.years))
run_seen_at = _mobilede_refresh_cycle_seen_at(redis_client, cycle_id=refresh_cycle_id)
result = scraper.sync_search(
start_page=actual_start_page,
@@ -443,6 +456,8 @@ def run_mobilede_sync_search_task(
mileage_max=mileage_max,
seen_at=run_seen_at,
progress_callback=_progress,
runtime_filters=runtime_filters,
limit=runtime_config.sync.limit,
)
inserted_count = int(result.get("upsert", {}).get("inserted", 0) or 0)
updated_count = int(result.get("upsert", {}).get("updated", 0) or 0)