qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Track sold cars

Browse Source
This commit is contained in:
qananasikq
2026-05-06 21:07:24 +03:00
parent 81e99a41e8
commit 519686c0b7
8 changed files with 322 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
mobilede_scraper/mobile_de/scraper.py
View File

@@ -4,6 +4,7 @@ import logging
import os
from collections.abc import Callable
from dataclasses import asdict
from datetime import datetime, timezone
from typing import Any
import requests
@@ -19,6 +20,7 @@ logger = logging.getLogger("mobile_de.scraper")
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
MOBILEDE_LIGHT_REFRESH_EXISTING = os.getenv("MOBILEDE_LIGHT_REFRESH_EXISTING", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SEARCH_STRATEGY_NOTE = (
"mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
"for full coverage split into narrower segments and deduplicate by id."
@@ -102,13 +104,15 @@ class MobileDeScraper:
skipped_existing: int,
existing_streak: int,
new_records_kept: int,
existing_origin_ids: set[str] | None = None,
) -> tuple[list[CarRecord], int, int, int, bool]:
if not only_new or not page_records:
return page_records, skipped_existing, existing_streak, new_records_kept, False
existing_origin_ids = self.persistence.get_existing_origin_ids(
[record.origin_id for record in page_records if record.origin_id]
)
if existing_origin_ids is None:
existing_origin_ids = self.persistence.get_existing_origin_ids(
[record.origin_id for record in page_records if record.origin_id]
)
head_cut_triggered = False
should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
@@ -289,10 +293,14 @@ class MobileDeScraper:
mileage_max: str | None = None,
sort_by: str | None = None,
sort_order: str | None = None,
seen_at: datetime | None = None,
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
) -> dict[str, Any]:
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane)
run_seen_at = seen_at or datetime.now(timezone.utc)
if run_seen_at.tzinfo is None:
run_seen_at = run_seen_at.replace(tzinfo=timezone.utc)
pages_payload: list[dict[str, Any]] = []
unique_ids: set[str] = set()
listing_count = 0
@@ -354,6 +362,17 @@ class MobileDeScraper:
page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
page_records = self._dedupe_page_records(page_records, seen_record_keys)
for record in page_records:
record.is_sold = False
record.first_seen_at = run_seen_at
record.last_seen_at = run_seen_at
record.sold_at = None
record.skip_image_sync = False
existing_origin_ids: set[str] = set()
if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING):
existing_origin_ids = self.persistence.get_existing_origin_ids(
[record.origin_id for record in page_records if record.origin_id]
)
page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
self._apply_only_new_page_policy(
page_records=page_records,
@@ -363,8 +382,13 @@ class MobileDeScraper:
skipped_existing=skipped_existing,
existing_streak=existing_streak,
new_records_kept=new_records_kept,
existing_origin_ids=existing_origin_ids,
)
)
if MOBILEDE_LIGHT_REFRESH_EXISTING and existing_origin_ids:
for record in page_records:
if record.origin_id and record.origin_id in existing_origin_ids:
record.skip_image_sync = True
if progress_callback is not None:
progress_callback(