refactor mobile.de parser, fix country mapping, update README
This commit is contained in:
qananasikq
@@ -314,9 +314,7 @@ class MobileDeScraper:
|
||||
max_pages,
|
||||
)
|
||||
try:
|
||||
data = self.collect_search(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
params = self._build_search_params(
|
||||
search_url=search_url,
|
||||
make_id=make_id,
|
||||
model_id=model_id,
|
||||
@@ -328,96 +326,121 @@ class MobileDeScraper:
|
||||
mileage_max=mileage_max,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
progress_callback=progress_callback,
|
||||
)
|
||||
|
||||
pages_payload = list(data.get("pages", []))
|
||||
listing_count = int(data.get("listing_count", 0) or 0)
|
||||
unique_ids = set(data.get("unique_listing_ids", []))
|
||||
pages_collected = len(pages_payload)
|
||||
def _on_page(page, meta: dict[str, int | None]) -> None:
|
||||
if progress_callback is None:
|
||||
return
|
||||
payload = {
|
||||
**meta,
|
||||
"page_url": page.url,
|
||||
"unique_ids_seen": len(unique_ids) + len({listing.id for listing in page.listings if listing.id}),
|
||||
}
|
||||
progress_callback("page_collected", payload)
|
||||
|
||||
records: list[CarRecord] = []
|
||||
for page in pages_payload:
|
||||
page_records = [self.mapper.listing_to_car_record(MobileDeListing(**item)) for item in page.get("listings", [])]
|
||||
records.extend(self._dedupe_page_records(page_records, seen_record_keys))
|
||||
pages_collected = 0
|
||||
early_stopped = False
|
||||
for page in self.client.iter_search_pages(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
):
|
||||
pages_collected += 1
|
||||
pages_payload.append(asdict(page))
|
||||
listing_count += len(page.listings)
|
||||
unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
|
||||
|
||||
if only_new and records:
|
||||
existing_origin_ids = self.persistence.get_existing_origin_ids(
|
||||
[record.origin_id for record in records if record.origin_id]
|
||||
)
|
||||
should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
|
||||
if should_cut_tail:
|
||||
filtered_records: list[CarRecord] = []
|
||||
for record_index, record in enumerate(records):
|
||||
is_existing = bool(record.origin_id and record.origin_id in existing_origin_ids)
|
||||
filtered_records.append(record)
|
||||
if is_existing:
|
||||
existing_streak += 1
|
||||
if (
|
||||
existing_streak >= MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK
|
||||
and new_records_kept >= MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS
|
||||
):
|
||||
head_cut_triggered = True
|
||||
skipped_existing += max(0, len(records) - record_index - 1)
|
||||
break
|
||||
else:
|
||||
existing_streak = 0
|
||||
new_records_kept += 1
|
||||
records = filtered_records
|
||||
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"records_mapped",
|
||||
{
|
||||
"record_count": len(records),
|
||||
"skipped_existing": skipped_existing,
|
||||
"only_new": bool(only_new),
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
},
|
||||
page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
|
||||
page_records = self._dedupe_page_records(page_records, seen_record_keys)
|
||||
page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
|
||||
self._apply_only_new_page_policy(
|
||||
page_records=page_records,
|
||||
only_new=only_new,
|
||||
sort_by=sort_by,
|
||||
sort_order=sort_order,
|
||||
skipped_existing=skipped_existing,
|
||||
existing_streak=existing_streak,
|
||||
new_records_kept=new_records_kept,
|
||||
)
|
||||
)
|
||||
|
||||
upsert = self.persistence.upsert_cars_batch(records) if records else {
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"images_upserted": 0,
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"records_mapped",
|
||||
{
|
||||
"record_count": len(page_records),
|
||||
"skipped_existing": skipped_existing,
|
||||
"only_new": bool(only_new),
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
"page_number": page.page_number,
|
||||
},
|
||||
)
|
||||
|
||||
upsert = self.persistence.upsert_cars_batch(page_records) if page_records else {
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"images_upserted": 0,
|
||||
}
|
||||
page_inserted = int(upsert.get("inserted", 0))
|
||||
page_updated = int(upsert.get("updated", 0))
|
||||
page_images = int(upsert.get("images_upserted", 0))
|
||||
|
||||
ids_fetched += len(page_records)
|
||||
inserted_total += page_inserted
|
||||
updated_total += page_updated
|
||||
images_upserted += page_images
|
||||
cars_upserted = inserted_total + updated_total
|
||||
|
||||
logger.debug(
|
||||
"mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
|
||||
run_id,
|
||||
page.page_number,
|
||||
page_inserted,
|
||||
page_updated,
|
||||
page_images,
|
||||
)
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"db_upsert_done",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
"pages_in_batch": 1,
|
||||
"page_number": page.page_number,
|
||||
"inserted": page_inserted,
|
||||
"updated": page_updated,
|
||||
"images_upserted": page_images,
|
||||
},
|
||||
)
|
||||
|
||||
if head_cut_triggered:
|
||||
early_stopped = True
|
||||
break
|
||||
|
||||
data = {
|
||||
"source": "mobile.de",
|
||||
"strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
|
||||
"search_url": search_url,
|
||||
"pages": pages_payload,
|
||||
"listing_count": listing_count,
|
||||
"unique_listing_count": len(unique_ids),
|
||||
"unique_listing_ids": sorted(unique_ids),
|
||||
"early_stopped": early_stopped,
|
||||
}
|
||||
ids_fetched = len(records)
|
||||
inserted_total = int(upsert.get("inserted", 0))
|
||||
updated_total = int(upsert.get("updated", 0))
|
||||
images_upserted = int(upsert.get("images_upserted", 0))
|
||||
cars_upserted = inserted_total + updated_total
|
||||
|
||||
logger.debug(
|
||||
"mobile.de sync_search batch upsert: run_id=%s pages=%s inserted=%s updated=%s images=%s",
|
||||
run_id,
|
||||
pages_collected,
|
||||
inserted_total,
|
||||
updated_total,
|
||||
images_upserted,
|
||||
)
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"db_upsert_done",
|
||||
"search_collection_done",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
"pages_in_batch": pages_collected,
|
||||
"inserted": inserted_total,
|
||||
"updated": updated_total,
|
||||
"images_upserted": images_upserted,
|
||||
"listing_count": listing_count,
|
||||
"unique_listing_count": len(unique_ids),
|
||||
},
|
||||
)
|
||||
|
||||
if head_cut_triggered:
|
||||
logger.info(
|
||||
"mobile.de only_new head-cut applied: kept=%s skipped_existing=%s streak=%s",
|
||||
len(records),
|
||||
skipped_existing,
|
||||
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK,
|
||||
)
|
||||
|
||||
data["early_stopped"] = head_cut_triggered
|
||||
self.persistence.finish_sync_run(
|
||||
run_id,
|
||||
status="success",
|
||||
@@ -432,7 +455,16 @@ class MobileDeScraper:
|
||||
data.get("listing_count", 0),
|
||||
data.get("unique_listing_count", 0),
|
||||
)
|
||||
return {"run_id": run_id, "upsert": upsert, "skipped_existing": skipped_existing, **data}
|
||||
return {
|
||||
"run_id": run_id,
|
||||
"upsert": {
|
||||
"inserted": inserted_total,
|
||||
"updated": updated_total,
|
||||
"images_upserted": images_upserted,
|
||||
},
|
||||
"skipped_existing": skipped_existing,
|
||||
**data,
|
||||
}
|
||||
except Exception as exc:
|
||||
self.persistence.finish_sync_run(
|
||||
run_id,
|
||||
|
||||
Reference in New Issue
Block a user