improve runtime sync flow
This commit is contained in:
qananasikq
@@ -21,6 +21,10 @@ logger = logging.getLogger("mobile_de.scraper")
|
||||
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
|
||||
MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
|
||||
MOBILEDE_LIGHT_REFRESH_EXISTING = os.getenv("MOBILEDE_LIGHT_REFRESH_EXISTING", "false").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED = os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED = os.getenv("MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
|
||||
MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", "20")))
|
||||
MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", "2")))
|
||||
MOBILEDE_SEARCH_STRATEGY_NOTE = (
|
||||
"mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
|
||||
"for full coverage split into narrower segments and deduplicate by id."
|
||||
@@ -94,6 +98,32 @@ class MobileDeScraper:
|
||||
deduped_page_records.append(record)
|
||||
return deduped_page_records
|
||||
|
||||
@staticmethod
|
||||
def _record_key(record: CarRecord) -> str:
|
||||
return record.origin_id or record.origin_url
|
||||
|
||||
@staticmethod
|
||||
def _record_needs_detail_enrich(record: CarRecord) -> bool:
|
||||
return any(
|
||||
(
|
||||
record.year is None,
|
||||
record.mileage == 0,
|
||||
record.engine_volume is None,
|
||||
record.body_type == "OTHER",
|
||||
record.color == "other",
|
||||
)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _detail_enrich_priority(record: CarRecord) -> tuple[int, int, int, int, int]:
|
||||
return (
|
||||
int(record.year is None),
|
||||
int(record.engine_volume is None),
|
||||
int(record.mileage == 0),
|
||||
int(record.body_type == "OTHER"),
|
||||
int(record.color == "other"),
|
||||
)
|
||||
|
||||
def _apply_only_new_page_policy(
|
||||
self,
|
||||
*,
|
||||
@@ -310,6 +340,8 @@ class MobileDeScraper:
|
||||
inserted_total = 0
|
||||
updated_total = 0
|
||||
images_upserted = 0
|
||||
detail_enriched = 0
|
||||
detail_enrich_failed = 0
|
||||
existing_streak = 0
|
||||
new_records_kept = 0
|
||||
head_cut_triggered = False
|
||||
@@ -348,19 +380,36 @@ class MobileDeScraper:
|
||||
|
||||
pages_collected = 0
|
||||
early_stopped = False
|
||||
for page in self.client.iter_search_pages(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
):
|
||||
concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
|
||||
if concurrent_pages > 1 and max_pages and max_pages > 1 and only_new is not True:
|
||||
page_iterator = self.client.fetch_search_pages_concurrent(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
workers=concurrent_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
)
|
||||
else:
|
||||
page_iterator = self.client.iter_search_pages(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
search_url=search_url,
|
||||
progress_callback=_on_page,
|
||||
**params,
|
||||
)
|
||||
for page in page_iterator:
|
||||
pages_collected += 1
|
||||
pages_payload.append(asdict(page))
|
||||
listing_count += len(page.listings)
|
||||
unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
|
||||
|
||||
page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
|
||||
listing_by_record_key = {
|
||||
self._record_key(record): listing
|
||||
for listing, record in zip(page.listings, page_records, strict=False)
|
||||
if self._record_key(record)
|
||||
}
|
||||
page_records = self._dedupe_page_records(page_records, seen_record_keys)
|
||||
for record in page_records:
|
||||
record.is_sold = False
|
||||
@@ -369,7 +418,7 @@ class MobileDeScraper:
|
||||
record.sold_at = None
|
||||
record.skip_image_sync = False
|
||||
existing_origin_ids: set[str] = set()
|
||||
if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING):
|
||||
if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING or MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED):
|
||||
existing_origin_ids = self.persistence.get_existing_origin_ids(
|
||||
[record.origin_id for record in page_records if record.origin_id]
|
||||
)
|
||||
@@ -389,6 +438,66 @@ class MobileDeScraper:
|
||||
for record in page_records:
|
||||
if record.origin_id and record.origin_id in existing_origin_ids:
|
||||
record.skip_image_sync = True
|
||||
if (
|
||||
MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED
|
||||
and detail_enriched < MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN
|
||||
and page_records
|
||||
):
|
||||
remaining_budget = MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN - detail_enriched
|
||||
page_budget = min(MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE, remaining_budget)
|
||||
candidate_records = [
|
||||
record
|
||||
for record in page_records
|
||||
if record.origin_id
|
||||
and record.origin_id not in existing_origin_ids
|
||||
and self._record_needs_detail_enrich(record)
|
||||
]
|
||||
candidate_records.sort(key=self._detail_enrich_priority, reverse=True)
|
||||
selected_keys = {
|
||||
self._record_key(record)
|
||||
for record in candidate_records[:page_budget]
|
||||
}
|
||||
if selected_keys:
|
||||
enriched_records: list[CarRecord] = []
|
||||
for record in page_records:
|
||||
record_key = self._record_key(record)
|
||||
listing = listing_by_record_key.get(record_key)
|
||||
if record_key not in selected_keys or listing is None:
|
||||
enriched_records.append(record)
|
||||
continue
|
||||
try:
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
"detail_enriching",
|
||||
{
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
"page_number": page.page_number,
|
||||
"listing_id": str(listing.id),
|
||||
"detail_enriched": detail_enriched,
|
||||
"detail_enrich_failed": detail_enrich_failed,
|
||||
},
|
||||
)
|
||||
enriched = self.mapper.detail_to_car_record(str(listing.id), self.client.fetch_detail(listing.id))
|
||||
enriched.is_sold = False
|
||||
enriched.first_seen_at = run_seen_at
|
||||
enriched.last_seen_at = run_seen_at
|
||||
enriched.sold_at = None
|
||||
enriched.skip_image_sync = False
|
||||
if not MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:
|
||||
enriched.images = record.images
|
||||
enriched_records.append(enriched)
|
||||
detail_enriched += 1
|
||||
except Exception:
|
||||
logger.warning(
|
||||
"mobile.de selective detail enrich failed: listing_id=%s title=%s",
|
||||
getattr(listing, "id", None),
|
||||
getattr(listing, "title", None),
|
||||
exc_info=True,
|
||||
)
|
||||
detail_enrich_failed += 1
|
||||
enriched_records.append(record)
|
||||
page_records = enriched_records
|
||||
|
||||
if progress_callback is not None:
|
||||
progress_callback(
|
||||
@@ -396,6 +505,8 @@ class MobileDeScraper:
|
||||
{
|
||||
"record_count": len(page_records),
|
||||
"skipped_existing": skipped_existing,
|
||||
"detail_enriched": detail_enriched,
|
||||
"detail_enrich_failed": detail_enrich_failed,
|
||||
"only_new": bool(only_new),
|
||||
"run_id": run_id,
|
||||
"pages_collected": pages_collected,
|
||||
@@ -486,6 +597,8 @@ class MobileDeScraper:
|
||||
"updated": updated_total,
|
||||
"images_upserted": images_upserted,
|
||||
},
|
||||
"detail_enriched": detail_enriched,
|
||||
"detail_enrich_failed": detail_enrich_failed,
|
||||
"skipped_existing": skipped_existing,
|
||||
**data,
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user