Queue gallery enrichment jobs
This commit is contained in:
qananasikq
@@ -15,7 +15,6 @@ from ..storage.db import PersistenceService
|
||||
from ..storage.schemas import CarRecord
|
||||
from .client import MobileDeClient
|
||||
from .mapper import MobileDeMapper
|
||||
from .models import MobileDeListing
|
||||
|
||||
logger = logging.getLogger("mobile_de.scraper")
|
||||
|
||||
@@ -344,8 +343,12 @@ class MobileDeScraper:
|
||||
skipped_existing = 0
|
||||
ids_fetched = 0
|
||||
cars_upserted = 0
|
||||
cars_failed = 0
|
||||
inserted_total = 0
|
||||
updated_total = 0
|
||||
changed_total = 0
|
||||
unchanged_total = 0
|
||||
reappeared_total = 0
|
||||
images_upserted = 0
|
||||
detail_enriched = 0
|
||||
detail_enrich_failed = 0
|
||||
@@ -377,7 +380,13 @@ class MobileDeScraper:
|
||||
sort_order=sort_order,
|
||||
)
|
||||
|
||||
pages_collected = 0
|
||||
observed_total_results: int | None = None
|
||||
|
||||
def _on_page(page, meta: dict[str, int | None]) -> None:
|
||||
nonlocal observed_total_results
|
||||
if page.total_results is not None:
|
||||
observed_total_results = int(page.total_results)
|
||||
if progress_callback is None:
|
||||
return
|
||||
payload = {
|
||||
@@ -387,10 +396,15 @@ class MobileDeScraper:
|
||||
}
|
||||
progress_callback("page_collected", payload)
|
||||
|
||||
pages_collected = 0
|
||||
early_stopped = False
|
||||
concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
|
||||
if concurrent_pages > 1 and max_pages and max_pages > 1 and only_new is not True:
|
||||
if (
|
||||
concurrent_pages > 1
|
||||
and max_pages
|
||||
and max_pages > 1
|
||||
and only_new is not True
|
||||
and hasattr(self.client, "fetch_search_pages_concurrent")
|
||||
):
|
||||
page_iterator = self.client.fetch_search_pages_concurrent(
|
||||
start_page=start_page,
|
||||
max_pages=max_pages,
|
||||
@@ -409,6 +423,8 @@ class MobileDeScraper:
|
||||
)
|
||||
for page in page_iterator:
|
||||
pages_collected += 1
|
||||
if page.total_results is not None:
|
||||
observed_total_results = int(page.total_results)
|
||||
pages_payload.append(asdict(page))
|
||||
listing_count += len(page.listings)
|
||||
unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
|
||||
@@ -545,24 +561,58 @@ class MobileDeScraper:
|
||||
upsert = self.persistence.upsert_cars_batch(page_records) if page_records else {
|
||||
"inserted": 0,
|
||||
"updated": 0,
|
||||
"changed": 0,
|
||||
"unchanged": 0,
|
||||
"reappeared": 0,
|
||||
"images_upserted": 0,
|
||||
"inserted_origin_ids": [],
|
||||
"changed_origin_ids": [],
|
||||
"reappeared_origin_ids": [],
|
||||
"failed": 0,
|
||||
"failed_origin_ids": [],
|
||||
}
|
||||
page_inserted = int(upsert.get("inserted", 0))
|
||||
page_updated = int(upsert.get("updated", 0))
|
||||
page_changed = int(upsert.get("changed", 0))
|
||||
page_unchanged = int(upsert.get("unchanged", 0))
|
||||
page_reappeared = int(upsert.get("reappeared", 0))
|
||||
page_images = int(upsert.get("images_upserted", 0))
|
||||
page_failed = int(upsert.get("failed", 0))
|
||||
inserted_origin_ids = [
|
||||
str(origin_id)
|
||||
for origin_id in upsert.get("inserted_origin_ids", [])
|
||||
if origin_id
|
||||
]
|
||||
changed_origin_ids = [
|
||||
str(origin_id)
|
||||
for origin_id in upsert.get("changed_origin_ids", [])
|
||||
if origin_id
|
||||
]
|
||||
reappeared_origin_ids = [
|
||||
str(origin_id)
|
||||
for origin_id in upsert.get("reappeared_origin_ids", [])
|
||||
if origin_id
|
||||
]
|
||||
|
||||
ids_fetched += len(page_records)
|
||||
inserted_total += page_inserted
|
||||
updated_total += page_updated
|
||||
changed_total += page_changed
|
||||
unchanged_total += page_unchanged
|
||||
reappeared_total += page_reappeared
|
||||
images_upserted += page_images
|
||||
cars_failed += page_failed
|
||||
cars_upserted = inserted_total + updated_total
|
||||
accepted_records += len(page_records)
|
||||
|
||||
logger.debug(
|
||||
"mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s updated=%s images=%s",
|
||||
"mobile.de sync_search page upsert: run_id=%s page=%s inserted=%s changed=%s unchanged=%s reappeared=%s updated=%s images=%s",
|
||||
run_id,
|
||||
page.page_number,
|
||||
page_inserted,
|
||||
page_changed,
|
||||
page_unchanged,
|
||||
page_reappeared,
|
||||
page_updated,
|
||||
page_images,
|
||||
)
|
||||
@@ -576,7 +626,13 @@ class MobileDeScraper:
|
||||
"page_number": page.page_number,
|
||||
"inserted": page_inserted,
|
||||
"updated": page_updated,
|
||||
"changed": page_changed,
|
||||
"unchanged": page_unchanged,
|
||||
"reappeared": page_reappeared,
|
||||
"images_upserted": page_images,
|
||||
"inserted_origin_ids": inserted_origin_ids,
|
||||
"changed_origin_ids": changed_origin_ids,
|
||||
"reappeared_origin_ids": reappeared_origin_ids,
|
||||
},
|
||||
)
|
||||
|
||||
@@ -589,6 +645,8 @@ class MobileDeScraper:
|
||||
"strategy_note": MOBILEDE_SEARCH_STRATEGY_NOTE,
|
||||
"search_url": search_url,
|
||||
"pages": pages_payload,
|
||||
"pages_collected": pages_collected,
|
||||
"total_results": observed_total_results,
|
||||
"listing_count": listing_count,
|
||||
"unique_listing_count": len(unique_ids),
|
||||
"unique_listing_ids": sorted(unique_ids),
|
||||
@@ -607,10 +665,10 @@ class MobileDeScraper:
|
||||
|
||||
self.persistence.finish_sync_run(
|
||||
run_id,
|
||||
status="success",
|
||||
status="partial_success" if cars_failed else "success",
|
||||
ids_fetched=ids_fetched,
|
||||
cars_upserted=cars_upserted,
|
||||
cars_failed=0,
|
||||
cars_failed=cars_failed,
|
||||
images_upserted=images_upserted,
|
||||
)
|
||||
logger.debug(
|
||||
@@ -624,7 +682,11 @@ class MobileDeScraper:
|
||||
"upsert": {
|
||||
"inserted": inserted_total,
|
||||
"updated": updated_total,
|
||||
"changed": changed_total,
|
||||
"unchanged": unchanged_total,
|
||||
"reappeared": reappeared_total,
|
||||
"images_upserted": images_upserted,
|
||||
"failed": cars_failed,
|
||||
},
|
||||
"detail_enriched": detail_enriched,
|
||||
"detail_enrich_failed": detail_enrich_failed,
|
||||
@@ -637,7 +699,7 @@ class MobileDeScraper:
|
||||
status="failed",
|
||||
ids_fetched=ids_fetched,
|
||||
cars_upserted=cars_upserted,
|
||||
cars_failed=0,
|
||||
cars_failed=cars_failed,
|
||||
images_upserted=images_upserted,
|
||||
error_summary=str(exc),
|
||||
)
|
||||
@@ -650,3 +712,11 @@ class MobileDeScraper:
|
||||
record = self.mapper.detail_to_car_record(str(listing_id), detail)
|
||||
result = self.persistence.upsert_car(record)
|
||||
return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}
|
||||
|
||||
def sync_gallery(self, listing_id: str, *, lane: str = "mobile_de_cars") -> dict[str, Any]:
|
||||
"""Fetch Detail solely to replace the complete image gallery."""
|
||||
self.persistence.create_tables()
|
||||
detail = self.client.fetch_detail(listing_id)
|
||||
images = self.mapper.detail_to_images(detail)
|
||||
result = self.persistence.replace_car_gallery(f"mobile.de:{listing_id}", images)
|
||||
return {"source": "mobile.de", "lane": lane, "listing_id": str(listing_id), "upsert": result}
|
||||
|
||||
Reference in New Issue
Block a user