qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Normalize image galleries

Browse Source
This commit is contained in:
qananasikq
2026-08-17 17:46:00 +03:00
parent 2d3260d8f9
commit e6f6592914
4 changed files with 140 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobilede_scraper/mobile_de/mapper.py
+47 -23
View File
@@ -114,8 +114,11 @@ _COLOR_MAP = {
}
_IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
_IMAGE_URL_MARKERS = ["img.classistatic.de", "/images/", "/image/", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"]
_MOBILEDE_IMAGE_RULE = "mo-640.jpg"
_MOBILEDE_IMAGE_HOST = "img.classistatic.de"
_MOBILEDE_IMAGE_PATH_PREFIX = "/api/v1/mo-prod/images/"
_MOBILEDE_FULLRES_IMAGE_RULE = "mo-1600"
_MOBILEDE_PREVIEW_IMAGE_RULE = "mo-360"
_MOBILEDE_IMAGE_ID_RE = re.compile(r"^[0-9a-f]{2}/[0-9a-f-]{8,}$", re.IGNORECASE)
_COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"}
_COUNTRY_VALUE_KEYS = {"country", "countrycode"}
_BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
@@ -633,10 +636,20 @@ class MobileDeMapper:
@staticmethod
def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
urls = MobileDeMapper._extract_image_urls(raw)
source_urls = dict.fromkeys(MobileDeMapper._extract_image_urls(raw))
return [
ImageRecord(fullres_image=url, preview_image=url, order_index=index)
for index, url in enumerate(dict.fromkeys(url for url in urls if url))
ImageRecord(
fullres_image=MobileDeMapper._with_mobilede_image_rule(
source_url,
_MOBILEDE_FULLRES_IMAGE_RULE,
),
preview_image=MobileDeMapper._with_mobilede_image_rule(
source_url,
_MOBILEDE_PREVIEW_IMAGE_RULE,
),
order_index=index,
)
for index, source_url in enumerate(source_urls)
]
@staticmethod
@@ -648,7 +661,7 @@ class MobileDeMapper:
if isinstance(value, str):
if parent_hint or MobileDeMapper._looks_like_image_url(value):
normalized = MobileDeMapper._normalize_image_url(value)
if MobileDeMapper._looks_like_image_url(normalized):
if normalized:
urls.append(normalized)
return urls
if isinstance(value, list):
@@ -669,12 +682,7 @@ class MobileDeMapper:
@staticmethod
def _looks_like_image_url(value: str) -> bool:
text = str(value or "").strip().lower()
if not text:
return False
if not (text.startswith("http://") or text.startswith("https://") or text.startswith("//") or text.startswith("/")):
return False
return any(marker in text for marker in _IMAGE_URL_MARKERS)
return bool(MobileDeMapper._normalize_image_url(value))
@staticmethod
def _normalize_image_url(value: str) -> str:
@@ -683,19 +691,35 @@ class MobileDeMapper:
return ""
if url.startswith("//"):
url = f"https:{url}"
elif url.startswith(_MOBILEDE_IMAGE_HOST):
url = f"https://{url}"
elif not (url.startswith("http://") or url.startswith("https://")):
url = f"https://{url.lstrip('/')}"
return MobileDeMapper._normalize_mobilede_image_rule(url)
return ""
parsed = urlsplit(url)
if parsed.netloc.lower() != _MOBILEDE_IMAGE_HOST:
return ""
if not parsed.path.startswith(_MOBILEDE_IMAGE_PATH_PREFIX):
return ""
image_id = parsed.path[len(_MOBILEDE_IMAGE_PATH_PREFIX):].strip("/")
if not _MOBILEDE_IMAGE_ID_RE.fullmatch(image_id):
return ""
query_pairs = [
(key, query_value)
for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
if key.lower() != "rule"
]
return urlunsplit(("https", _MOBILEDE_IMAGE_HOST, parsed.path, urlencode(query_pairs), ""))
@staticmethod
def _normalize_mobilede_image_rule(url: str) -> str:
def _with_mobilede_image_rule(url: str, rule: str) -> str:
parsed = urlsplit(url)
if "img.classistatic.de" not in parsed.netloc.lower():
return url
if "/api/v1/mo-prod/images/" not in parsed.path:
return url
query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
if any(key.lower() == "rule" for key, _value in query_pairs):
return url
query_pairs.append(("rule", _MOBILEDE_IMAGE_RULE))
query_pairs = [
(key, query_value)
for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
if key.lower() != "rule"
]
query_pairs.append(("rule", rule))
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment))
mobilede_scraper/storage/db.py
+24 -12
View File
@@ -252,6 +252,8 @@ class PersistenceService:
images: list[dict[str, object]],
origin_id: str,
) -> int:
if not images:
return 0
nested = session.begin_nested()
try:
session.execute(delete(Image).where(Image.car_id == car_id))
@@ -325,6 +327,14 @@ class PersistenceService:
entry["car_id"] = car_id
insert_images_by_car_id: dict[int, list[dict[str, object]]] = {}
updated_entries = [
entry for entry in entries
if str(entry.get("action") or "updated") == "updated"
]
existing_images_map = self._load_existing_image_urls(
session,
{int(entry["car_id"]) for entry in updated_entries},
)
updated_entries_needing_compare: list[dict[str, object]] = []
for entry in entries:
@@ -332,11 +342,13 @@ class PersistenceService:
action = str(entry.get("action") or "updated")
images = entry["images"]
skip_image_sync = bool(entry.get("skip_image_sync"))
if action == "updated" and (MOBILEDE_SKIP_IMAGES_FOR_UPDATED or skip_image_sync):
continue
if action == "inserted":
insert_images_by_car_id[car_id] = images
continue
if not images or skip_image_sync:
continue
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and existing_images_map.get(car_id):
continue
updated_entries_needing_compare.append(entry)
for car_id, images in insert_images_by_car_id.items():
@@ -344,8 +356,6 @@ class PersistenceService:
images_total += len(images)
if updated_entries_needing_compare:
car_ids = {int(entry["car_id"]) for entry in updated_entries_needing_compare}
existing_images_map = self._load_existing_image_urls(session, car_ids)
images_by_car_id: dict[int, list[dict[str, object]]] = {}
replace_ids: list[int] = []
@@ -474,7 +484,7 @@ class PersistenceService:
existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
new_cars: list[tuple[Car, list[dict]]] = []
update_image_candidates: list[tuple[Car, list[dict]]] = []
updated_cars: list[tuple[Car, list[dict], bool]] = []
for record in records:
payload = self._car_payload(record)
@@ -489,9 +499,7 @@ class PersistenceService:
else:
self._apply_update_payload(car, payload)
updated += 1
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED or self._skip_image_sync(record):
continue
update_image_candidates.append((car, images))
updated_cars.append((car, images, self._skip_image_sync(record)))
# Один flush.
session.flush()
@@ -502,12 +510,16 @@ class PersistenceService:
images_total += len(images)
update_cars_needing_images: list[tuple[Car, list[dict]]] = []
if update_image_candidates:
update_ids = [int(car.id) for car, _ in update_image_candidates]
if updated_cars:
update_ids = [int(item[0].id) for item in updated_cars]
existing_images_map = self._load_existing_image_urls(session, set(update_ids))
for car, images in update_image_candidates:
new_image_urls = {img.get("fullres_image", "") for img in images}
for car, images, skip_image_sync in updated_cars:
old_image_urls = existing_images_map.get(int(car.id), set())
if not images or skip_image_sync:
continue
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and old_image_urls:
continue
new_image_urls = {img.get("fullres_image", "") for img in images}
if new_image_urls != old_image_urls:
update_cars_needing_images.append((car, images))
else: