Normalize image galleries

mobilede_scraper/mobile_de/mapper.py

@@ -114,8 +114,11 @@ _COLOR_MAP = {
 }
 
 _IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
-_IMAGE_URL_MARKERS = ["img.classistatic.de", "/images/", "/image/", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"]
-_MOBILEDE_IMAGE_RULE = "mo-640.jpg"
+_MOBILEDE_IMAGE_HOST = "img.classistatic.de"
+_MOBILEDE_IMAGE_PATH_PREFIX = "/api/v1/mo-prod/images/"
+_MOBILEDE_FULLRES_IMAGE_RULE = "mo-1600"
+_MOBILEDE_PREVIEW_IMAGE_RULE = "mo-360"
+_MOBILEDE_IMAGE_ID_RE = re.compile(r"^[0-9a-f]{2}/[0-9a-f-]{8,}$", re.IGNORECASE)
 _COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"}
 _COUNTRY_VALUE_KEYS = {"country", "countrycode"}
 _BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
@@ -633,10 +636,20 @@ class MobileDeMapper:
 
     @staticmethod
     def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
-        urls = MobileDeMapper._extract_image_urls(raw)
+        source_urls = dict.fromkeys(MobileDeMapper._extract_image_urls(raw))
         return [
-            ImageRecord(fullres_image=url, preview_image=url, order_index=index)
-            for index, url in enumerate(dict.fromkeys(url for url in urls if url))
+            ImageRecord(
+                fullres_image=MobileDeMapper._with_mobilede_image_rule(
+                    source_url,
+                    _MOBILEDE_FULLRES_IMAGE_RULE,
+                ),
+                preview_image=MobileDeMapper._with_mobilede_image_rule(
+                    source_url,
+                    _MOBILEDE_PREVIEW_IMAGE_RULE,
+                ),
+                order_index=index,
+            )
+            for index, source_url in enumerate(source_urls)
         ]
 
     @staticmethod
@@ -648,7 +661,7 @@ class MobileDeMapper:
         if isinstance(value, str):
             if parent_hint or MobileDeMapper._looks_like_image_url(value):
                 normalized = MobileDeMapper._normalize_image_url(value)
-                if MobileDeMapper._looks_like_image_url(normalized):
+                if normalized:
                     urls.append(normalized)
             return urls
         if isinstance(value, list):
@@ -669,12 +682,7 @@ class MobileDeMapper:
 
     @staticmethod
     def _looks_like_image_url(value: str) -> bool:
-        text = str(value or "").strip().lower()
-        if not text:
-            return False
-        if not (text.startswith("http://") or text.startswith("https://") or text.startswith("//") or text.startswith("/")):
-            return False
-        return any(marker in text for marker in _IMAGE_URL_MARKERS)
+        return bool(MobileDeMapper._normalize_image_url(value))
 
     @staticmethod
     def _normalize_image_url(value: str) -> str:
@@ -683,19 +691,35 @@ class MobileDeMapper:
             return ""
         if url.startswith("//"):
             url = f"https:{url}"
+        elif url.startswith(_MOBILEDE_IMAGE_HOST):
+            url = f"https://{url}"
         elif not (url.startswith("http://") or url.startswith("https://")):
-            url = f"https://{url.lstrip('/')}"
-        return MobileDeMapper._normalize_mobilede_image_rule(url)
+            return ""
+
+        parsed = urlsplit(url)
+        if parsed.netloc.lower() != _MOBILEDE_IMAGE_HOST:
+            return ""
+        if not parsed.path.startswith(_MOBILEDE_IMAGE_PATH_PREFIX):
+            return ""
+
+        image_id = parsed.path[len(_MOBILEDE_IMAGE_PATH_PREFIX):].strip("/")
+        if not _MOBILEDE_IMAGE_ID_RE.fullmatch(image_id):
+            return ""
+
+        query_pairs = [
+            (key, query_value)
+            for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
+            if key.lower() != "rule"
+        ]
+        return urlunsplit(("https", _MOBILEDE_IMAGE_HOST, parsed.path, urlencode(query_pairs), ""))
 
     @staticmethod
-    def _normalize_mobilede_image_rule(url: str) -> str:
+    def _with_mobilede_image_rule(url: str, rule: str) -> str:
         parsed = urlsplit(url)
-        if "img.classistatic.de" not in parsed.netloc.lower():
-            return url
-        if "/api/v1/mo-prod/images/" not in parsed.path:
-            return url
-        query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
-        if any(key.lower() == "rule" for key, _value in query_pairs):
-            return url
-        query_pairs.append(("rule", _MOBILEDE_IMAGE_RULE))
+        query_pairs = [
+            (key, query_value)
+            for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
+            if key.lower() != "rule"
+        ]
+        query_pairs.append(("rule", rule))
         return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment))