qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Normalize image galleries

Browse Source
This commit is contained in:
qananasikq
2026-08-17 17:46:00 +03:00
parent 2d3260d8f9
commit e6f6592914
4 changed files with 140 additions and 53 deletions
Download Patch File Download Diff File Expand all files Collapse all files
mobilede_scraper/mobile_de/mapper.py
+47 -23
View File
@@ -114,8 +114,11 @@ _COLOR_MAP = {
} }
_IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"] _IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
_IMAGE_URL_MARKERS = ["img.classistatic.de", "/images/", "/image/", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"] _MOBILEDE_IMAGE_HOST = "img.classistatic.de"
_MOBILEDE_IMAGE_RULE = "mo-640.jpg" _MOBILEDE_IMAGE_PATH_PREFIX = "/api/v1/mo-prod/images/"
_MOBILEDE_FULLRES_IMAGE_RULE = "mo-1600"
_MOBILEDE_PREVIEW_IMAGE_RULE = "mo-360"
_MOBILEDE_IMAGE_ID_RE = re.compile(r"^[0-9a-f]{2}/[0-9a-f-]{8,}$", re.IGNORECASE)
_COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"} _COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"}
_COUNTRY_VALUE_KEYS = {"country", "countrycode"} _COUNTRY_VALUE_KEYS = {"country", "countrycode"}
_BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"} _BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
@@ -633,10 +636,20 @@ class MobileDeMapper:
@staticmethod @staticmethod
def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]: def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
urls = MobileDeMapper._extract_image_urls(raw) source_urls = dict.fromkeys(MobileDeMapper._extract_image_urls(raw))
return [ return [
ImageRecord(fullres_image=url, preview_image=url, order_index=index) ImageRecord(
for index, url in enumerate(dict.fromkeys(url for url in urls if url)) fullres_image=MobileDeMapper._with_mobilede_image_rule(
source_url,
_MOBILEDE_FULLRES_IMAGE_RULE,
),
preview_image=MobileDeMapper._with_mobilede_image_rule(
source_url,
_MOBILEDE_PREVIEW_IMAGE_RULE,
),
order_index=index,
)
for index, source_url in enumerate(source_urls)
] ]
@staticmethod @staticmethod
@@ -648,7 +661,7 @@ class MobileDeMapper:
if isinstance(value, str): if isinstance(value, str):
if parent_hint or MobileDeMapper._looks_like_image_url(value): if parent_hint or MobileDeMapper._looks_like_image_url(value):
normalized = MobileDeMapper._normalize_image_url(value) normalized = MobileDeMapper._normalize_image_url(value)
if MobileDeMapper._looks_like_image_url(normalized): if normalized:
urls.append(normalized) urls.append(normalized)
return urls return urls
if isinstance(value, list): if isinstance(value, list):
@@ -669,12 +682,7 @@ class MobileDeMapper:
@staticmethod @staticmethod
def _looks_like_image_url(value: str) -> bool: def _looks_like_image_url(value: str) -> bool:
text = str(value or "").strip().lower() return bool(MobileDeMapper._normalize_image_url(value))
if not text:
return False
if not (text.startswith("http://") or text.startswith("https://") or text.startswith("//") or text.startswith("/")):
return False
return any(marker in text for marker in _IMAGE_URL_MARKERS)
@staticmethod @staticmethod
def _normalize_image_url(value: str) -> str: def _normalize_image_url(value: str) -> str:
@@ -683,19 +691,35 @@ class MobileDeMapper:
return "" return ""
if url.startswith("//"): if url.startswith("//"):
url = f"https:{url}" url = f"https:{url}"
elif url.startswith(_MOBILEDE_IMAGE_HOST):
url = f"https://{url}"
elif not (url.startswith("http://") or url.startswith("https://")): elif not (url.startswith("http://") or url.startswith("https://")):
url = f"https://{url.lstrip('/')}" return ""
return MobileDeMapper._normalize_mobilede_image_rule(url)
parsed = urlsplit(url)
if parsed.netloc.lower() != _MOBILEDE_IMAGE_HOST:
return ""
if not parsed.path.startswith(_MOBILEDE_IMAGE_PATH_PREFIX):
return ""
image_id = parsed.path[len(_MOBILEDE_IMAGE_PATH_PREFIX):].strip("/")
if not _MOBILEDE_IMAGE_ID_RE.fullmatch(image_id):
return ""
query_pairs = [
(key, query_value)
for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
if key.lower() != "rule"
]
return urlunsplit(("https", _MOBILEDE_IMAGE_HOST, parsed.path, urlencode(query_pairs), ""))
@staticmethod @staticmethod
def _normalize_mobilede_image_rule(url: str) -> str: def _with_mobilede_image_rule(url: str, rule: str) -> str:
parsed = urlsplit(url) parsed = urlsplit(url)
if "img.classistatic.de" not in parsed.netloc.lower(): query_pairs = [
return url (key, query_value)
if "/api/v1/mo-prod/images/" not in parsed.path: for key, query_value in parse_qsl(parsed.query, keep_blank_values=True)
return url if key.lower() != "rule"
query_pairs = parse_qsl(parsed.query, keep_blank_values=True) ]
if any(key.lower() == "rule" for key, _value in query_pairs): query_pairs.append(("rule", rule))
return url
query_pairs.append(("rule", _MOBILEDE_IMAGE_RULE))
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment)) return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment))
mobilede_scraper/storage/db.py
+24 -12
View File
@@ -252,6 +252,8 @@ class PersistenceService:
images: list[dict[str, object]], images: list[dict[str, object]],
origin_id: str, origin_id: str,
) -> int: ) -> int:
if not images:
return 0
nested = session.begin_nested() nested = session.begin_nested()
try: try:
session.execute(delete(Image).where(Image.car_id == car_id)) session.execute(delete(Image).where(Image.car_id == car_id))
@@ -325,6 +327,14 @@ class PersistenceService:
entry["car_id"] = car_id entry["car_id"] = car_id
insert_images_by_car_id: dict[int, list[dict[str, object]]] = {} insert_images_by_car_id: dict[int, list[dict[str, object]]] = {}
updated_entries = [
entry for entry in entries
if str(entry.get("action") or "updated") == "updated"
]
existing_images_map = self._load_existing_image_urls(
session,
{int(entry["car_id"]) for entry in updated_entries},
)
updated_entries_needing_compare: list[dict[str, object]] = [] updated_entries_needing_compare: list[dict[str, object]] = []
for entry in entries: for entry in entries:
@@ -332,11 +342,13 @@ class PersistenceService:
action = str(entry.get("action") or "updated") action = str(entry.get("action") or "updated")
images = entry["images"] images = entry["images"]
skip_image_sync = bool(entry.get("skip_image_sync")) skip_image_sync = bool(entry.get("skip_image_sync"))
if action == "updated" and (MOBILEDE_SKIP_IMAGES_FOR_UPDATED or skip_image_sync):
continue
if action == "inserted": if action == "inserted":
insert_images_by_car_id[car_id] = images insert_images_by_car_id[car_id] = images
continue continue
if not images or skip_image_sync:
continue
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and existing_images_map.get(car_id):
continue
updated_entries_needing_compare.append(entry) updated_entries_needing_compare.append(entry)
for car_id, images in insert_images_by_car_id.items(): for car_id, images in insert_images_by_car_id.items():
@@ -344,8 +356,6 @@ class PersistenceService:
images_total += len(images) images_total += len(images)
if updated_entries_needing_compare: if updated_entries_needing_compare:
car_ids = {int(entry["car_id"]) for entry in updated_entries_needing_compare}
existing_images_map = self._load_existing_image_urls(session, car_ids)
images_by_car_id: dict[int, list[dict[str, object]]] = {} images_by_car_id: dict[int, list[dict[str, object]]] = {}
replace_ids: list[int] = [] replace_ids: list[int] = []
@@ -474,7 +484,7 @@ class PersistenceService:
existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls) existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
new_cars: list[tuple[Car, list[dict]]] = [] new_cars: list[tuple[Car, list[dict]]] = []
update_image_candidates: list[tuple[Car, list[dict]]] = [] updated_cars: list[tuple[Car, list[dict], bool]] = []
for record in records: for record in records:
payload = self._car_payload(record) payload = self._car_payload(record)
@@ -489,9 +499,7 @@ class PersistenceService:
else: else:
self._apply_update_payload(car, payload) self._apply_update_payload(car, payload)
updated += 1 updated += 1
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED or self._skip_image_sync(record): updated_cars.append((car, images, self._skip_image_sync(record)))
continue
update_image_candidates.append((car, images))
# Один flush. # Один flush.
session.flush() session.flush()
@@ -502,12 +510,16 @@ class PersistenceService:
images_total += len(images) images_total += len(images)
update_cars_needing_images: list[tuple[Car, list[dict]]] = [] update_cars_needing_images: list[tuple[Car, list[dict]]] = []
if update_image_candidates: if updated_cars:
update_ids = [int(car.id) for car, _ in update_image_candidates] update_ids = [int(item[0].id) for item in updated_cars]
existing_images_map = self._load_existing_image_urls(session, set(update_ids)) existing_images_map = self._load_existing_image_urls(session, set(update_ids))
for car, images in update_image_candidates: for car, images, skip_image_sync in updated_cars:
new_image_urls = {img.get("fullres_image", "") for img in images}
old_image_urls = existing_images_map.get(int(car.id), set()) old_image_urls = existing_images_map.get(int(car.id), set())
if not images or skip_image_sync:
continue
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and old_image_urls:
continue
new_image_urls = {img.get("fullres_image", "") for img in images}
if new_image_urls != old_image_urls: if new_image_urls != old_image_urls:
update_cars_needing_images.append((car, images)) update_cars_needing_images.append((car, images))
else: else:
tests/test_db.py
+30
View File
@@ -101,6 +101,36 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(len(images), 1) self.assertEqual(len(images), 1)
self.assertIn("imageKeys=2", images[0].fullres_image) self.assertIn("imageKeys=2", images[0].fullres_image)
def test_empty_update_preserves_existing_images(self) -> None:
first = self._record("mobile.de:keep-images")
self.persistence.upsert_car(first)
update = self._record("mobile.de:keep-images", price=1500)
update.images = []
result = self.persistence.upsert_car(update)
with self.persistence.session_scope() as session:
images = session.execute(select(Image)).scalars().all()
self.assertEqual(result["action"], "updated")
self.assertEqual(result["images_upserted"], 0)
self.assertEqual(len(images), 1)
def test_batch_update_adds_images_when_existing_gallery_is_empty(self) -> None:
first = self._record("mobile.de:add-images")
first.images = []
self.persistence.upsert_car(first)
update = self._record("mobile.de:add-images", price=1500)
result = self.persistence.upsert_cars_batch([update])
with self.persistence.session_scope() as session:
images = session.execute(select(Image)).scalars().all()
self.assertEqual(result["updated"], 1)
self.assertEqual(result["images_upserted"], 1)
self.assertEqual(len(images), 1)
def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None: def test_start_sync_run_marks_stale_running_runs_as_failed(self) -> None:
first_run_id = self.persistence.start_sync_run("lane-a") first_run_id = self.persistence.start_sync_run("lane-a")
second_run_id = self.persistence.start_sync_run("lane-b") second_run_id = self.persistence.start_sync_run("lane-b")
tests/test_mappers.py
+39 -18
View File
@@ -11,10 +11,12 @@ class TestMobileDeMapper(unittest.TestCase):
self.mapper = MobileDeMapper() self.mapper = MobileDeMapper()
def test_deduplicates_images(self) -> None: def test_deduplicates_images(self) -> None:
first = "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
second = "https://img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"
urls = [ urls = [
"https://img.classistatic.de/api/v1/mo-prod/images/1", f"{first}?rule=mo-80w",
"https://img.classistatic.de/api/v1/mo-prod/images/1", f"{first}?rule=mo-1600",
"//img.classistatic.de/api/v1/mo-prod/images/2", second.replace("https:", ""),
] ]
record = self.mapper.listing_to_car_record( record = self.mapper.listing_to_car_record(
@@ -27,8 +29,10 @@ class TestMobileDeMapper(unittest.TestCase):
) )
self.assertEqual(len(record.images), 2) self.assertEqual(len(record.images), 2)
self.assertEqual(record.images[0].fullres_image, f"{urls[0]}?rule=mo-640.jpg") self.assertEqual(record.images[0].fullres_image, f"{first}?rule=mo-1600")
self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2?rule=mo-640.jpg") self.assertEqual(record.images[0].preview_image, f"{first}?rule=mo-360")
self.assertEqual(record.images[1].fullres_image, f"{second}?rule=mo-1600")
self.assertEqual(record.images[1].preview_image, f"{second}?rule=mo-360")
def test_extracts_nested_gallery_images_without_detail_fetch(self) -> None: def test_extracts_nested_gallery_images_without_detail_fetch(self) -> None:
record = self.mapper.listing_to_car_record( record = self.mapper.listing_to_car_record(
@@ -37,11 +41,11 @@ class TestMobileDeMapper(unittest.TestCase):
url="https://suchen.mobile.de/fahrzeuge/details.html?id=124", url="https://suchen.mobile.de/fahrzeuge/details.html?id=124",
title="Honda Civic", title="Honda Civic",
raw={ raw={
"image": "https://img.classistatic.de/api/v1/mo-prod/images/main", "image": "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111",
"images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/ref-1"}], "images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222"}],
"mediaGallery": [ "mediaGallery": [
{"uri": "//img.classistatic.de/api/v1/mo-prod/images/gallery-1"}, {"uri": "//img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333"},
{"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/gallery-2"}}, {"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/d4/d4444444-4444-4444-8444-444444444444"}},
], ],
"trackingUrl": "https://example.test/not-an-image", "trackingUrl": "https://example.test/not-an-image",
}, },
@@ -51,27 +55,44 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertEqual( self.assertEqual(
[image.fullres_image for image in record.images], [image.fullres_image for image in record.images],
[ [
"https://img.classistatic.de/api/v1/mo-prod/images/main?rule=mo-640.jpg", "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111?rule=mo-1600",
"https://img.classistatic.de/api/v1/mo-prod/images/ref-1?rule=mo-640.jpg", "https://img.classistatic.de/api/v1/mo-prod/images/b2/b2222222-2222-4222-8222-222222222222?rule=mo-1600",
"https://img.classistatic.de/api/v1/mo-prod/images/gallery-1?rule=mo-640.jpg", "https://img.classistatic.de/api/v1/mo-prod/images/c3/c3333333-3333-4333-8333-333333333333?rule=mo-1600",
"https://img.classistatic.de/api/v1/mo-prod/images/gallery-2?rule=mo-640.jpg", "https://img.classistatic.de/api/v1/mo-prod/images/d4/d4444444-4444-4444-8444-444444444444?rule=mo-1600",
], ],
) )
def test_keeps_existing_mobilede_image_rule(self) -> None: def test_replaces_existing_mobilede_image_rule(self) -> None:
source = "https://img.classistatic.de/api/v1/mo-prod/images/a1/a1111111-1111-4111-8111-111111111111"
record = self.mapper.listing_to_car_record( record = self.mapper.listing_to_car_record(
MobileDeListing( MobileDeListing(
id="125", id="125",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=125", url="https://suchen.mobile.de/fahrzeuge/details.html?id=125",
title="BMW 320", title="BMW 320",
raw={"images": [{"uri": "img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"}]}, raw={"images": [{"uri": f"{source}?rule=mo-80w"}]},
) )
) )
self.assertEqual( self.assertEqual(record.images[0].fullres_image, f"{source}?rule=mo-1600")
[image.fullres_image for image in record.images], self.assertEqual(record.images[0].preview_image, f"{source}?rule=mo-360")
["https://img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"],
def test_rejects_non_vehicle_mobilede_images(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="126",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=126",
title="BMW 320",
raw={
"images": [
"https://img.classistatic.de/api/v1/mo-prod/images/co2class-G?rule=mo-640",
"https://img.classistatic.de/images/logo.svg",
"https://example.test/image.jpg",
]
},
) )
)
self.assertEqual(record.images, [])
def test_origin_id_uses_canonical_prefix(self) -> None: def test_origin_id_uses_canonical_prefix(self) -> None:
record = self.mapper.listing_to_car_record( record = self.mapper.listing_to_car_record(