mobile.de/mobilede_scraper/parsing/fast_mapper.py

from __future__ import annotations

import re
from datetime import datetime, timezone
from typing import Any
from urllib.parse import urlparse

from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text
from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES
from ..storage.schemas import CarRecord, ImageRecord

ORIGIN_PREFIX = "mobilede:"
ORIGIN_URL_BASE = "https://www.MOBILEDE.com/VehicleDetail"
COLOR_ALIASES = {
    "grau": "gray",
    "grau metallic": "gray",
    "gray": "gray",
    "grey": "gray",
    "silber": "silver",
    "silver": "silver",
    "schwarz": "black",
    "black": "black",
    "weiss": "white",
    "weiß": "white",
    "white": "white",
    "blau": "blue",
    "blue": "blue",
    "rot": "red",
    "red": "red",
    "gruen": "green",
    "grün": "green",
    "green": "green",
    "braun": "brown",
    "brown": "brown",
    "beige": "beige",
    "orange": "orange",
    "gelb": "yellow",
    "yellow": "yellow",
    "violett": "purple",
    "lila": "purple",
    "purple": "purple",
    "gold": "gold",
}
DAMAGE_NEUTRAL_VALUES = {
    "NORMAL WEAR & TEAR",
    "NORMAL WEAR",
    "NORMALWEAR&TEAR",
    "NONE",
    "NO DAMAGE",
    "NO VISIBLE DAMAGE",
    "MINOR DENT/SCRATCHES",
}
INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"}


class FastCarMapper:
    """Maps MOBILEDE ProductDetailsVM payloads directly to project CarRecord."""

    def map_payload_to_record(
        self,
        *,
        detail_payload: dict[str, Any],
        vehicle_url: str | None = None,
        listing_vehicle: FastListingVehicle | None = None,
    ) -> CarRecord:
        inventory_view = detail_payload.get("inventoryView")
        if not isinstance(inventory_view, dict):
            raise RuntimeError("detail payload missing inventoryView")
        attributes = inventory_view.get("attributes")
        if not isinstance(attributes, dict):
            raise RuntimeError("detail payload missing inventoryView.attributes")

        inventory_id = parse_text(attributes.get("Id"))
        if not inventory_id and listing_vehicle is not None:
            inventory_id = listing_vehicle.inventory_id
        if not inventory_id and vehicle_url:
            inventory_id = self._inventory_id_from_url(vehicle_url)
        if not inventory_id:
            raise RuntimeError("missing inventory id")

        brand = self._limit_text(self._attr_text(attributes, "Make") or "UNKNOWN", 50)
        model = self._build_model_name(
            self._attr_text(attributes, "Model"),
            self._attr_text(attributes, "Series", "Trim", "Variant"),
        ) or "UNKNOWN"
        model = self._limit_text(model, 50)
        year = self._parse_year(
            self._attr_text(
                attributes,
                "Year",
                "FirstRegistration",
                "InitialRegistration",
                "Erstzulassung",
                "ModelYear",
            )
        )
        descriptor_text = " ".join(
            filter(
                None,
                [
                    brand,
                    model,
                    self._attr_text(attributes, "BodyStyleName", "VehicleClass", "Category", "CategoryDescription"),
                    self._attr_text(attributes, "DriveLineTypeDesc", "DriveType", "DriveTrain", "DriveTrainType", "DriveDescription"),
                    self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType"),
                ],
            )
        )

        auction_info = detail_payload.get("auctionInformation")
        auction_info = auction_info if isinstance(auction_info, dict) else {}
        bidding_info = auction_info.get("biddingInformation")
        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
        prebid_info = auction_info.get("prebidInformation")
        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}

        high_bid = self._first_positive_int(
            prebid_info.get("decimalHighBidAmount"),
            prebid_info.get("highBidAmount"),
            bidding_info.get("highBidAmount"),
        )
        buy_now = self._first_positive_int(
            bidding_info.get("buyNowAmount"),
            prebid_info.get("buyNowPrice"),
            bidding_info.get("buyNowPrice"),
        )
        price = high_bid if high_bid is not None else buy_now

        image_dimensions = inventory_view.get("imageDimensions")
        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
        keys_container = image_dimensions.get("keys")
        keys_container = keys_container if isinstance(keys_container, dict) else {}
        image_keys = keys_container.get("$values")
        image_keys = image_keys if isinstance(image_keys, list) else []
        images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)]

        primary_damage = self._attr_text(attributes, "PrimaryDamageDesc")
        secondary_damage = self._attr_text(attributes, "SecondaryDamageDesc")
        origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}"
        normalized_origin_id = self._normalize_origin_inventory_id(inventory_id)
        origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}"
        drive_source = self._attr_text(
            attributes,
            "DriveLineTypeDesc",
            "DriveType",
            "DriveTrain",
            "DriveTrainType",
            "DriveDescription",
            "Antrieb",
        ) or descriptor_text
        gearbox_source = self._attr_text(attributes, "Transmission", "Gearbox", "TransmissionType") or descriptor_text
        body_type_source = self._attr_text(
            attributes,
            "BodyStyleName",
            "VehicleClass",
            "Category",
            "CategoryDescription",
            "BodyType",
        ) or descriptor_text
        engine_source = self._attr_text(
            attributes,
            "EngineSize",
            "EngineInformation",
            "EngineDisplacement",
            "Displacement",
            "CubicCapacity",
        )
        color_source = self._attr_text(attributes, "ExteriorColor", "ColorDesc", "Color")

        return CarRecord(
            parser_id=self._generate_parser_id(origin_id),
            brand=brand,
            model=model,
            year=year,
            price=price,
            currency=self._map_currency(self._attr_text(attributes, "Currency") or (listing_vehicle.currency if listing_vehicle else None)),
            mileage=self._parse_non_negative_int(self._attr_text(attributes, "ODOValue", "Mileage", "Kilometerstand", "Odometer")) or 0,
            country=self._map_country(inventory_id, tenant=listing_vehicle.tenant if listing_vehicle else None),
            is_sold=self._is_sold(listing_vehicle),
            color=self._normalize_color(color_source),
            drive=self._map_drive(drive_source),
            gearbox=self._map_gearbox(gearbox_source),
            steering_wheel="LEFT",
            body_type=self._map_body_type(body_type_source),
            engine_volume=self._parse_engine_volume(engine_source),
            selling_type="AUCTION",
            one_owner=False,
            new_car=False,
            is_hidden=not bool(images),
            origin="MOBILEDE",
            origin_url=origin_url,
            origin_id=origin_id,
            is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage),
            evaluation=self._attr_text(attributes, "VehicleGrade", "PriceRating"),
            non_smoking=True,
            rental=False,
            repair_history=False,
            slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))),
            last_seen_at=datetime.now(timezone.utc),
            images=images,
        )

    def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]:
        inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {}
        inventory_view = inventory_view if isinstance(inventory_view, dict) else {}
        attr = inventory_view.get("attributes")
        attr = attr if isinstance(attr, dict) else {}
        auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {}
        auction_info = auction_info if isinstance(auction_info, dict) else {}
        bidding_info = auction_info.get("biddingInformation")
        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
        prebid_info = auction_info.get("prebidInformation")
        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
        image_dimensions = inventory_view.get("imageDimensions")
        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
        keys_container = image_dimensions.get("keys")
        keys_container = keys_container if isinstance(keys_container, dict) else {}
        image_keys = keys_container.get("$values")
        image_keys = image_keys if isinstance(image_keys, list) else []
        image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)]
        engine_text = self._attr_text(attr, "EngineInformation", "EngineSize", "EngineDisplacement", "Displacement")
        return {
            "source_url": vehicle_url,
            "lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"),
            "year": attr.get("Year") or attr.get("FirstRegistration") or attr.get("Erstzulassung"),
            "make": attr.get("Make"),
            "model": attr.get("Model"),
            "trim": attr.get("Series"),
            "body_type": attr.get("BodyStyleName") or attr.get("VehicleClass") or attr.get("Category"),
            "drive": attr.get("DriveLineTypeDesc") or attr.get("DriveType") or attr.get("DriveTrain"),
            "engine": engine_text,
            "fuel_type": attr.get("FuelTypeCode"),
            "gearbox": attr.get("Transmission") or attr.get("Gearbox") or attr.get("TransmissionType"),
            "color": attr.get("ExteriorColor") or attr.get("ColorDesc") or attr.get("Color"),
            "primary_damage": attr.get("PrimaryDamageDesc"),
            "secondary_damage": attr.get("SecondaryDamageDesc"),
            "odometer": attr.get("ODOValue") or attr.get("Mileage") or attr.get("Kilometerstand"),
            "location": attr.get("BranchName"),
            "auction_date": attr.get("AuctionDateTime"),
            "title": attr.get("Title"),
            "current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"),
            "buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"),
            "actual_cash_value": attr.get("ProviderACV"),
            "estimated_repair_cost": attr.get("EstRepairCost"),
            "image_urls": image_urls,
        }

    @staticmethod
    def _inventory_id_from_url(vehicle_url: str) -> str | None:
        tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
        return tail or None

    @staticmethod
    def _normalize_origin_inventory_id(inventory_id: str) -> str:
        return inventory_id.strip().split("~", 1)[0]

    @staticmethod
    def _build_model_name(model: str | None, series: str | None) -> str:
        unique_parts: list[str] = []
        seen: set[str] = set()
        for value in (model, series):
            if not value:
                continue
            normalized = " ".join(value.split())
            key = normalized.casefold()
            if key in seen:
                continue
            seen.add(key)
            unique_parts.append(normalized)
        return " ".join(unique_parts).strip()

    @staticmethod
    def _parse_year(value: Any) -> int | None:
        if isinstance(value, str):
            match = re.search(r"(19|20)\d{2}", value)
            if match is not None:
                parsed = int(match.group(0))
                if 1900 <= parsed <= 2100:
                    return parsed
        parsed = parse_int(value)
        if parsed is None or parsed < 1900 or parsed > 2100:
            return None
        return parsed

    @staticmethod
    def _parse_non_negative_int(value: Any) -> int | None:
        if isinstance(value, str):
            text = value.strip()
            if not text:
                return None
            grouped_match = re.search(r"\d{1,3}(?:[.,\s]\d{3})+(?!\d)", text)
            if grouped_match is not None:
                digits_only = re.sub(r"\D", "", grouped_match.group(0))
                return int(digits_only) if digits_only else None
            if re.fullmatch(r"\d{1,3}(?:[.,\s]\d{3})+", text):
                digits_only = re.sub(r"\D", "", text)
                return int(digits_only) if digits_only else None
        parsed = parse_int(value)
        if parsed is None or parsed < 0:
            return None
        return parsed

    @staticmethod
    def _attr_text(attributes: dict[str, Any], *keys: str) -> str | None:
        for key in keys:
            value = parse_text(attributes.get(key))
            if value:
                return value
        return None

    @classmethod
    def _first_positive_int(cls, *values: Any) -> int | None:
        for value in values:
            parsed = cls._parse_non_negative_int(value)
            if parsed is not None and parsed > 0:
                return parsed
        return None

    @staticmethod
    def _normalize_color(value: str | None) -> str:
        if not value:
            return "other"
        normalized = FastCarMapper._simplify_text(value)
        if "/" in normalized:
            normalized = normalized.split("/", 1)[0].strip()
        for alias, mapped in COLOR_ALIASES.items():
            if alias in normalized:
                return mapped
        return normalized or "other"

    @staticmethod
    def _map_currency(value: str | None) -> str:
        normalized = (value or "USD").strip().upper()
        return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD"

    @staticmethod
    def _map_country(inventory_id: str, tenant: str | None = None) -> str:
        upper_id = inventory_id.strip().upper()
        if upper_id.endswith("~CA"):
            return "CA"
        if upper_id.endswith("~US"):
            return "US"
        tenant_normalized = (tenant or "").strip().upper()
        if tenant_normalized in {"US", "CA", "JP", "KR"}:
            return tenant_normalized
        return "NA"

    @staticmethod
    def _map_drive(value: str | None) -> str | None:
        if not value:
            return None
        normalized = FastCarMapper._simplify_text(value)
        candidates: tuple[str, ...] | None = None
        if any(marker in normalized for marker in ("front wheel", "front-wheel", "frontantrieb", "fwd", "traction avant")):
            candidates = ("FWD",)
        elif any(
            marker in normalized
            for marker in (
                "all wheel",
                "all-wheel",
                "4x4",
                "awd",
                "four wheel",
                "allrad",
                "4matic",
                "4motion",
                "quattro",
                "xdrive",
            )
        ):
            candidates = ("4WD", "FOUR_WD")
        elif any(marker in normalized for marker in ("rear wheel", "rear-wheel", "heckantrieb", "rwd", "propulsion")):
            candidates = ("RWD",)
        elif "2wd" in normalized or "two wheel" in normalized:
            candidates = ("2WD", "TWO_WD")
        elif "unknown" in normalized or normalized in {"na", "n/a"}:
            candidates = ("NA",)
        return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None

    @staticmethod
    def _map_gearbox(value: str | None) -> str | None:
        if not value:
            return None
        normalized = FastCarMapper._simplify_text(value)
        candidates: tuple[str, ...] | None = None
        if "cvt" in normalized:
            candidates = ("CVT",)
        elif any(marker in normalized for marker in ("manual", "mt", "schaltgetriebe", "schaltung", "stick shift")):
            candidates = ("MT",)
        elif "electric" in normalized or normalized == "ev":
            candidates = ("EV",)
        elif any(marker in normalized for marker in ("auto", "at", "automatik", "automatic", "dsg", "doppelkupplung", "semi automatic", "halbautomatik")):
            candidates = ("AT",)
        elif "unknown" in normalized or normalized in {"na", "n/a"}:
            candidates = ("NA",)
        return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None

    @staticmethod
    def _map_body_type(value: str | None) -> str:
        if not value:
            return "OTHER"
        normalized = FastCarMapper._simplify_text(value)
        candidates: tuple[str, ...] | None = None
        if any(marker in normalized for marker in ("sedan", "limousine", "saloon")):
            candidates = ("SEDAN",)
        elif any(marker in normalized for marker in ("sport utility", "suv", "crossover", "gelandewagen", "gelaendewagen", "off-road")):
            candidates = ("SUV",)
        elif any(marker in normalized for marker in ("hatch", "kleinwagen", "compact", "city car")):
            candidates = ("HATCHBACK",)
        elif any(marker in normalized for marker in ("wagon", "kombi", "estate", "touring", "variant", "avant", "shooting brake")):
            candidates = ("STATION_WAGON", "Station Wagon")
        elif "coupe" in normalized:
            candidates = ("COUPE",)
        elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized):
            candidates = ("PICKUP", "Pickup")
        elif any(marker in normalized for marker in ("convertible", "roadster", "cabrio", "cabriolet", "spyder")):
            candidates = ("OPEN", "Open")
        elif any(marker in normalized for marker in ("van", "bus", "people mover", "mpv", "minivan", "tourer")):
            candidates = ("MINIVAN",)
        elif any(marker in normalized for marker in ("truck", "chassis", "pritsche")):
            candidates = ("TRUCK", "Truck")
        elif "rv" in normalized or "motorized" in normalized:
            candidates = ("RV",)
        elif normalized in {"other", "unknown"}:
            candidates = ("OTHER", "Other")
        return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER"

    @staticmethod
    def _parse_engine_volume(value: str | None) -> int | None:
        if not value:
            return None
        normalized = value.strip().replace(",", ".")
        match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", normalized)
        if not match:
            cc_match = re.search(r"(\d{3,5})\s*(?:ccm|cm3|cm³|cc)\b", normalized, flags=re.IGNORECASE)
            if cc_match is not None:
                cc = int(cc_match.group(1))
                return cc if 0 < cc <= 10000 else None
            raw_digits = re.fullmatch(r"\s*(\d{3,5})\s*", normalized)
            if raw_digits is not None:
                cc = int(raw_digits.group(1))
                return cc if 0 < cc <= 10000 else None
            return None
        try:
            liters = float(match.group(1))
        except ValueError:
            return None
        cc = int(round(liters * 1000))
        if cc <= 0 or cc > 10000:
            return None
        return cc

    @staticmethod
    def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool:
        neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES}
        for value in (primary_damage, secondary_damage):
            if not value:
                continue
            normalized = value.replace(" ", "").strip().upper()
            if normalized and normalized not in neutral:
                return True
        return False

    @staticmethod
    def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool:
        if listing_vehicle is None:
            return False
        if listing_vehicle.timed_auction_closed:
            return True
        status = (listing_vehicle.inventory_status or "").strip().upper()
        return status in INACTIVE_STATUS_VALUES

    @staticmethod
    def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None:
        if not candidates:
            return fallback if fallback in allowed else None
        allowed_set = set(allowed)
        for candidate in candidates:
            if candidate in allowed_set:
                return candidate
        return fallback if fallback in allowed_set else None

    @staticmethod
    def _limit_text(value: str, max_length: int) -> str:
        return value if len(value) <= max_length else value[:max_length].rstrip()

    @staticmethod
    def _simplify_text(value: str) -> str:
        return (
            value.strip()
            .lower()
            .replace("ä", "ae")
            .replace("ö", "oe")
            .replace("ü", "ue")
            .replace("ß", "ss")
        )

    @staticmethod
    def _slugify(value: str) -> str:
        return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"

    @staticmethod
    def _generate_parser_id(origin_id: str) -> str:
        import hashlib
        from string import ascii_letters, digits

        digest = hashlib.sha256(origin_id.encode()).digest()
        alphabet = ascii_letters + digits
        base = len(alphabet)
        num = int.from_bytes(digest[:17], "big")
        chars: list[str] = []
        for _ in range(22):
            num, idx = divmod(num, base)
            chars.append(alphabet[idx])
        return "car-" + "".join(chars)