Fast IAAI implementation

2026-04-24 00:20:42 +03:00
parent 2b0cffc118
commit 3cdb717789
31 changed files with 4880 additions and 0 deletions
--- a/tests/test_sync_engine.py
+++ b/tests/test_sync_engine.py
@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from iaai_sync_service.client import ListingVehicle
+from iaai_sync_service.models import Base, Car, Image
+from iaai_sync_service.runtime_config import RuntimeConfig, RuntimeFilters
+from iaai_sync_service.settings import Settings
+from iaai_sync_service.sync_engine import (
+    prepare_car,
+    replace_images_if_changed,
+    run_sync_once,
+)
+
+
+def _session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
+    Base.metadata.create_all(engine)
+    return Session(bind=engine)
+
+
+def _settings() -> Settings:
+    return Settings(
+        database_url="sqlite+pysqlite:///:memory:",
+        celery_broker_url="redis://localhost:6379/0",
+        celery_result_backend="redis://localhost:6379/1",
+        iaai_base_url="https://www.iaai.com",
+        iaai_listing_start_url="",
+        sync_runtime_config_file=Path("sync_runtime_config.json"),
+        iaai_session_cookies="",
+        iaai_storage_state_path=Path("iaai_storage_state.json"),
+        iaai_login="",
+        iaai_password="",
+        http_timeout=10,
+        http_retries=1,
+        http_retry_backoff_ms=1,
+        fetch_concurrency=2,
+        db_commit_batch_size=20,
+        session_keepalive_enabled=True,
+        session_keepalive_interval_minutes=15,
+        schema_bootstrap_enabled=False,
+        advisory_lock_key=1,
+        error_summary_max_len=2000,
+    )
+
+
+def _listing_vehicle(inventory_id: str, *, status: str = "RS", closed: bool = False) -> ListingVehicle:
+    return ListingVehicle(
+        inventory_id=inventory_id,
+        tenant="US",
+        auction_id="1_1",
+        auction_date="2026-04-08T08:30:00+00:00",
+        inventory_status=status,
+        currency="USD",
+        timed_auction_closed=closed,
+        timed_auction_indicator=False,
+        prebid_indicator=True,
+        buynow_indicator=False,
+    )
+
+
+def _detail_payload(
+    inventory_id: str,
+    *,
+    make: str = "ACURA",
+    model: str = "RSX",
+    series: str = "BASE",
+    year: int = 2002,
+    bid: int = 500,
+) -> dict:
+    return {
+        "inventoryView": {
+            "attributes": {
+                "Id": inventory_id,
+                "Year": str(year),
+                "Make": make,
+                "Model": model,
+                "Series": series,
+                "Currency": "USD",
+                "ODOValue": "219187",
+                "ExteriorColor": "GRAY",
+                "DriveLineTypeDesc": "FWD",
+                "Transmission": "Automatic",
+                "BodyStyleName": "COUPE",
+                "EngineSize": "2.0L I-4",
+                "VehicleGrade": "50",
+                "PrimaryDamageDesc": "NORMAL WEAR & TEAR",
+                "SecondaryDamageDesc": " ",
+            },
+            "imageDimensions": {
+                "keys": {
+                    "$values": [
+                        {"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0},
+                        {"k": f"{inventory_id}~I2", "w": 1600, "h": 1200, "i": 1},
+                    ]
+                }
+            },
+        },
+        "auctionInformation": {
+            "biddingInformation": {
+                "buyNowAmount": 0,
+                "buyNowPrice": "$0",
+            },
+            "prebidInformation": {
+                "decimalHighBidAmount": str(bid),
+                "highBidAmount": f"${bid}",
+                "buyNowPrice": "$0",
+            },
+        },
+    }
+
+
+class FakeClient:
+    def __init__(self, listing: list[ListingVehicle], detail_payloads: dict[str, dict]) -> None:
+        self._listing = listing
+        self._detail_payloads = detail_payloads
+        self.detail_calls = 0
+        self.persist_calls = 0
+
+    def iter_listing_vehicles(self, filters: RuntimeFilters):  # noqa: ANN201, ARG002
+        return iter(self._listing)
+
+    def fetch_vehicle_detail_payload(self, inventory_id: str):  # noqa: ANN201
+        self.detail_calls += 1
+        return self._detail_payloads[inventory_id]
+
+    def persist_session_state(self) -> None:
+        self.persist_calls += 1
+
+
+def test_prepare_car_maps_fields() -> None:
+    row = prepare_car(
+        listing_vehicle=_listing_vehicle("45078011~US"),
+        detail_payload=_detail_payload("45078011~US", bid=600),
+        currency_labels={"USD", "CAD", "NA"},
+        country_labels={"US", "CA", "NA"},
+        drive_labels={"FWD", "RWD", "4WD", "NA"},
+        gearbox_labels={"AT", "MT", "NA"},
+        body_type_labels={"COUPE", "OTHER"},
+        body_type_default="OTHER",
+        steering_left="LEFT",
+        selling_type_auction="AUCTION",
+        origin_code="IAAI",
+    )
+
+    assert row.origin_id == "iaai:45078011~US"
+    assert row.origin_url == "https://www.iaai.com/VehicleDetail/45078011~US"
+    assert row.brand == "ACURA"
+    assert row.model == "RSX BASE"
+    assert row.year == 2002
+    assert row.price == 600
+    assert row.currency == "USD"
+    assert row.country == "US"
+    assert row.drive == "FWD"
+    assert row.gearbox == "AT"
+    assert row.body_type == "COUPE"
+    assert row.engine_volume == 2000
+    assert row.is_damaged is False
+    assert len(row.images) == 2
+
+
+def test_replace_images_if_changed_returns_zero_for_same_set() -> None:
+    session = _session()
+    car = Car(
+        parser_id="car-aaaaaaaaaaaaaaaaaaaaaa",
+        brand="ACURA",
+        model="RSX",
+        origin_id="iaai:45078011~US",
+        origin_url="https://www.iaai.com/VehicleDetail/45078011~US",
+        slug="acura-rsx-2002",
+        is_sold=False,
+        last_seen_at=datetime(2026, 4, 1, 12, 0, tzinfo=UTC),
+    )
+    session.add(car)
+    session.flush()
+    session.add(
+        Image(
+            car_id=car.id,
+            fullres_image="https://vis.iaai.com/resizer?imageKeys=a&width=1600&height=1200",
+            preview_image="https://vis.iaai.com/resizer?imageKeys=a&width=640&height=480",
+            order_index=0,
+        )
+    )
+    session.commit()
+
+    written = replace_images_if_changed(
+        session=session,
+        car_id=car.id,
+        images=[
+            {
+                "order_index": 0,
+                "fullres_image": "https://vis.iaai.com/resizer?imageKeys=a&width=1600&height=1200",
+                "preview_image": "https://vis.iaai.com/resizer?imageKeys=a&width=640&height=480",
+            }
+        ],
+    )
+
+    assert written == 0
+
+
+def test_run_sync_once_upserts_and_marks_missing_as_sold() -> None:
+    session = _session()
+
+    listing_first = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
+    detail_first = {
+        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
+        "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
+    }
+    client_first = FakeClient(listing_first, detail_first)
+
+    stats1, errors1 = run_sync_once(session=session, client=client_first, settings=_settings())
+    assert errors1 == []
+    assert stats1.ids_fetched == 2
+    assert stats1.cars_upserted == 2
+    assert client_first.persist_calls == 1
+
+    listing_second = [_listing_vehicle("45078011~US")]
+    detail_second = {
+        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=650),
+    }
+    client_second = FakeClient(listing_second, detail_second)
+
+    stats2, errors2 = run_sync_once(session=session, client=client_second, settings=_settings())
+    assert errors2 == []
+    assert stats2.ids_fetched == 1
+    assert stats2.cars_upserted == 1
+
+    active = session.query(Car).filter_by(origin_id="iaai:45078011~US").one()
+    sold = session.query(Car).filter_by(origin_id="iaai:45268167~US").one()
+    assert active.is_sold is False
+    assert active.price == 650
+    assert sold.is_sold is True
+
+
+def test_run_sync_once_skips_reconcile_when_runtime_filters_enabled() -> None:
+    session = _session()
+    session.add(
+        Car(
+            parser_id="car-zzzzzzzzzzzzzzzzzzzzzz",
+            brand="BMW",
+            model="X5",
+            origin_id="iaai:99999999~US",
+            origin_url="https://www.iaai.com/VehicleDetail/99999999~US",
+            slug="bmw-x5-2020",
+            is_sold=False,
+            last_seen_at=datetime(2026, 4, 1, 12, 0, tzinfo=UTC),
+        )
+    )
+    session.commit()
+
+    listing = [_listing_vehicle("45078011~US")]
+    details = {"45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500)}
+    runtime = RuntimeConfig(filters=RuntimeFilters(brands={"bmw"}))
+
+    stats, errors = run_sync_once(
+        session=session,
+        client=FakeClient(listing, details),
+        settings=_settings(),
+        runtime_config=runtime,
+    )
+
+    assert errors == []
+    assert stats.ids_fetched == 0
+
+    row = session.query(Car).filter_by(origin_id="iaai:99999999~US").one()
+    assert row.is_sold is False
+
+
+def test_run_sync_once_condition_check_skips_closed_rows() -> None:
+    session = _session()
+    listing = [_listing_vehicle("45078011~US", status="SOLD", closed=True)]
+    details = {"45078011~US": _detail_payload("45078011~US", bid=500)}
+
+    runtime = RuntimeConfig(condition_check_enabled=True)
+    stats, errors = run_sync_once(
+        session=session,
+        client=FakeClient(listing, details),
+        settings=_settings(),
+        runtime_config=runtime,
+    )
+
+    assert errors == []
+    assert stats.ids_fetched == 0
+    assert session.query(Car).count() == 0