Fast IAAI implementation

iaai_sync_service/__init__.py

@@ -0,0 +1 @@
+"""IAAI Cars sync service package."""

iaai_sync_service/__main__.py

@@ -0,0 +1,4 @@
+from iaai_sync_service.cli import main
+
+if __name__ == "__main__":
+    main()

iaai_sync_service/celery_app.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from celery import Celery
+from celery.schedules import crontab
+
+from iaai_sync_service.settings import get_settings
+
+settings = get_settings()
+
+app = Celery("iaai_sync_service")
+app.conf.broker_url = settings.celery_broker_url
+app.conf.result_backend = settings.celery_result_backend
+app.conf.task_default_queue = "iaai_sync"
+app.conf.timezone = "UTC"
+app.conf.enable_utc = True
+app.conf.broker_connection_retry = True
+app.conf.broker_connection_retry_on_startup = True
+app.conf.broker_connection_max_retries = None
+
+def build_beat_schedule() -> dict[str, dict]:
+    schedule: dict[str, dict] = {
+        "sync-iaai-cars-every-hour": {
+            "task": "iaai.sync_cars_feed",
+            "schedule": crontab(minute=0),
+            "options": {"queue": "iaai_sync"},
+        }
+    }
+    if settings.session_keepalive_enabled:
+        schedule["keep-iaai-session-alive"] = {
+            "task": "iaai.keep_session_alive",
+            "schedule": crontab(minute=f"*/{settings.session_keepalive_interval_minutes}"),
+            "options": {"queue": "iaai_sync"},
+        }
+    return schedule
+
+
+app.conf.beat_schedule = build_beat_schedule()
+app.conf.imports = ("iaai_sync_service.tasks",)

iaai_sync_service/cli.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import json
+
+from iaai_sync_service.tasks import execute_sync_job
+
+
+def main() -> None:
+    result = execute_sync_job()
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

iaai_sync_service/client.py

@@ -0,0 +1,950 @@
+from __future__ import annotations
+
+import html
+import json
+import logging
+import math
+import re
+import threading
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote, urljoin
+
+import requests
+from requests.adapters import HTTPAdapter
+
+from iaai_sync_service.runtime_config import RuntimeFilters
+from iaai_sync_service.settings import Settings
+
+logger = logging.getLogger(__name__)
+
+TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
+CHALLENGE_MARKERS = (
+    "_incapsula_resource",
+    "incapsula",
+    "incident id",
+    "request unsuccessful",
+    "access denied",
+)
+COOKIE_ACCEPT_SELECTORS = (
+    "button:has-text('Accept All')",
+    "button:has-text('Accept all')",
+    "button:has-text('I Agree')",
+    "button:has-text('Agree')",
+    "button:has-text('Only necessary')",
+    "button:has-text('Только необходимые')",
+    "button:has-text('Принять все')",
+    "[id*='accept']",
+    "[class*='accept']",
+)
+LISTING_MARKER = 'id="GBPSearchQuery"'
+DETAIL_MARKER = 'id="ProductDetailsVM"'
+RESIZER_URL = "https://vis.iaai.com/resizer"
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/124.0.0.0 Safari/537.36"
+)
+MAX_CHALLENGE_REFRESH_ATTEMPTS = 3
+PLAYWRIGHT_REFRESH_POLLS = 8
+
+
+@dataclass(frozen=True)
+class ListingVehicle:
+    inventory_id: str
+    tenant: str | None
+    auction_id: str | None
+    auction_date: str | None
+    inventory_status: str | None
+    currency: str | None
+    timed_auction_closed: bool
+    timed_auction_indicator: bool
+    prebid_indicator: bool
+    buynow_indicator: bool
+
+
+@dataclass(frozen=True)
+class ListingPage:
+    vehicles: list[ListingVehicle]
+    result_count: int
+    page_size: int
+    current_page: int
+    gbp_search_query: dict[str, Any]
+
+
+class HybridSessionAuth:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._thread_local = threading.local()
+        self._lock = threading.Lock()
+        self._refresh_lock = threading.Lock()
+        self._bootstrap_cookies_loaded = False
+        self._anonymous_bootstrap_attempted = False
+        self._refresh_generation = 0
+        self._latest_refresh_cookies: list[dict[str, Any]] = []
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        timeout: int,
+        retries: int,
+        retry_backoff_ms: int,
+        headers: dict[str, str] | None = None,
+        data: Any | None = None,
+        json_body: Any | None = None,
+        expected_marker: str | None = None,
+    ) -> requests.Response:
+        session = self._get_session()
+        self._ensure_anonymous_session_bootstrap(session=session)
+        self._sync_session_with_latest_refresh(session)
+        last_error: Exception | None = None
+        refresh_attempts = 0
+        attempt = 0
+        while attempt <= retries:
+            try:
+                response = session.request(
+                    method=method,
+                    url=url,
+                    headers=headers,
+                    data=data,
+                    json=json_body,
+                    timeout=timeout,
+                )
+            except requests.RequestException as exc:
+                last_error = exc
+                if attempt >= retries:
+                    break
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries:
+                response.close()
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if is_challenge_response(
+                status_code=response.status_code,
+                body_text=response.text,
+                expected_marker=expected_marker,
+            ):
+                response.close()
+                if refresh_attempts >= MAX_CHALLENGE_REFRESH_ATTEMPTS:
+                    raise RuntimeError(
+                        "IAAI challenge persisted after "
+                        f"{refresh_attempts} Playwright refresh attempts for url={url}"
+                    )
+                refresh_attempts += 1
+                logger.warning(
+                    "Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s",
+                    url,
+                    response.status_code,
+                    expected_marker,
+                    refresh_attempts,
+                    MAX_CHALLENGE_REFRESH_ATTEMPTS,
+                )
+                self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+                if refresh_attempts > 1:
+                    self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
+                continue
+
+            return response
+
+        if last_error is not None:
+            raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error
+        raise RuntimeError(f"Request failed url={url} after retries")
+
+    def persist_storage_state(self) -> None:
+        session = self._get_session()
+        with self._lock:
+            self._save_storage_state(session)
+
+    def _get_session(self) -> requests.Session:
+        session = getattr(self._thread_local, "session", None)
+        if session is None:
+            session = requests.Session()
+            adapter = HTTPAdapter(pool_connections=20, pool_maxsize=20)
+            session.mount("http://", adapter)
+            session.mount("https://", adapter)
+            session.headers.update(
+                {
+                    "user-agent": DEFAULT_USER_AGENT,
+                    "accept-language": "en-US,en;q=0.9",
+                    "cache-control": "no-cache",
+                    "pragma": "no-cache",
+                }
+            )
+            with self._lock:
+                self._bootstrap_session_cookies(session)
+            self._thread_local.session = session
+            self._thread_local.session_generation = 0
+            self._sync_session_with_latest_refresh(session)
+        return session
+
+    def _sync_session_with_latest_refresh(self, session: requests.Session) -> None:
+        with self._lock:
+            latest_generation = self._refresh_generation
+            session_generation = getattr(self._thread_local, "session_generation", 0)
+            if latest_generation <= session_generation or not self._latest_refresh_cookies:
+                return
+            cookies = list(self._latest_refresh_cookies)
+        self._apply_cookies_to_session(session, cookies)
+        self._thread_local.session_generation = latest_generation
+
+    def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None:
+        if self._anonymous_bootstrap_attempted:
+            return
+        if self._session_has_iaai_cookies(session):
+            self._anonymous_bootstrap_attempted = True
+            return
+        with self._lock:
+            if self._anonymous_bootstrap_attempted:
+                return
+            self._anonymous_bootstrap_attempted = True
+        logger.info("No IAAI cookies preloaded. Attempting anonymous session bootstrap via Playwright.")
+        try:
+            self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s",
+                exc,
+            )
+
+    @staticmethod
+    def _session_has_iaai_cookies(session: requests.Session) -> bool:
+        cookies = getattr(session, "cookies", None)
+        if cookies is None:
+            return False
+        for item in cookies:
+            domain = ""
+            if isinstance(item, dict):
+                domain = parse_text(item.get("domain")) or ""
+            else:
+                domain = str(getattr(item, "domain", "") or "")
+            if not domain:
+                return True
+            if "iaai.com" in domain.lower():
+                return True
+        return False
+
+    def _bootstrap_session_cookies(self, session: requests.Session) -> None:
+        if self._bootstrap_cookies_loaded:
+            return
+        self._load_storage_state_cookies(session)
+        self._load_env_cookies(session)
+        self._bootstrap_cookies_loaded = True
+
+    def _load_storage_state_cookies(self, session: requests.Session) -> None:
+        path = self._settings.iaai_storage_state_path
+        if not path.exists():
+            return
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Failed to read storage state file '%s': %s", path, exc)
+            return
+        cookies = payload.get("cookies") if isinstance(payload, dict) else None
+        if not isinstance(cookies, list):
+            return
+        applied = 0
+        for item in cookies:
+            if not isinstance(item, dict):
+                continue
+            name = parse_text(item.get("name"))
+            value = parse_text(item.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(item.get("domain")) or ".iaai.com"
+            cookie_path = parse_text(item.get("path")) or "/"
+            expires_raw = item.get("expires")
+            expires = parse_int(expires_raw)
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+            applied += 1
+        if applied:
+            logger.info("Loaded %s cookies from storage state", applied)
+
+    def _load_env_cookies(self, session: requests.Session) -> None:
+        raw = self._settings.iaai_session_cookies
+        if not raw:
+            return
+
+        try:
+            payload = json.loads(raw)
+        except Exception:  # noqa: BLE001
+            payload = None
+
+        applied = 0
+        if isinstance(payload, list):
+            for item in payload:
+                if not isinstance(item, dict):
+                    continue
+                name = parse_text(item.get("name"))
+                value = parse_text(item.get("value"))
+                if not name or value is None:
+                    continue
+                domain = parse_text(item.get("domain")) or ".iaai.com"
+                cookie_path = parse_text(item.get("path")) or "/"
+                expires = parse_int(item.get("expires"))
+                session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+                applied += 1
+        elif isinstance(payload, dict):
+            for name, value in payload.items():
+                if not isinstance(name, str) or not isinstance(value, str):
+                    continue
+                session.cookies.set(name.strip(), value.strip(), domain=".iaai.com", path="/")
+                applied += 1
+        else:
+            for part in raw.split(";"):
+                if "=" not in part:
+                    continue
+                name, value = part.split("=", 1)
+                name = name.strip()
+                value = value.strip()
+                if not name:
+                    continue
+                session.cookies.set(name, value, domain=".iaai.com", path="/")
+                applied += 1
+
+        if applied:
+            logger.info("Loaded %s cookies from IAAI_SESSION_COOKIES", applied)
+
+    def _refresh_session_via_playwright(
+        self,
+        *,
+        expected_marker: str | None = None,
+        session: requests.Session | None = None,
+    ) -> None:
+        target_session = session or self._get_session()
+        with self._lock:
+            baseline_generation = self._refresh_generation
+
+        with self._refresh_lock:
+            with self._lock:
+                if self._refresh_generation > baseline_generation and self._latest_refresh_cookies:
+                    self._apply_cookies_to_session(target_session, self._latest_refresh_cookies)
+                    self._thread_local.session_generation = self._refresh_generation
+                    return
+
+            logger.warning("IAAI session challenge detected. Refreshing session via Playwright.")
+            cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
+            self._apply_cookies_to_session(target_session, cookies)
+
+            with self._lock:
+                self._refresh_generation += 1
+                self._latest_refresh_cookies = list(cookies)
+                self._anonymous_bootstrap_attempted = True
+                refreshed_generation = self._refresh_generation
+            self._thread_local.session_generation = refreshed_generation
+            self._save_storage_state(target_session)
+
+    def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
+        try:
+            from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+            from playwright.sync_api import sync_playwright
+        except Exception as exc:  # noqa: BLE001
+            raise RuntimeError(
+                "Playwright is required for challenge fallback. Install dependency and browsers."
+            ) from exc
+
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=True)
+            try:
+                context = browser.new_context(
+                    locale="en-US",
+                    viewport={"width": 1366, "height": 768},
+                    user_agent=DEFAULT_USER_AGENT,
+                )
+                page = context.new_page()
+                home_target = self._settings.iaai_base_url + "/"
+                target = urljoin(self._settings.iaai_base_url + "/", "Vehiclelisting/Cars")
+                timeout_ms = max(30_000, self._settings.http_timeout * 1000)
+                page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                self._try_login(page)
+                self._wait_until_non_challenge(
+                    page=page,
+                    target=target,
+                    timeout_ms=timeout_ms,
+                    expected_marker=expected_marker or LISTING_MARKER,
+                )
+                state = context.storage_state()
+            except PlaywrightTimeoutError as exc:
+                raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
+            finally:
+                browser.close()
+
+        cookies = state.get("cookies") if isinstance(state, dict) else None
+        if not isinstance(cookies, list) or not cookies:
+            raise RuntimeError("Playwright refresh did not return cookies")
+        return [cookie for cookie in cookies if isinstance(cookie, dict)]
+
+    @staticmethod
+    def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None:
+        session.cookies.clear()
+        for cookie in cookies:
+            name = parse_text(cookie.get("name"))
+            value = parse_text(cookie.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(cookie.get("domain")) or ".iaai.com"
+            cookie_path = parse_text(cookie.get("path")) or "/"
+            expires = parse_int(cookie.get("expires"))
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+
+    @staticmethod
+    def _wait_until_non_challenge(
+        *,
+        page: Any,
+        target: str,
+        timeout_ms: int,
+        expected_marker: str | None,
+    ) -> None:
+        # Give Incapsula redirect/challenge flow time to settle and verify page really opened.
+        poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
+        navigation_error_count = 0
+        for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
+            try:
+                page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
+            except Exception:  # noqa: BLE001
+                # The challenge page frequently redirects; continue polling.
+                pass
+            page.wait_for_timeout(poll_ms)
+
+            body: str | None = None
+            for _ in range(3):
+                try:
+                    body = page.content()
+                    break
+                except Exception as exc:  # noqa: BLE001
+                    message = str(exc).lower()
+                    if "page.content" not in message or "navigating and changing the content" not in message:
+                        raise
+                    navigation_error_count += 1
+                    page.wait_for_timeout(max(200, poll_ms // 4))
+                    continue
+
+            if body is not None and not is_challenge_response(
+                status_code=200,
+                body_text=body,
+                expected_marker=expected_marker,
+            ):
+                return
+            try:
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+            except Exception as exc:  # noqa: BLE001
+                logger.debug(
+                    "Playwright challenge retry navigation failed poll=%s/%s: %s",
+                    poll_index + 1,
+                    PLAYWRIGHT_REFRESH_POLLS,
+                    exc,
+                )
+
+        raise RuntimeError(
+            "Playwright refresh completed but challenge page is still active "
+            f"(navigation_content_errors={navigation_error_count})"
+        )
+
+    def _try_login(self, page: Any) -> None:
+        if not self._settings.iaai_login or not self._settings.iaai_password:
+            return
+
+        try:
+            login_selectors = (
+                "input[type='email']",
+                "input[name*='email' i]",
+                "input[id*='email' i]",
+                "input[name*='username' i]",
+            )
+            password_selector = "input[type='password']"
+            submit_selector = "button[type='submit'],input[type='submit']"
+
+            email_locator = None
+            for selector in login_selectors:
+                locator = page.locator(selector)
+                if locator.count() > 0:
+                    email_locator = locator.first
+                    break
+            if email_locator is None:
+                return
+
+            email_locator.fill(self._settings.iaai_login)
+            password_locator = page.locator(password_selector)
+            if password_locator.count() == 0:
+                return
+            password_locator.first.fill(self._settings.iaai_password)
+            submit_locator = page.locator(submit_selector)
+            if submit_locator.count() > 0:
+                submit_locator.first.click()
+                page.wait_for_load_state("domcontentloaded", timeout=30_000)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Playwright login step failed: %s", exc)
+
+    def _accept_cookie_banner(self, page: Any) -> None:
+        for selector in COOKIE_ACCEPT_SELECTORS:
+            try:
+                locator = page.locator(selector).first
+                if locator.count() == 0:
+                    continue
+                if not locator.is_visible(timeout=500):
+                    continue
+                locator.click(timeout=2_000)
+                page.wait_for_timeout(250)
+                return
+            except Exception:  # noqa: BLE001
+                continue
+
+    def _save_storage_state(self, session: requests.Session) -> None:
+        path = self._settings.iaai_storage_state_path
+        cookies: list[dict[str, Any]] = []
+        for cookie in session.cookies:
+            cookie_payload: dict[str, Any] = {
+                "name": cookie.name,
+                "value": cookie.value,
+                "domain": cookie.domain or ".iaai.com",
+                "path": cookie.path or "/",
+                "httpOnly": False,
+                "secure": bool(cookie.secure),
+                "sameSite": "Lax",
+            }
+            if cookie.expires is not None:
+                cookie_payload["expires"] = int(cookie.expires)
+            cookies.append(cookie_payload)
+
+        payload = {"cookies": cookies, "origins": []}
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    @staticmethod
+    def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None:
+        if retry_backoff_ms <= 0:
+            return
+        delay = retry_backoff_ms * (2**attempt) / 1000
+        time.sleep(delay)
+
+
+class IAAIClient:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._auth = HybridSessionAuth(settings)
+
+    def persist_session_state(self) -> None:
+        self._auth.persist_storage_state()
+
+    def keep_session_alive(self) -> dict[str, Any]:
+        started = time.perf_counter()
+        scope = resolve_listing_scope_paths(
+            listing_start_url=self._settings.iaai_listing_start_url,
+            brands=set(),
+        )[0]
+        page_html = self._fetch_listing_first_page(scope)
+        marker_present = LISTING_MARKER in page_html
+        result_count = parse_int(parse_hidden_input_value(page_html, "ResultCount"))
+        self.persist_session_state()
+        return {
+            "scope": scope,
+            "marker_present": marker_present,
+            "result_count": result_count,
+            "elapsed_ms": int((time.perf_counter() - started) * 1000),
+        }
+
+    def iter_listing_vehicles(self, filters: RuntimeFilters) -> Iterator[ListingVehicle]:
+        seen_inventory_ids: set[str] = set()
+        if self._settings.iaai_listing_start_url.strip() and filters.brands:
+            logger.info(
+                "Using explicit listing start URL; runtime brand scopes in listing step are ignored."
+            )
+        scope_paths = resolve_listing_scope_paths(
+            listing_start_url=self._settings.iaai_listing_start_url,
+            brands=filters.brands,
+        )
+        listing_started_at = time.perf_counter()
+
+        for scope_index, scope_path in enumerate(scope_paths, start=1):
+            logger.info(
+                "IAAI listing scope start scope=%s/%s path=%s",
+                scope_index,
+                len(scope_paths),
+                scope_path,
+            )
+            first_page_html = self._fetch_listing_first_page(scope_path)
+            first_page = parse_listing_page(first_page_html)
+            for vehicle in first_page.vehicles:
+                if vehicle.inventory_id in seen_inventory_ids:
+                    continue
+                seen_inventory_ids.add(vehicle.inventory_id)
+                yield vehicle
+
+            page_size = max(1, first_page.page_size)
+            total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size))
+            gbp_search_query = first_page.gbp_search_query
+            logger.info(
+                "IAAI listing scope loaded scope=%s/%s path=%s result_count=%s page_size=%s total_pages=%s unique_ids=%s elapsed=%.1fs",
+                scope_index,
+                len(scope_paths),
+                scope_path,
+                first_page.result_count,
+                page_size,
+                total_pages,
+                len(seen_inventory_ids),
+                time.perf_counter() - listing_started_at,
+            )
+
+            for page_number in range(2, total_pages + 1):
+                page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size)
+                parsed_page = parse_listing_page(page_html)
+                gbp_search_query = parsed_page.gbp_search_query
+                for vehicle in parsed_page.vehicles:
+                    if vehicle.inventory_id in seen_inventory_ids:
+                        continue
+                    seen_inventory_ids.add(vehicle.inventory_id)
+                    yield vehicle
+                if page_number % 10 == 0 or page_number == total_pages:
+                    logger.info(
+                        "IAAI listing page progress scope=%s/%s path=%s page=%s/%s unique_ids=%s elapsed=%.1fs",
+                        scope_index,
+                        len(scope_paths),
+                        scope_path,
+                        page_number,
+                        total_pages,
+                        len(seen_inventory_ids),
+                        time.perf_counter() - listing_started_at,
+                    )
+
+    def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]:
+        escaped_id = quote(inventory_id, safe="~")
+        url = urljoin(self._settings.iaai_base_url + "/", f"VehicleDetail/{escaped_id}")
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=self._settings.http_timeout,
+            retries=self._settings.http_retries,
+            retry_backoff_ms=self._settings.http_retry_backoff_ms,
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=DETAIL_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
+            return parse_product_details_vm(response.text)
+
+    def _fetch_listing_first_page(self, scope_path: str) -> str:
+        if scope_path.lower().startswith(("http://", "https://")):
+            url = scope_path
+        else:
+            url = urljoin(self._settings.iaai_base_url + "/", scope_path.lstrip("/"))
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=self._settings.http_timeout,
+            retries=self._settings.http_retries,
+            retry_backoff_ms=self._settings.http_retry_backoff_ms,
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=LISTING_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}")
+            return response.text
+
+    def _fetch_listing_page(
+        self,
+        scope_path: str,
+        gbp_search_query: dict[str, Any],
+        page_number: int,
+        page_size: int,
+    ) -> str:
+        query_payload = dict(gbp_search_query)
+        query_payload["CurrentPage"] = page_number
+        query_payload["PageSize"] = page_size
+
+        search_url = urljoin(self._settings.iaai_base_url + "/", "Search")
+        common_headers = {
+            "accept": "text/html,application/xhtml+xml,*/*",
+            "x-requested-with": "XMLHttpRequest",
+        }
+
+        attempts: list[tuple[dict[str, str], Any, Any]] = [
+            ({**common_headers, "content-type": "application/json"}, None, query_payload),
+            ({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}),
+            ({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None),
+            (
+                {**common_headers, "content-type": "application/json"},
+                json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}),
+                None,
+            ),
+        ]
+
+        last_error: Exception | None = None
+        for headers, data, json_body in attempts:
+            try:
+                response = self._auth.request(
+                    "POST",
+                    search_url,
+                    timeout=self._settings.http_timeout,
+                    retries=self._settings.http_retries,
+                    retry_backoff_ms=self._settings.http_retry_backoff_ms,
+                    headers=headers,
+                    data=data,
+                    json_body=json_body,
+                    expected_marker=LISTING_MARKER,
+                )
+                with response:
+                    if response.status_code >= 400:
+                        raise RuntimeError(
+                            f"Listing page request failed status={response.status_code} page={page_number}"
+                        )
+                    body = response.text
+                    if LISTING_MARKER not in body:
+                        raise RuntimeError("Listing page response does not include GBPSearchQuery")
+                    return body
+            except Exception as exc:  # noqa: BLE001
+                last_error = exc
+                continue
+
+        if last_error is not None:
+            raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error
+        raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}")
+
+
+def build_brand_scope_paths(brands: set[str]) -> list[str]:
+    if not brands:
+        return ["/Vehiclelisting/Cars"]
+
+    paths: list[str] = []
+    for brand in sorted(brands):
+        raw = brand.strip()
+        if not raw:
+            continue
+        slug_hyphen = quote(raw.replace(" ", "-"), safe="-")
+        slug_raw = quote(raw, safe="")
+        for slug in (slug_hyphen, slug_raw):
+            path = f"/Vehiclelisting/Cars/{slug}"
+            if path not in paths:
+                paths.append(path)
+    return paths or ["/Vehiclelisting/Cars"]
+
+
+def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
+    explicit_scope = listing_start_url.strip()
+    if explicit_scope:
+        return [explicit_scope]
+    return build_brand_scope_paths(brands)
+
+
+def parse_listing_page(html_text: str) -> ListingPage:
+    gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
+    vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
+    result_count_raw = parse_hidden_input_value(html_text, "ResultCount")
+    page_size_raw = parse_hidden_input_value(html_text, "PageSize")
+    current_page_raw = parse_hidden_input_value(html_text, "CurrentPage")
+
+    if not gbp_raw:
+        raise RuntimeError("Listing page missing GBPSearchQuery")
+    if vehicle_raw is None:
+        raise RuntimeError("Listing page missing VehicleDetails")
+
+    gbp_payload = json.loads(gbp_raw)
+    if not isinstance(gbp_payload, dict):
+        raise RuntimeError("GBPSearchQuery payload is not object")
+
+    vehicle_payload = json.loads(vehicle_raw)
+    if not isinstance(vehicle_payload, list):
+        raise RuntimeError("VehicleDetails payload is not array")
+
+    vehicles: list[ListingVehicle] = []
+    for item in vehicle_payload:
+        if not isinstance(item, dict):
+            continue
+        inventory_id = parse_text(item.get("Id"))
+        if not inventory_id:
+            continue
+        vehicles.append(
+            ListingVehicle(
+                inventory_id=inventory_id,
+                tenant=parse_text(item.get("Tenant")),
+                auction_id=parse_text(item.get("ActnLnId")),
+                auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")),
+                inventory_status=parse_text(item.get("InventoryStatus")),
+                currency=parse_text(item.get("Currency")),
+                timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")),
+                timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")),
+                prebid_indicator=parse_bool(item.get("PreBidIndicator")),
+                buynow_indicator=parse_bool(item.get("BuyNowIndicator")),
+            )
+        )
+
+    return ListingPage(
+        vehicles=vehicles,
+        result_count=parse_int(result_count_raw) or len(vehicles),
+        page_size=parse_int(page_size_raw) or max(1, len(vehicles)),
+        current_page=parse_int(current_page_raw) or 1,
+        gbp_search_query=gbp_payload,
+    )
+
+
+def parse_product_details_vm(html_text: str) -> dict[str, Any]:
+    match = re.search(
+        r"<script[^>]*id=\"ProductDetailsVM\"[^>]*>\s*(\{.*?\})\s*</script>",
+        html_text,
+        flags=re.DOTALL,
+    )
+    if match is None:
+        raise RuntimeError("ProductDetailsVM script not found")
+
+    payload = json.loads(match.group(1))
+    if not isinstance(payload, dict):
+        raise RuntimeError("ProductDetailsVM root is not object")
+    return payload
+
+
+def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
+    escaped_id = re.escape(input_id)
+    patterns = (
+        rf"<input[^>]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"",
+        rf"<input[^>]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, html_text, flags=re.IGNORECASE)
+        if match is not None:
+            return html.unescape(match.group(1))
+    return None
+
+
+def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
+    seen_fullres: set[str] = set()
+    images: list[dict[str, str | int]] = []
+
+    for index, item in enumerate(image_keys):
+        if not isinstance(item, dict):
+            continue
+        key = parse_text(item.get("k"))
+        if key is None:
+            continue
+
+        width = parse_int(item.get("w")) or 1600
+        height = parse_int(item.get("h")) or 1200
+        if width <= 0:
+            width = 1600
+        if height <= 0:
+            height = 1200
+
+        order_index = parse_int(item.get("i"))
+        if order_index is None:
+            order_index = parse_int(item.get("in"))
+        if order_index is None:
+            order_index = index
+
+        preview_width = min(640, width)
+        preview_height = max(1, int(round(height * (preview_width / width))))
+
+        escaped_key = quote(key, safe="~")
+        fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}"
+        preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}"
+
+        if fullres in seen_fullres:
+            continue
+        seen_fullres.add(fullres)
+        images.append(
+            {
+                "order_index": order_index,
+                "fullres_image": fullres,
+                "preview_image": preview,
+            }
+        )
+
+    images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image"))))
+    return images
+
+
+def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool:
+    if status_code in {401, 403}:
+        return True
+
+    if _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        # If expected listing/detail marker is present, this is a valid page even if
+        # Incapsula script references are embedded in the HTML.
+        return False
+
+    lowered = (body_text or "").lower()
+    if any(marker in lowered for marker in CHALLENGE_MARKERS):
+        return True
+
+    if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        # Expected hidden marker/script missing from HTML often means anti-bot interstitial.
+        if "<html" in lowered or "<body" in lowered:
+            return True
+    return False
+
+
+def _expected_marker_present(*, body_text: str, expected_marker: str | None) -> bool:
+    if not expected_marker:
+        return False
+    if expected_marker in body_text:
+        return True
+
+    # Accept quote variants for marker fragments like id="GBPSearchQuery" / id='GBPSearchQuery'.
+    if '"' in expected_marker:
+        single_quoted = expected_marker.replace('"', "'")
+        if single_quoted in body_text:
+            return True
+    if "'" in expected_marker:
+        double_quoted = expected_marker.replace("'", '"')
+        if double_quoted in body_text:
+            return True
+
+    marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker)
+    if marker_match is None:
+        return False
+    marker_id = re.escape(marker_match.group(1))
+    return bool(
+        re.search(
+            rf"id\s*=\s*['\"]{marker_id}['\"]",
+            body_text,
+            flags=re.IGNORECASE,
+        )
+    )
+
+
+def parse_text(value: Any) -> str | None:
+    if isinstance(value, str):
+        text = value.strip()
+        return text if text else None
+    return None
+
+
+def parse_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        return normalized in {"true", "1", "yes", "on"}
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return value != 0
+    return False
+
+
+def parse_int(value: Any) -> int | None:
+    if value is None or isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(round(value))
+    if isinstance(value, str):
+        text = value.strip()
+        if not text:
+            return None
+        normalized = text.replace(",", "").replace(" ", "")
+        try:
+            return int(round(float(normalized)))
+        except ValueError:
+            return None
+    return None

iaai_sync_service/database.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from iaai_sync_service.settings import get_settings
+
+_settings = get_settings()
+
+engine = create_engine(_settings.database_url, pool_pre_ping=True, future=True)
+SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False)
+
+
+def get_session() -> Session:
+    return SessionLocal()

iaai_sync_service/models.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import (
+    BigInteger,
+    Boolean,
+    DateTime,
+    Enum,
+    ForeignKey,
+    Integer,
+    String,
+    Text,
+    func,
+)
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+CAR_TABLE_NAME = "iaai_cars"
+IMAGE_TABLE_NAME = "iaai_images"
+SYNC_RUN_TABLE_NAME = "iaai_sync_runs"
+
+
+CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
+DRIVE_ENUM_VALUES = ("FWD", "RWD", "TWO_WD", "FOUR_WD", "2WD", "4WD", "NA")
+GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
+STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "left", "right", "NA")
+BODY_TYPE_ENUM_VALUES = (
+    "COUPE",
+    "SUV",
+    "HATCHBACK",
+    "MINIVAN",
+    "SEDAN",
+    "NA",
+    "Station Wagon",
+    "Pickup",
+    "Truck",
+    "Open",
+    "RV",
+    "Other",
+    "STATION_WAGON",
+    "PICKUP",
+    "TRUCK",
+    "OPEN",
+    "OTHER",
+)
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
+ORIGIN_ENUM_VALUES = (
+    "TAU",
+    "CARSENSOR",
+    "HANAMARU",
+    "ENCAR",
+    "KURUMA_TRADER",
+    "carsensor",
+    "encar",
+    "kuruma_trader",
+    "asnet",
+    "kababa",
+    "ACV",
+    "COPART",
+    "IAAI",
+    "copart",
+    "iaai",
+    "NA",
+    "ASNET",
+    "KABABA",
+)
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "stock", "auction", "tender", "NA")
+
+
+class Car(Base):
+    __tablename__ = CAR_TABLE_NAME
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
+    brand: Mapped[str] = mapped_column(String(50), nullable=False)
+    model: Mapped[str] = mapped_column(String(50), nullable=False)
+    year: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    currency: Mapped[str] = mapped_column(
+        Enum(
+            *CURRENCY_ENUM_VALUES,
+            name="currencyenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="USD",
+    )
+    mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    country: Mapped[str] = mapped_column(
+        Enum(
+            *COUNTRY_ENUM_VALUES,
+            name="countryenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="NA",
+    )
+    is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
+    drive: Mapped[str | None] = mapped_column(
+        Enum(
+            *DRIVE_ENUM_VALUES,
+            name="driveenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=True,
+    )
+    gearbox: Mapped[str | None] = mapped_column(
+        Enum(
+            *GEARBOX_ENUM_VALUES,
+            name="gearboxenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=True,
+    )
+    steering_wheel: Mapped[str | None] = mapped_column(
+        Enum(
+            *STEERING_WHEEL_ENUM_VALUES,
+            name="steeringwheelenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=True,
+    )
+    body_type: Mapped[str] = mapped_column(
+        Enum(
+            *BODY_TYPE_ENUM_VALUES,
+            name="bodytypeenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="OTHER",
+    )
+    engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    selling_type: Mapped[str] = mapped_column(
+        Enum(
+            *SELLING_TYPE_ENUM_VALUES,
+            name="sellingtypeenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="NA",
+    )
+    one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    origin: Mapped[str] = mapped_column(
+        Enum(
+            *ORIGIN_ENUM_VALUES,
+            name="originenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="NA",
+    )
+    origin_url: Mapped[str] = mapped_column(String(), nullable=False)
+    origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True)
+    is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
+    non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    slug: Mapped[str] = mapped_column(String(), nullable=False)
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+
+    images: Mapped[list["Image"]] = relationship(
+        "Image",
+        back_populates="car",
+        cascade="all, delete-orphan",
+    )
+
+
+class Image(Base):
+    __tablename__ = IMAGE_TABLE_NAME
+
+    id: Mapped[int] = mapped_column(
+        BigInteger().with_variant(Integer, "sqlite"),
+        primary_key=True,
+        autoincrement=True,
+    )
+    fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
+    preview_image: Mapped[str] = mapped_column(String(), nullable=False)
+    order_index: Mapped[int] = mapped_column(Integer, nullable=False)
+    car_id: Mapped[int] = mapped_column(
+        Integer,
+        ForeignKey(f"{CAR_TABLE_NAME}.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+
+    car: Mapped[Car] = relationship("Car", back_populates="images")
+
+
+class SyncRun(Base):
+    __tablename__ = SYNC_RUN_TABLE_NAME
+
+    id: Mapped[int] = mapped_column(
+        BigInteger().with_variant(Integer, "sqlite"),
+        primary_key=True,
+        autoincrement=True,
+    )
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    status: Mapped[str] = mapped_column(Text, nullable=False)
+    lane: Mapped[str] = mapped_column(Text, nullable=False)
+    ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)

iaai_sync_service/runtime_config.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class RuntimeFilters:
+    brands: set[str] = field(default_factory=set)
+    models: set[str] = field(default_factory=set)
+    years: set[int] = field(default_factory=set)
+    body_types: set[str] = field(default_factory=set)
+
+    def is_enabled(self) -> bool:
+        return bool(self.brands or self.models or self.years or self.body_types)
+
+
+@dataclass(frozen=True)
+class RuntimeConfig:
+    condition_check_enabled: bool | None = None
+    filters: RuntimeFilters = field(default_factory=RuntimeFilters)
+
+
+def load_runtime_config(path: Path, logger: logging.Logger | None = None) -> RuntimeConfig:
+    if not path.exists():
+        return RuntimeConfig()
+
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except Exception as exc:  # noqa: BLE001
+        if logger is not None:
+            logger.warning("Failed to parse runtime config file '%s': %s", path, exc)
+        return RuntimeConfig()
+
+    if not isinstance(payload, dict):
+        if logger is not None:
+            logger.warning("Runtime config file '%s' root must be object.", path)
+        return RuntimeConfig()
+
+    sync_block = payload.get("sync")
+    filters_block = payload.get("filters")
+    if not isinstance(sync_block, dict):
+        sync_block = {}
+    if not isinstance(filters_block, dict):
+        filters_block = {}
+
+    return RuntimeConfig(
+        condition_check_enabled=_to_optional_bool(sync_block.get("condition_check_enabled")),
+        filters=RuntimeFilters(
+            brands=_to_ci_set(filters_block.get("brands")),
+            models=_to_ci_set(filters_block.get("models")),
+            years=_to_int_set(filters_block.get("years"), min_value=1900),
+            body_types=_to_ci_set(filters_block.get("body_types")),
+        )
+    )
+
+
+def _to_optional_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        if normalized in {"true", "1", "yes", "on"}:
+            return True
+        if normalized in {"false", "0", "no", "off"}:
+            return False
+    return None
+
+
+def _to_ci_set(value: Any) -> set[str]:
+    if not isinstance(value, list):
+        return set()
+    result: set[str] = set()
+    for item in value:
+        if not isinstance(item, str):
+            continue
+        text = item.strip().casefold()
+        if text:
+            result.add(text)
+    return result
+
+
+def _to_int_set(value: Any, *, min_value: int) -> set[int]:
+    if not isinstance(value, list):
+        return set()
+    result: set[int] = set()
+    for item in value:
+        if isinstance(item, bool):
+            continue
+        if isinstance(item, int):
+            if item >= min_value:
+                result.add(item)
+            continue
+        if isinstance(item, str) and item.strip().isdigit():
+            parsed = int(item.strip())
+            if parsed >= min_value:
+                result.add(parsed)
+    return result

iaai_sync_service/schema_bootstrap.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from iaai_sync_service.models import (
+    BODY_TYPE_ENUM_VALUES,
+    CAR_TABLE_NAME,
+    COUNTRY_ENUM_VALUES,
+    CURRENCY_ENUM_VALUES,
+    DRIVE_ENUM_VALUES,
+    GEARBOX_ENUM_VALUES,
+    IMAGE_TABLE_NAME,
+    ORIGIN_ENUM_VALUES,
+    SELLING_TYPE_ENUM_VALUES,
+    SYNC_RUN_TABLE_NAME,
+    STEERING_WHEEL_ENUM_VALUES,
+)
+
+SCHEMA_BOOTSTRAP_LOCK_KEY = 901_700_001
+
+
+def _create_enum_if_missing(session: Session, type_name: str, values: tuple[str, ...]) -> None:
+    values_sql = ", ".join("'" + value.replace("'", "''") + "'" for value in values)
+    session.execute(
+        text(
+            f"""
+            DO $$
+            BEGIN
+                IF NOT EXISTS (
+                    SELECT 1
+                    FROM pg_type t
+                    JOIN pg_namespace n ON n.oid = t.typnamespace
+                    WHERE n.nspname = 'public'
+                      AND t.typname = '{type_name}'
+                ) THEN
+                    CREATE TYPE public.{type_name} AS ENUM ({values_sql});
+                END IF;
+            END
+            $$;
+            """
+        )
+    )
+
+
+def _add_enum_value_if_missing(session: Session, type_name: str, value: str) -> None:
+    escaped = value.replace("'", "''")
+    session.execute(text(f"ALTER TYPE public.{type_name} ADD VALUE IF NOT EXISTS '{escaped}'"))
+
+
+def _acquire_schema_bootstrap_lock(session: Session) -> None:
+    session.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": SCHEMA_BOOTSTRAP_LOCK_KEY})
+
+
+def ensure_schema(session: Session) -> None:
+    bind = session.get_bind()
+    if bind is None or bind.dialect.name != "postgresql":
+        return
+
+    _acquire_schema_bootstrap_lock(session)
+
+    _create_enum_if_missing(session, "currencyenum", CURRENCY_ENUM_VALUES)
+    _create_enum_if_missing(session, "driveenum", DRIVE_ENUM_VALUES)
+    _create_enum_if_missing(session, "gearboxenum", GEARBOX_ENUM_VALUES)
+    _create_enum_if_missing(session, "steeringwheelenum", STEERING_WHEEL_ENUM_VALUES)
+    _create_enum_if_missing(session, "bodytypeenum", BODY_TYPE_ENUM_VALUES)
+    _create_enum_if_missing(session, "countryenum", COUNTRY_ENUM_VALUES)
+    _create_enum_if_missing(session, "originenum", ORIGIN_ENUM_VALUES)
+    _create_enum_if_missing(session, "sellingtypeenum", SELLING_TYPE_ENUM_VALUES)
+
+    for value in CURRENCY_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "currencyenum", value)
+    for value in DRIVE_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "driveenum", value)
+    for value in GEARBOX_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "gearboxenum", value)
+    for value in STEERING_WHEEL_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "steeringwheelenum", value)
+    for value in BODY_TYPE_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "bodytypeenum", value)
+    for value in COUNTRY_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "countryenum", value)
+    for value in ORIGIN_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "originenum", value)
+    for value in SELLING_TYPE_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "sellingtypeenum", value)
+
+    session.execute(
+        text(
+            f"""
+            CREATE TABLE IF NOT EXISTS public.{CAR_TABLE_NAME} (
+                id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+                parser_id VARCHAR(50) NOT NULL UNIQUE,
+                brand VARCHAR(50) NOT NULL,
+                model VARCHAR(50) NOT NULL,
+                year INTEGER NULL,
+                price BIGINT NULL,
+                currency public.currencyenum NOT NULL DEFAULT 'USD',
+                mileage INTEGER NOT NULL DEFAULT 0,
+                country public.countryenum NOT NULL DEFAULT 'NA',
+                is_sold BOOLEAN NOT NULL DEFAULT FALSE,
+                color VARCHAR NOT NULL DEFAULT 'other',
+                drive public.driveenum NULL,
+                gearbox public.gearboxenum NULL,
+                steering_wheel public.steeringwheelenum NULL,
+                body_type public.bodytypeenum NOT NULL DEFAULT 'OTHER',
+                engine_volume INTEGER NULL,
+                selling_type public.sellingtypeenum NOT NULL DEFAULT 'NA',
+                one_owner BOOLEAN NOT NULL DEFAULT FALSE,
+                new_car BOOLEAN NOT NULL DEFAULT FALSE,
+                is_hidden BOOLEAN NOT NULL DEFAULT FALSE,
+                origin public.originenum NOT NULL DEFAULT 'NA',
+                origin_url VARCHAR NOT NULL,
+                origin_id VARCHAR NOT NULL UNIQUE,
+                is_damaged BOOLEAN NOT NULL DEFAULT FALSE,
+                evaluation VARCHAR NULL,
+                non_smoking BOOLEAN NOT NULL DEFAULT TRUE,
+                rental BOOLEAN NOT NULL DEFAULT FALSE,
+                repair_history BOOLEAN NOT NULL DEFAULT FALSE,
+                slug VARCHAR NOT NULL,
+                last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+            )
+            """
+        )
+    )
+    session.execute(
+        text(
+            f"""
+            CREATE TABLE IF NOT EXISTS public.{IMAGE_TABLE_NAME} (
+                id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+                fullres_image VARCHAR NOT NULL,
+                preview_image VARCHAR NOT NULL,
+                order_index INTEGER NOT NULL,
+                car_id INTEGER NOT NULL REFERENCES public.{CAR_TABLE_NAME}(id) ON DELETE CASCADE
+            )
+            """
+        )
+    )
+    session.execute(
+        text(
+            f"""
+            CREATE TABLE IF NOT EXISTS public.{SYNC_RUN_TABLE_NAME} (
+                id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+                started_at TIMESTAMPTZ NOT NULL,
+                finished_at TIMESTAMPTZ NULL,
+                status TEXT NOT NULL,
+                lane TEXT NOT NULL,
+                ids_fetched INTEGER NOT NULL DEFAULT 0,
+                cars_upserted INTEGER NOT NULL DEFAULT 0,
+                cars_failed INTEGER NOT NULL DEFAULT 0,
+                images_upserted INTEGER NOT NULL DEFAULT 0,
+                error_summary TEXT NULL
+            )
+            """
+        )
+    )
+
+    session.execute(text(f"CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON public.{IMAGE_TABLE_NAME} (car_id)"))
+    session.execute(
+        text(f"CREATE INDEX IF NOT EXISTS ix_iaai_images_car_order ON public.{IMAGE_TABLE_NAME} (car_id, order_index)")
+    )
+    session.execute(
+        text(
+            f"CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_id ON public.{CAR_TABLE_NAME} "
+            "(last_seen_at DESC, id DESC)"
+        )
+    )
+    session.execute(
+        text(
+            f"CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_feed ON public.{CAR_TABLE_NAME} "
+            "(last_seen_at DESC, id DESC) WHERE is_sold = FALSE AND is_hidden = FALSE"
+        )
+    )
+    session.execute(
+        text(f"CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_started_at ON public.{SYNC_RUN_TABLE_NAME} (started_at)")
+    )
+    session.execute(
+        text(f"CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON public.{SYNC_RUN_TABLE_NAME} (status)")
+    )

iaai_sync_service/settings.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+
+def _read_env(name: str, default: str | None = None, *, required: bool = False) -> str:
+    value = os.getenv(name, default)
+    if required and (value is None or not value.strip()):
+        raise RuntimeError(f"Missing required environment variable: {name}")
+    if value is None:
+        return ""
+    return value.strip()
+
+
+def _read_int(name: str, default: int) -> int:
+    raw = _read_env(name, str(default))
+    try:
+        return int(raw)
+    except ValueError as exc:
+        raise RuntimeError(f"Environment variable {name} must be integer, got: {raw}") from exc
+
+
+def _read_bool(name: str, default: bool) -> bool:
+    raw = _read_env(name, "true" if default else "false")
+    normalized = raw.strip().lower()
+    if normalized in {"1", "true", "yes", "on"}:
+        return True
+    if normalized in {"0", "false", "no", "off"}:
+        return False
+    raise RuntimeError(f"Environment variable {name} must be boolean, got: {raw}")
+
+
+def _normalize_database_url(raw_url: str) -> str:
+    normalized = raw_url.strip()
+    if normalized.startswith("jdbc:"):
+        normalized = normalized[len("jdbc:") :]
+    if normalized.startswith("postgres://"):
+        normalized = "postgresql+psycopg://" + normalized[len("postgres://") :]
+    elif normalized.startswith("postgresql://"):
+        normalized = "postgresql+psycopg://" + normalized[len("postgresql://") :]
+    return normalized
+
+
+@dataclass(frozen=True)
+class Settings:
+    database_url: str
+    celery_broker_url: str
+    celery_result_backend: str
+    iaai_base_url: str
+    iaai_listing_start_url: str
+    sync_runtime_config_file: Path
+    iaai_session_cookies: str
+    iaai_storage_state_path: Path
+    iaai_login: str
+    iaai_password: str
+    http_timeout: int
+    http_retries: int
+    http_retry_backoff_ms: int
+    fetch_concurrency: int
+    db_commit_batch_size: int
+    session_keepalive_enabled: bool
+    session_keepalive_interval_minutes: int
+    schema_bootstrap_enabled: bool
+    advisory_lock_key: int
+    error_summary_max_len: int
+
+    @classmethod
+    def from_env(cls) -> "Settings":
+        settings = cls(
+            database_url=_normalize_database_url(_read_env("DATABASE_URL", required=True)),
+            celery_broker_url=_read_env("CELERY_BROKER_URL", required=True),
+            celery_result_backend=_read_env("CELERY_RESULT_BACKEND", "redis://redis:6379/1"),
+            iaai_base_url=_read_env("IAAI_BASE_URL", "https://www.iaai.com").rstrip("/"),
+            iaai_listing_start_url=_read_env("IAAI_LISTING_START_URL", ""),
+            sync_runtime_config_file=Path(_read_env("SYNC_RUNTIME_CONFIG_FILE", "sync_runtime_config.json")),
+            iaai_session_cookies=_read_env("IAAI_SESSION_COOKIES", ""),
+            iaai_storage_state_path=Path(_read_env("IAAI_STORAGE_STATE_PATH", "iaai_storage_state.json")),
+            iaai_login=_read_env("IAAI_LOGIN", ""),
+            iaai_password=_read_env("IAAI_PASSWORD", ""),
+            http_timeout=_read_int("IAAI_HTTP_TIMEOUT", 30),
+            http_retries=_read_int("IAAI_HTTP_RETRIES", 2),
+            http_retry_backoff_ms=_read_int("IAAI_HTTP_RETRY_BACKOFF_MS", 700),
+            fetch_concurrency=_read_int("IAAI_FETCH_CONCURRENCY", 8),
+            db_commit_batch_size=_read_int("IAAI_DB_COMMIT_BATCH_SIZE", 20),
+            session_keepalive_enabled=_read_bool("IAAI_SESSION_KEEPALIVE_ENABLED", True),
+            session_keepalive_interval_minutes=_read_int("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES", 15),
+            schema_bootstrap_enabled=_read_bool("IAAI_SCHEMA_BOOTSTRAP_ENABLED", False),
+            advisory_lock_key=_read_int("SYNC_ADVISORY_LOCK_KEY", 7642200),
+            error_summary_max_len=_read_int("SYNC_ERROR_SUMMARY_MAX_LEN", 2000),
+        )
+
+        if settings.http_timeout <= 0:
+            raise RuntimeError("IAAI_HTTP_TIMEOUT must be greater than 0")
+        if settings.http_retries < 0:
+            raise RuntimeError("IAAI_HTTP_RETRIES cannot be negative")
+        if settings.http_retry_backoff_ms < 0:
+            raise RuntimeError("IAAI_HTTP_RETRY_BACKOFF_MS cannot be negative")
+        if settings.fetch_concurrency <= 0:
+            raise RuntimeError("IAAI_FETCH_CONCURRENCY must be greater than 0")
+        if settings.db_commit_batch_size <= 0:
+            raise RuntimeError("IAAI_DB_COMMIT_BATCH_SIZE must be greater than 0")
+        if settings.session_keepalive_interval_minutes <= 0:
+            raise RuntimeError("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES must be greater than 0")
+        if settings.session_keepalive_interval_minutes > 59:
+            raise RuntimeError("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES must be <= 59")
+        if settings.error_summary_max_len <= 0:
+            raise RuntimeError("SYNC_ERROR_SUMMARY_MAX_LEN must be greater than 0")
+        if settings.iaai_base_url.lower().startswith("http") is False:
+            raise RuntimeError("IAAI_BASE_URL must be absolute URL")
+        if settings.iaai_listing_start_url and not (
+            settings.iaai_listing_start_url.lower().startswith("http")
+            or settings.iaai_listing_start_url.startswith("/")
+        ):
+            raise RuntimeError("IAAI_LISTING_START_URL must be absolute URL or path starting with '/'")
+        return settings
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    return Settings.from_env()

iaai_sync_service/tasks.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from celery.utils.log import get_task_logger
+from sqlalchemy import text
+from sqlalchemy.exc import ProgrammingError
+
+from iaai_sync_service.celery_app import app
+from iaai_sync_service.client import IAAIClient
+from iaai_sync_service.database import get_session
+from iaai_sync_service.models import SYNC_RUN_TABLE_NAME, SyncRun
+from iaai_sync_service.runtime_config import load_runtime_config
+from iaai_sync_service.schema_bootstrap import ensure_schema
+from iaai_sync_service.settings import get_settings
+from iaai_sync_service.sync_engine import run_sync_once
+
+logger = get_task_logger(__name__)
+
+SYNC_LANE = "iaai_cars"
+
+
+def utc_now() -> datetime:
+    return datetime.now(UTC)
+
+
+def trim_error_summary(errors: list[str], max_len: int) -> str | None:
+    if not errors:
+        return None
+    joined = " | ".join(errors)
+    if len(joined) <= max_len:
+        return joined
+    return joined[: max_len - 3] + "..."
+
+
+def _is_missing_sync_runs_error(exc: ProgrammingError) -> bool:
+    message = str(exc).lower()
+    return "does not exist" in message and SYNC_RUN_TABLE_NAME in message
+
+
+def _create_sync_run(session, lane: str) -> SyncRun:  # noqa: ANN001
+    run = SyncRun(
+        status="running",
+        lane=lane,
+        started_at=utc_now(),
+        ids_fetched=0,
+        cars_upserted=0,
+        cars_failed=0,
+        images_upserted=0,
+    )
+    session.add(run)
+    session.commit()
+    return run
+
+
+@app.task(name="iaai.sync_cars_feed", queue="iaai_sync", bind=True)
+def sync_cars_feed(self: Any) -> dict[str, Any]:
+    return execute_sync_job()
+
+
+@app.task(name="iaai.keep_session_alive", queue="iaai_sync", bind=True)
+def keep_session_alive(self: Any) -> dict[str, Any]:
+    return execute_keepalive_job()
+
+
+def execute_sync_job() -> dict[str, Any]:
+    settings = get_settings()
+    session = get_session()
+    run_id: int | None = None
+    lock_acquired = False
+
+    try:
+        logger.info("IAAI sync job started base_url=%s", settings.iaai_base_url)
+        if settings.schema_bootstrap_enabled:
+            ensure_schema(session)
+            session.commit()
+            logger.info("Schema bootstrap is enabled and completed.")
+        else:
+            logger.info("Schema bootstrap is disabled.")
+
+        lock_acquired = bool(
+            session.execute(
+                text("SELECT pg_try_advisory_lock(:key)"),
+                {"key": settings.advisory_lock_key},
+            ).scalar_one()
+        )
+        if not lock_acquired:
+            logger.info("IAAI sync skipped: advisory lock is already held.")
+            return {"status": "skipped_locked"}
+
+        try:
+            run = _create_sync_run(session=session, lane=SYNC_LANE)
+        except ProgrammingError as exc:
+            session.rollback()
+            if not _is_missing_sync_runs_error(exc):
+                raise
+            logger.warning(
+                "Table '%s' is missing. Running one-time schema bootstrap and retrying.",
+                SYNC_RUN_TABLE_NAME,
+            )
+            ensure_schema(session)
+            session.commit()
+            run = _create_sync_run(session=session, lane=SYNC_LANE)
+        run_id = run.id
+        logger.info("IAAI sync run created run_id=%s", run_id)
+
+        client = IAAIClient(settings=settings)
+        runtime_config = load_runtime_config(settings.sync_runtime_config_file, logger=logger)
+        logger.info("Loaded runtime config from '%s'", settings.sync_runtime_config_file)
+
+        stats, errors = run_sync_once(
+            session=session,
+            client=client,
+            settings=settings,
+            runtime_config=runtime_config,
+        )
+
+        run = session.get(SyncRun, run_id)
+        if run is None:
+            raise RuntimeError(f"sync_runs row disappeared for id={run_id}")
+        run.finished_at = utc_now()
+        run.ids_fetched = stats.ids_fetched
+        run.cars_upserted = stats.cars_upserted
+        run.cars_failed = stats.cars_failed
+        run.images_upserted = stats.images_upserted
+        run.status = "partial" if stats.cars_failed > 0 else "success"
+        run.error_summary = trim_error_summary(errors, settings.error_summary_max_len)
+        session.commit()
+
+        logger.info(
+            "IAAI sync completed status=%s ids=%s cars_upserted=%s cars_failed=%s images_upserted=%s",
+            run.status,
+            run.ids_fetched,
+            run.cars_upserted,
+            run.cars_failed,
+            run.images_upserted,
+        )
+        return {
+            "status": run.status,
+            "ids_fetched": run.ids_fetched,
+            "cars_upserted": run.cars_upserted,
+            "cars_failed": run.cars_failed,
+            "images_upserted": run.images_upserted,
+            "run_id": run.id,
+        }
+    except Exception as exc:  # noqa: BLE001
+        session.rollback()
+        if run_id is not None:
+            failed = session.get(SyncRun, run_id)
+            if failed is not None:
+                failed.finished_at = utc_now()
+                failed.status = "failed"
+                failed.error_summary = trim_error_summary([str(exc)], settings.error_summary_max_len)
+                session.commit()
+        logger.exception("IAAI sync failed: %s", exc)
+        raise
+    finally:
+        if lock_acquired:
+            try:
+                session.execute(
+                    text("SELECT pg_advisory_unlock(:key)"),
+                    {"key": settings.advisory_lock_key},
+                )
+                session.commit()
+            except Exception:  # noqa: BLE001
+                session.rollback()
+                logger.exception("Failed to release advisory lock.")
+        session.close()
+
+
+def execute_keepalive_job() -> dict[str, Any]:
+    settings = get_settings()
+    session = get_session()
+    lock_acquired = False
+
+    try:
+        lock_acquired = bool(
+            session.execute(
+                text("SELECT pg_try_advisory_lock(:key)"),
+                {"key": settings.advisory_lock_key},
+            ).scalar_one()
+        )
+        if not lock_acquired:
+            logger.info("IAAI keepalive skipped: advisory lock is already held.")
+            return {"status": "skipped_locked"}
+
+        logger.info("IAAI keepalive job started base_url=%s", settings.iaai_base_url)
+        client = IAAIClient(settings=settings)
+        result = client.keep_session_alive()
+        logger.info(
+            "IAAI keepalive completed scope=%s marker_present=%s result_count=%s elapsed_ms=%s",
+            result.get("scope"),
+            result.get("marker_present"),
+            result.get("result_count"),
+            result.get("elapsed_ms"),
+        )
+        return {"status": "success", **result}
+    except Exception as exc:  # noqa: BLE001
+        logger.exception("IAAI keepalive failed: %s", exc)
+        raise
+    finally:
+        if lock_acquired:
+            try:
+                session.execute(
+                    text("SELECT pg_advisory_unlock(:key)"),
+                    {"key": settings.advisory_lock_key},
+                )
+                session.commit()
+            except Exception:  # noqa: BLE001
+                session.rollback()
+                logger.exception("Failed to release advisory lock for keepalive.")
+        session.close()