Fast IAAI implementation

.env.example

@@ -0,0 +1,21 @@
+POSTGRES_HOST=postgres
+POSTGRES_PORT=5432
+POSTGRES_USER=postgres
+POSTGRES_PASSWORD=postgres
+POSTGRES_DB=iaai
+IAAI_DATABASE_URL=postgresql+psycopg://postgres:postgres@postgres:5432/iaai
+CELERY_BROKER_URL=redis://redis:6379/0
+IAAI_BASE_URL=https://www.iaai.com
+IAAI_LISTING_START_URL=https://www.iaai.com/Search?url=B%2bU066dM8%2flZtRvnzTwIEi8Pib9%2fM2fLpCTQDIgQRm4%3d
+IAAI_STORAGE_STATE_PATH=/app/iaai_storage_state.json
+IAAI_HTTP_TIMEOUT=30
+IAAI_HTTP_RETRIES=2
+IAAI_HTTP_RETRY_BACKOFF_MS=700
+IAAI_FETCH_CONCURRENCY=24
+IAAI_DB_COMMIT_BATCH_SIZE=200
+IAAI_SESSION_KEEPALIVE_ENABLED=true
+IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES=15
+IAAI_SCHEMA_BOOTSTRAP_ENABLED=true
+SYNC_RUNTIME_CONFIG_FILE=/app/sync_runtime_config.json
+SYNC_ADVISORY_LOCK_KEY=7642200
+SYNC_ERROR_SUMMARY_MAX_LEN=2000

.gitignore

@@ -0,0 +1,219 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[codz]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#   Usually these files are written by a python script from a template
+#   before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py.cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+# Pipfile.lock
+
+# UV
+#   Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+# uv.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+# poetry.lock
+# poetry.toml
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#   pdm recommends including project-wide configuration in pdm.toml, but excluding .pdm-python.
+#   https://pdm-project.org/en/latest/usage/project/#working-with-version-control
+# pdm.lock
+# pdm.toml
+.pdm-python
+.pdm-build/
+
+# pixi
+#   Similar to Pipfile.lock, it is generally recommended to include pixi.lock in version control.
+# pixi.lock
+#   Pixi creates a virtual environment in the .pixi directory, just like venv module creates one
+#   in the .venv directory. It is recommended not to include this directory in version control.
+.pixi
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# Redis
+*.rdb
+*.aof
+*.pid
+
+# RabbitMQ
+mnesia/
+rabbitmq/
+rabbitmq-data/
+
+# ActiveMQ
+activemq-data/
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.envrc
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#   JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#   be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#   and can be added to the global gitignore or merged into this file.  For a more nuclear
+#   option (not recommended) you can uncomment the following to ignore the entire idea folder.
+# .idea/
+
+# Abstra
+#   Abstra is an AI-powered process automation framework.
+#   Ignore directories containing user credentials, local state, and settings.
+#   Learn more at https://abstra.io/docs
+.abstra/
+
+# Visual Studio Code
+#   Visual Studio Code specific template is maintained in a separate VisualStudioCode.gitignore 
+#   that can be found at https://github.com/github/gitignore/blob/main/Global/VisualStudioCode.gitignore
+#   and can be added to the global gitignore or merged into this file. However, if you prefer, 
+#   you could uncomment the following to ignore the entire vscode folder
+# .vscode/
+
+# Ruff stuff:
+.ruff_cache/
+
+# PyPI configuration file
+.pypirc
+
+# Marimo
+marimo/_static/
+marimo/_lsp/
+__marimo__/
+
+# Streamlit
+.streamlit/secrets.toml
+iaai_storage_state.json
+html/catalog.html
+html/page.html

Dockerfile

@@ -0,0 +1,19 @@
+FROM python:3.13-slim
+
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV PYTHONUNBUFFERED=1
+
+WORKDIR /app
+
+RUN pip install --no-cache-dir --upgrade pip
+
+COPY pyproject.toml /app/
+COPY iaai_sync_service /app/iaai_sync_service
+COPY alembic.ini /app/alembic.ini
+COPY alembic /app/alembic
+COPY sync_runtime_config.json /app/sync_runtime_config.json
+
+RUN pip install --no-cache-dir . \
+    && playwright install --with-deps chromium
+
+CMD ["python", "-m", "iaai_sync_service.cli"]

alembic.ini

@@ -0,0 +1,38 @@
+[alembic]
+script_location = alembic
+prepend_sys_path = .
+version_path_separator = os
+sqlalchemy.url = postgresql+psycopg://postgres:postgres@localhost:5432/iaai
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s

alembic/env.py

@@ -0,0 +1,85 @@
+from __future__ import annotations
+
+import os
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool, text
+
+from iaai_sync_service.models import Base
+
+config = context.config
+
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+target_metadata = Base.metadata
+SYNC_VERSION_TABLE = "iaai_sync_alembic_version"
+LEGACY_VERSION_TABLE = "alembic_version"
+SYNC_REVISIONS = {
+    "0001_ensure_sync_schema",
+    "0002_extend_origin_enum_for_iaai",
+    "0003_create_iaai_source_tables",
+}
+
+
+def _normalize_database_url(url: str) -> str:
+    normalized = url.strip()
+    if normalized.startswith("jdbc:"):
+        normalized = normalized[len("jdbc:") :]
+    if normalized.startswith("postgres://"):
+        normalized = "postgresql+psycopg://" + normalized[len("postgres://") :]
+    elif normalized.startswith("postgresql://"):
+        normalized = "postgresql+psycopg://" + normalized[len("postgresql://") :]
+    return normalized
+
+
+def _database_url() -> str:
+    raw_url = os.getenv("DATABASE_URL", config.get_main_option("sqlalchemy.url"))
+    return _normalize_database_url(raw_url)
+
+
+def _select_version_table(connection) -> str:  # noqa: ANN001
+    inspector = connection.dialect
+    if inspector.has_table(connection, SYNC_VERSION_TABLE):
+        return SYNC_VERSION_TABLE
+    if inspector.has_table(connection, LEGACY_VERSION_TABLE):
+        version_num = connection.execute(text(f"SELECT version_num FROM {LEGACY_VERSION_TABLE} LIMIT 1")).scalar()
+        if version_num in SYNC_REVISIONS:
+            return LEGACY_VERSION_TABLE
+    return SYNC_VERSION_TABLE
+
+
+def run_migrations_offline() -> None:
+    url = _database_url()
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        version_table=SYNC_VERSION_TABLE,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    configuration = config.get_section(config.config_ini_section) or {}
+    configuration["sqlalchemy.url"] = _database_url()
+    connectable = engine_from_config(configuration, prefix="sqlalchemy.", poolclass=pool.NullPool)
+
+    with connectable.connect() as connection:
+        version_table = _select_version_table(connection)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table=version_table,
+        )
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,27 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, Sequence[str], None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/0001_ensure_sync_schema.py

@@ -0,0 +1,165 @@
+"""bootstrap standalone iaai schema
+
+Revision ID: 0001_ensure_sync_schema
+Revises:
+Create Date: 2026-04-08 00:00:00
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "0001_ensure_sync_schema"
+down_revision: Union[str, Sequence[str], None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _create_enum_if_missing(type_name: str, values: tuple[str, ...]) -> None:
+    values_sql = ", ".join("'" + value.replace("'", "''") + "'" for value in values)
+    op.execute(
+        f"""
+        DO $$
+        BEGIN
+            IF NOT EXISTS (
+                SELECT 1
+                FROM pg_type t
+                JOIN pg_namespace n ON n.oid = t.typnamespace
+                WHERE t.typname = '{type_name}'
+                  AND n.nspname = 'public'
+            ) THEN
+                CREATE TYPE public.{type_name} AS ENUM ({values_sql});
+            END IF;
+        END
+        $$;
+        """
+    )
+
+
+def upgrade() -> None:
+    _create_enum_if_missing("currencyenum", ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD"))
+    _create_enum_if_missing("driveenum", ("FWD", "RWD", "TWO_WD", "FOUR_WD", "2WD", "4WD", "NA"))
+    _create_enum_if_missing("gearboxenum", ("AT", "CVT", "MT", "EV", "NA"))
+    _create_enum_if_missing("steeringwheelenum", ("LEFT", "RIGHT", "left", "right", "NA"))
+    _create_enum_if_missing(
+        "bodytypeenum",
+        (
+            "COUPE",
+            "SUV",
+            "HATCHBACK",
+            "MINIVAN",
+            "SEDAN",
+            "NA",
+            "Station Wagon",
+            "Pickup",
+            "Truck",
+            "Open",
+            "RV",
+            "Other",
+            "STATION_WAGON",
+            "PICKUP",
+            "TRUCK",
+            "OPEN",
+            "OTHER",
+        ),
+    )
+    _create_enum_if_missing("countryenum", ("JP", "KR", "US", "CA", "NA"))
+    _create_enum_if_missing(
+        "originenum",
+        (
+            "TAU",
+            "CARSENSOR",
+            "HANAMARU",
+            "ENCAR",
+            "KURUMA_TRADER",
+            "carsensor",
+            "encar",
+            "kuruma_trader",
+            "asnet",
+            "kababa",
+            "ACV",
+            "COPART",
+            "copart",
+            "NA",
+            "ASNET",
+            "KABABA",
+        ),
+    )
+    _create_enum_if_missing("sellingtypeenum", ("STOCK", "AUCTION", "TENDER", "stock", "auction", "tender", "NA"))
+
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS public.cars (
+            id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+            parser_id VARCHAR(50) NOT NULL UNIQUE,
+            brand VARCHAR(50) NOT NULL,
+            model VARCHAR(50) NOT NULL,
+            year INTEGER NULL,
+            price BIGINT NULL,
+            currency public.currencyenum NOT NULL DEFAULT 'USD',
+            mileage INTEGER NOT NULL DEFAULT 0,
+            country public.countryenum NOT NULL DEFAULT 'NA',
+            is_sold BOOLEAN NOT NULL DEFAULT FALSE,
+            color VARCHAR NOT NULL DEFAULT 'other',
+            drive public.driveenum NULL,
+            gearbox public.gearboxenum NULL,
+            steering_wheel public.steeringwheelenum NULL,
+            body_type public.bodytypeenum NOT NULL DEFAULT 'OTHER',
+            engine_volume INTEGER NULL,
+            selling_type public.sellingtypeenum NOT NULL DEFAULT 'NA',
+            one_owner BOOLEAN NOT NULL DEFAULT FALSE,
+            new_car BOOLEAN NOT NULL DEFAULT FALSE,
+            is_hidden BOOLEAN NOT NULL DEFAULT FALSE,
+            origin public.originenum NOT NULL DEFAULT 'NA',
+            origin_url VARCHAR NOT NULL,
+            origin_id VARCHAR NOT NULL UNIQUE,
+            is_damaged BOOLEAN NOT NULL DEFAULT FALSE,
+            evaluation VARCHAR NULL,
+            non_smoking BOOLEAN NOT NULL DEFAULT TRUE,
+            rental BOOLEAN NOT NULL DEFAULT FALSE,
+            repair_history BOOLEAN NOT NULL DEFAULT FALSE,
+            slug VARCHAR NOT NULL,
+            last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+        )
+        """
+    )
+
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS public.images (
+            id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+            fullres_image VARCHAR NOT NULL,
+            preview_image VARCHAR NOT NULL,
+            order_index INTEGER NOT NULL,
+            car_id INTEGER NOT NULL REFERENCES public.cars(id) ON DELETE CASCADE
+        )
+        """
+    )
+
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS public.sync_runs (
+            id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+            started_at TIMESTAMPTZ NOT NULL,
+            finished_at TIMESTAMPTZ NULL,
+            status TEXT NOT NULL,
+            lane TEXT NOT NULL,
+            ids_fetched INTEGER NOT NULL DEFAULT 0,
+            cars_upserted INTEGER NOT NULL DEFAULT 0,
+            cars_failed INTEGER NOT NULL DEFAULT 0,
+            images_upserted INTEGER NOT NULL DEFAULT 0,
+            error_summary TEXT NULL
+        )
+        """
+    )
+
+    op.execute("CREATE INDEX IF NOT EXISTS ix_images_car_id ON public.images (car_id)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_sync_runs_started_at ON public.sync_runs (started_at)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_sync_runs_status ON public.sync_runs (status)")
+
+
+def downgrade() -> None:
+    # Compatibility-only migration for shared production databases.
+    pass

alembic/versions/0002_extend_origin_enum_for_iaai.py

@@ -0,0 +1,86 @@
+"""extend origin enum for iaai
+
+Revision ID: 0002_extend_origin_enum_for_iaai
+Revises: 0001_ensure_sync_schema
+Create Date: 2026-04-08 00:00:00
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision: str = "0002_extend_origin_enum_for_iaai"
+down_revision: Union[str, Sequence[str], None] = "0001_ensure_sync_schema"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _enum_type_exists(type_name: str) -> bool:
+    bind = op.get_bind()
+    if bind.dialect.name != "postgresql":
+        return False
+    exists = bind.execute(
+        sa.text(
+            """
+            SELECT 1
+            FROM pg_type t
+            JOIN pg_namespace n ON n.oid = t.typnamespace
+            WHERE n.nspname = 'public'
+              AND t.typname = :type_name
+              AND t.typtype = 'e'
+            LIMIT 1
+            """
+        ),
+        {"type_name": type_name},
+    ).scalar_one_or_none()
+    return bool(exists)
+
+
+def _enum_type_for_column(table_name: str, column_name: str) -> str | None:
+    bind = op.get_bind()
+    if bind.dialect.name != "postgresql":
+        return None
+    return bind.execute(
+        sa.text(
+            """
+            SELECT t.typname
+            FROM pg_attribute a
+            JOIN pg_class c ON c.oid = a.attrelid
+            JOIN pg_namespace n ON n.oid = c.relnamespace
+            JOIN pg_type t ON t.oid = a.atttypid
+            WHERE n.nspname = 'public'
+              AND c.relname = :table_name
+              AND a.attname = :column_name
+              AND a.attnum > 0
+              AND NOT a.attisdropped
+              AND t.typtype = 'e'
+            LIMIT 1
+            """
+        ),
+        {"table_name": table_name, "column_name": column_name},
+    ).scalar_one_or_none()
+
+
+def _add_enum_value_if_missing(type_name: str | None, value: str) -> None:
+    if not type_name:
+        return
+    if not _enum_type_exists(type_name):
+        return
+    bind = op.get_bind()
+    if bind.dialect.name != "postgresql":
+        return
+    escaped = value.replace("'", "''")
+    op.execute(f"ALTER TYPE public.{type_name} ADD VALUE IF NOT EXISTS '{escaped}'")
+
+
+def upgrade() -> None:
+    origin_type = _enum_type_for_column("cars", "origin") or "originenum"
+    _add_enum_value_if_missing(origin_type, "IAAI")
+
+
+def downgrade() -> None:
+    # Enum value removal is intentionally skipped for compatibility.
+    pass

alembic/versions/0003_create_iaai_source_tables.py

@@ -0,0 +1,100 @@
+"""create iaai-specific storage tables
+
+Revision ID: 0003_create_iaai_source_tables
+Revises: 0002_extend_origin_enum_for_iaai
+Create Date: 2026-04-09 00:00:00
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+
+
+# revision identifiers, used by Alembic.
+revision: str = "0003_create_iaai_source_tables"
+down_revision: Union[str, Sequence[str], None] = "0002_extend_origin_enum_for_iaai"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS public.iaai_cars (
+            id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+            parser_id VARCHAR(50) NOT NULL UNIQUE,
+            brand VARCHAR(50) NOT NULL,
+            model VARCHAR(50) NOT NULL,
+            year INTEGER NULL,
+            price BIGINT NULL,
+            currency public.currencyenum NOT NULL DEFAULT 'USD',
+            mileage INTEGER NOT NULL DEFAULT 0,
+            country public.countryenum NOT NULL DEFAULT 'NA',
+            is_sold BOOLEAN NOT NULL DEFAULT FALSE,
+            color VARCHAR NOT NULL DEFAULT 'other',
+            drive public.driveenum NULL,
+            gearbox public.gearboxenum NULL,
+            steering_wheel public.steeringwheelenum NULL,
+            body_type public.bodytypeenum NOT NULL DEFAULT 'OTHER',
+            engine_volume INTEGER NULL,
+            selling_type public.sellingtypeenum NOT NULL DEFAULT 'NA',
+            one_owner BOOLEAN NOT NULL DEFAULT FALSE,
+            new_car BOOLEAN NOT NULL DEFAULT FALSE,
+            is_hidden BOOLEAN NOT NULL DEFAULT FALSE,
+            origin public.originenum NOT NULL DEFAULT 'NA',
+            origin_url VARCHAR NOT NULL,
+            origin_id VARCHAR NOT NULL UNIQUE,
+            is_damaged BOOLEAN NOT NULL DEFAULT FALSE,
+            evaluation VARCHAR NULL,
+            non_smoking BOOLEAN NOT NULL DEFAULT TRUE,
+            rental BOOLEAN NOT NULL DEFAULT FALSE,
+            repair_history BOOLEAN NOT NULL DEFAULT FALSE,
+            slug VARCHAR NOT NULL,
+            last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+        )
+        """
+    )
+
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS public.iaai_images (
+            id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+            fullres_image VARCHAR NOT NULL,
+            preview_image VARCHAR NOT NULL,
+            order_index INTEGER NOT NULL,
+            car_id INTEGER NOT NULL REFERENCES public.iaai_cars(id) ON DELETE CASCADE
+        )
+        """
+    )
+
+    op.execute(
+        """
+        CREATE TABLE IF NOT EXISTS public.iaai_sync_runs (
+            id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+            started_at TIMESTAMPTZ NOT NULL,
+            finished_at TIMESTAMPTZ NULL,
+            status TEXT NOT NULL,
+            lane TEXT NOT NULL,
+            ids_fetched INTEGER NOT NULL DEFAULT 0,
+            cars_upserted INTEGER NOT NULL DEFAULT 0,
+            cars_failed INTEGER NOT NULL DEFAULT 0,
+            images_upserted INTEGER NOT NULL DEFAULT 0,
+            error_summary TEXT NULL
+        )
+        """
+    )
+
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON public.iaai_images (car_id)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_order ON public.iaai_images (car_id, order_index)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_id ON public.iaai_cars (last_seen_at DESC, id DESC)")
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_feed ON public.iaai_cars "
+        "(last_seen_at DESC, id DESC) WHERE is_sold = FALSE AND is_hidden = FALSE"
+    )
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_started_at ON public.iaai_sync_runs (started_at)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON public.iaai_sync_runs (status)")
+
+
+def downgrade() -> None:
+    # Compatibility-only migration for shared production databases.
+    pass

docker-compose.yml

@@ -0,0 +1,105 @@
+x-app-env: &app-env
+  POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
+  POSTGRES_PORT: ${POSTGRES_PORT:-5432}
+  POSTGRES_USER: ${POSTGRES_USER:-postgres}
+  POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+  POSTGRES_DB: ${POSTGRES_DB:-iaai}
+  DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-iaai}}
+  CELERY_BROKER_URL: redis://redis:6379/0
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/1}
+  IAAI_BASE_URL: ${IAAI_BASE_URL:-https://www.iaai.com}
+  IAAI_LISTING_START_URL: ${IAAI_LISTING_START_URL:-}
+  IAAI_SESSION_COOKIES: ${IAAI_SESSION_COOKIES:-}
+  IAAI_STORAGE_STATE_PATH: /app/iaai_storage_state.json
+  IAAI_LOGIN: ${IAAI_LOGIN:-}
+  IAAI_PASSWORD: ${IAAI_PASSWORD:-}
+  IAAI_HTTP_TIMEOUT: ${IAAI_HTTP_TIMEOUT:-30}
+  IAAI_HTTP_RETRIES: ${IAAI_HTTP_RETRIES:-2}
+  IAAI_HTTP_RETRY_BACKOFF_MS: ${IAAI_HTTP_RETRY_BACKOFF_MS:-700}
+  IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-8}
+  IAAI_DB_COMMIT_BATCH_SIZE: ${IAAI_DB_COMMIT_BATCH_SIZE:-20}
+  IAAI_SESSION_KEEPALIVE_ENABLED: ${IAAI_SESSION_KEEPALIVE_ENABLED:-true}
+  IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES: ${IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES:-15}
+  IAAI_SCHEMA_BOOTSTRAP_ENABLED: ${IAAI_SCHEMA_BOOTSTRAP_ENABLED:-true}
+  SYNC_RUNTIME_CONFIG_FILE: /app/sync_runtime_config.json
+
+services:
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-iaai}
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+      PGDATA: /var/lib/postgresql/data/pgdata
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-iaai}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+    ports:
+      - "6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  migrate:
+    build: .
+    environment: *app-env
+    command: ["alembic", "upgrade", "head"]
+    volumes:
+      - ./sync_runtime_config.json:/app/sync_runtime_config.json:ro
+      - ./iaai_storage_state.json:/app/iaai_storage_state.json
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  worker:
+    build: .
+    environment: *app-env
+    command:
+      [
+        "celery",
+        "-A",
+        "iaai_sync_service.celery_app:app",
+        "worker",
+        "--queues=iaai_sync",
+        "--concurrency=1",
+        "--loglevel=INFO"
+      ]
+    volumes:
+      - ./sync_runtime_config.json:/app/sync_runtime_config.json:ro
+      - ./iaai_storage_state.json:/app/iaai_storage_state.json
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+  beat:
+    build: .
+    environment: *app-env
+    command: ["celery", "-A", "iaai_sync_service.celery_app:app", "beat", "--loglevel=INFO"]
+    volumes:
+      - ./sync_runtime_config.json:/app/sync_runtime_config.json:ro
+      - ./iaai_storage_state.json:/app/iaai_storage_state.json
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+volumes:
+  postgres_data:

iaai_sync_service/__init__.py

@@ -0,0 +1 @@
+"""IAAI Cars sync service package."""

iaai_sync_service/__main__.py

@@ -0,0 +1,4 @@
+from iaai_sync_service.cli import main
+
+if __name__ == "__main__":
+    main()

iaai_sync_service/celery_app.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from celery import Celery
+from celery.schedules import crontab
+
+from iaai_sync_service.settings import get_settings
+
+settings = get_settings()
+
+app = Celery("iaai_sync_service")
+app.conf.broker_url = settings.celery_broker_url
+app.conf.result_backend = settings.celery_result_backend
+app.conf.task_default_queue = "iaai_sync"
+app.conf.timezone = "UTC"
+app.conf.enable_utc = True
+app.conf.broker_connection_retry = True
+app.conf.broker_connection_retry_on_startup = True
+app.conf.broker_connection_max_retries = None
+
+def build_beat_schedule() -> dict[str, dict]:
+    schedule: dict[str, dict] = {
+        "sync-iaai-cars-every-hour": {
+            "task": "iaai.sync_cars_feed",
+            "schedule": crontab(minute=0),
+            "options": {"queue": "iaai_sync"},
+        }
+    }
+    if settings.session_keepalive_enabled:
+        schedule["keep-iaai-session-alive"] = {
+            "task": "iaai.keep_session_alive",
+            "schedule": crontab(minute=f"*/{settings.session_keepalive_interval_minutes}"),
+            "options": {"queue": "iaai_sync"},
+        }
+    return schedule
+
+
+app.conf.beat_schedule = build_beat_schedule()
+app.conf.imports = ("iaai_sync_service.tasks",)

iaai_sync_service/cli.py

@@ -0,0 +1,14 @@
+from __future__ import annotations
+
+import json
+
+from iaai_sync_service.tasks import execute_sync_job
+
+
+def main() -> None:
+    result = execute_sync_job()
+    print(json.dumps(result, ensure_ascii=False, indent=2))
+
+
+if __name__ == "__main__":
+    main()

iaai_sync_service/client.py

@@ -0,0 +1,950 @@
+from __future__ import annotations
+
+import html
+import json
+import logging
+import math
+import re
+import threading
+import time
+from collections.abc import Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+from urllib.parse import quote, urljoin
+
+import requests
+from requests.adapters import HTTPAdapter
+
+from iaai_sync_service.runtime_config import RuntimeFilters
+from iaai_sync_service.settings import Settings
+
+logger = logging.getLogger(__name__)
+
+TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
+CHALLENGE_MARKERS = (
+    "_incapsula_resource",
+    "incapsula",
+    "incident id",
+    "request unsuccessful",
+    "access denied",
+)
+COOKIE_ACCEPT_SELECTORS = (
+    "button:has-text('Accept All')",
+    "button:has-text('Accept all')",
+    "button:has-text('I Agree')",
+    "button:has-text('Agree')",
+    "button:has-text('Only necessary')",
+    "button:has-text('Только необходимые')",
+    "button:has-text('Принять все')",
+    "[id*='accept']",
+    "[class*='accept']",
+)
+LISTING_MARKER = 'id="GBPSearchQuery"'
+DETAIL_MARKER = 'id="ProductDetailsVM"'
+RESIZER_URL = "https://vis.iaai.com/resizer"
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/124.0.0.0 Safari/537.36"
+)
+MAX_CHALLENGE_REFRESH_ATTEMPTS = 3
+PLAYWRIGHT_REFRESH_POLLS = 8
+
+
+@dataclass(frozen=True)
+class ListingVehicle:
+    inventory_id: str
+    tenant: str | None
+    auction_id: str | None
+    auction_date: str | None
+    inventory_status: str | None
+    currency: str | None
+    timed_auction_closed: bool
+    timed_auction_indicator: bool
+    prebid_indicator: bool
+    buynow_indicator: bool
+
+
+@dataclass(frozen=True)
+class ListingPage:
+    vehicles: list[ListingVehicle]
+    result_count: int
+    page_size: int
+    current_page: int
+    gbp_search_query: dict[str, Any]
+
+
+class HybridSessionAuth:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._thread_local = threading.local()
+        self._lock = threading.Lock()
+        self._refresh_lock = threading.Lock()
+        self._bootstrap_cookies_loaded = False
+        self._anonymous_bootstrap_attempted = False
+        self._refresh_generation = 0
+        self._latest_refresh_cookies: list[dict[str, Any]] = []
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        timeout: int,
+        retries: int,
+        retry_backoff_ms: int,
+        headers: dict[str, str] | None = None,
+        data: Any | None = None,
+        json_body: Any | None = None,
+        expected_marker: str | None = None,
+    ) -> requests.Response:
+        session = self._get_session()
+        self._ensure_anonymous_session_bootstrap(session=session)
+        self._sync_session_with_latest_refresh(session)
+        last_error: Exception | None = None
+        refresh_attempts = 0
+        attempt = 0
+        while attempt <= retries:
+            try:
+                response = session.request(
+                    method=method,
+                    url=url,
+                    headers=headers,
+                    data=data,
+                    json=json_body,
+                    timeout=timeout,
+                )
+            except requests.RequestException as exc:
+                last_error = exc
+                if attempt >= retries:
+                    break
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries:
+                response.close()
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if is_challenge_response(
+                status_code=response.status_code,
+                body_text=response.text,
+                expected_marker=expected_marker,
+            ):
+                response.close()
+                if refresh_attempts >= MAX_CHALLENGE_REFRESH_ATTEMPTS:
+                    raise RuntimeError(
+                        "IAAI challenge persisted after "
+                        f"{refresh_attempts} Playwright refresh attempts for url={url}"
+                    )
+                refresh_attempts += 1
+                logger.warning(
+                    "Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s",
+                    url,
+                    response.status_code,
+                    expected_marker,
+                    refresh_attempts,
+                    MAX_CHALLENGE_REFRESH_ATTEMPTS,
+                )
+                self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+                if refresh_attempts > 1:
+                    self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
+                continue
+
+            return response
+
+        if last_error is not None:
+            raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error
+        raise RuntimeError(f"Request failed url={url} after retries")
+
+    def persist_storage_state(self) -> None:
+        session = self._get_session()
+        with self._lock:
+            self._save_storage_state(session)
+
+    def _get_session(self) -> requests.Session:
+        session = getattr(self._thread_local, "session", None)
+        if session is None:
+            session = requests.Session()
+            adapter = HTTPAdapter(pool_connections=20, pool_maxsize=20)
+            session.mount("http://", adapter)
+            session.mount("https://", adapter)
+            session.headers.update(
+                {
+                    "user-agent": DEFAULT_USER_AGENT,
+                    "accept-language": "en-US,en;q=0.9",
+                    "cache-control": "no-cache",
+                    "pragma": "no-cache",
+                }
+            )
+            with self._lock:
+                self._bootstrap_session_cookies(session)
+            self._thread_local.session = session
+            self._thread_local.session_generation = 0
+            self._sync_session_with_latest_refresh(session)
+        return session
+
+    def _sync_session_with_latest_refresh(self, session: requests.Session) -> None:
+        with self._lock:
+            latest_generation = self._refresh_generation
+            session_generation = getattr(self._thread_local, "session_generation", 0)
+            if latest_generation <= session_generation or not self._latest_refresh_cookies:
+                return
+            cookies = list(self._latest_refresh_cookies)
+        self._apply_cookies_to_session(session, cookies)
+        self._thread_local.session_generation = latest_generation
+
+    def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None:
+        if self._anonymous_bootstrap_attempted:
+            return
+        if self._session_has_iaai_cookies(session):
+            self._anonymous_bootstrap_attempted = True
+            return
+        with self._lock:
+            if self._anonymous_bootstrap_attempted:
+                return
+            self._anonymous_bootstrap_attempted = True
+        logger.info("No IAAI cookies preloaded. Attempting anonymous session bootstrap via Playwright.")
+        try:
+            self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning(
+                "Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s",
+                exc,
+            )
+
+    @staticmethod
+    def _session_has_iaai_cookies(session: requests.Session) -> bool:
+        cookies = getattr(session, "cookies", None)
+        if cookies is None:
+            return False
+        for item in cookies:
+            domain = ""
+            if isinstance(item, dict):
+                domain = parse_text(item.get("domain")) or ""
+            else:
+                domain = str(getattr(item, "domain", "") or "")
+            if not domain:
+                return True
+            if "iaai.com" in domain.lower():
+                return True
+        return False
+
+    def _bootstrap_session_cookies(self, session: requests.Session) -> None:
+        if self._bootstrap_cookies_loaded:
+            return
+        self._load_storage_state_cookies(session)
+        self._load_env_cookies(session)
+        self._bootstrap_cookies_loaded = True
+
+    def _load_storage_state_cookies(self, session: requests.Session) -> None:
+        path = self._settings.iaai_storage_state_path
+        if not path.exists():
+            return
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Failed to read storage state file '%s': %s", path, exc)
+            return
+        cookies = payload.get("cookies") if isinstance(payload, dict) else None
+        if not isinstance(cookies, list):
+            return
+        applied = 0
+        for item in cookies:
+            if not isinstance(item, dict):
+                continue
+            name = parse_text(item.get("name"))
+            value = parse_text(item.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(item.get("domain")) or ".iaai.com"
+            cookie_path = parse_text(item.get("path")) or "/"
+            expires_raw = item.get("expires")
+            expires = parse_int(expires_raw)
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+            applied += 1
+        if applied:
+            logger.info("Loaded %s cookies from storage state", applied)
+
+    def _load_env_cookies(self, session: requests.Session) -> None:
+        raw = self._settings.iaai_session_cookies
+        if not raw:
+            return
+
+        try:
+            payload = json.loads(raw)
+        except Exception:  # noqa: BLE001
+            payload = None
+
+        applied = 0
+        if isinstance(payload, list):
+            for item in payload:
+                if not isinstance(item, dict):
+                    continue
+                name = parse_text(item.get("name"))
+                value = parse_text(item.get("value"))
+                if not name or value is None:
+                    continue
+                domain = parse_text(item.get("domain")) or ".iaai.com"
+                cookie_path = parse_text(item.get("path")) or "/"
+                expires = parse_int(item.get("expires"))
+                session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+                applied += 1
+        elif isinstance(payload, dict):
+            for name, value in payload.items():
+                if not isinstance(name, str) or not isinstance(value, str):
+                    continue
+                session.cookies.set(name.strip(), value.strip(), domain=".iaai.com", path="/")
+                applied += 1
+        else:
+            for part in raw.split(";"):
+                if "=" not in part:
+                    continue
+                name, value = part.split("=", 1)
+                name = name.strip()
+                value = value.strip()
+                if not name:
+                    continue
+                session.cookies.set(name, value, domain=".iaai.com", path="/")
+                applied += 1
+
+        if applied:
+            logger.info("Loaded %s cookies from IAAI_SESSION_COOKIES", applied)
+
+    def _refresh_session_via_playwright(
+        self,
+        *,
+        expected_marker: str | None = None,
+        session: requests.Session | None = None,
+    ) -> None:
+        target_session = session or self._get_session()
+        with self._lock:
+            baseline_generation = self._refresh_generation
+
+        with self._refresh_lock:
+            with self._lock:
+                if self._refresh_generation > baseline_generation and self._latest_refresh_cookies:
+                    self._apply_cookies_to_session(target_session, self._latest_refresh_cookies)
+                    self._thread_local.session_generation = self._refresh_generation
+                    return
+
+            logger.warning("IAAI session challenge detected. Refreshing session via Playwright.")
+            cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
+            self._apply_cookies_to_session(target_session, cookies)
+
+            with self._lock:
+                self._refresh_generation += 1
+                self._latest_refresh_cookies = list(cookies)
+                self._anonymous_bootstrap_attempted = True
+                refreshed_generation = self._refresh_generation
+            self._thread_local.session_generation = refreshed_generation
+            self._save_storage_state(target_session)
+
+    def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
+        try:
+            from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+            from playwright.sync_api import sync_playwright
+        except Exception as exc:  # noqa: BLE001
+            raise RuntimeError(
+                "Playwright is required for challenge fallback. Install dependency and browsers."
+            ) from exc
+
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=True)
+            try:
+                context = browser.new_context(
+                    locale="en-US",
+                    viewport={"width": 1366, "height": 768},
+                    user_agent=DEFAULT_USER_AGENT,
+                )
+                page = context.new_page()
+                home_target = self._settings.iaai_base_url + "/"
+                target = urljoin(self._settings.iaai_base_url + "/", "Vehiclelisting/Cars")
+                timeout_ms = max(30_000, self._settings.http_timeout * 1000)
+                page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                self._try_login(page)
+                self._wait_until_non_challenge(
+                    page=page,
+                    target=target,
+                    timeout_ms=timeout_ms,
+                    expected_marker=expected_marker or LISTING_MARKER,
+                )
+                state = context.storage_state()
+            except PlaywrightTimeoutError as exc:
+                raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
+            finally:
+                browser.close()
+
+        cookies = state.get("cookies") if isinstance(state, dict) else None
+        if not isinstance(cookies, list) or not cookies:
+            raise RuntimeError("Playwright refresh did not return cookies")
+        return [cookie for cookie in cookies if isinstance(cookie, dict)]
+
+    @staticmethod
+    def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None:
+        session.cookies.clear()
+        for cookie in cookies:
+            name = parse_text(cookie.get("name"))
+            value = parse_text(cookie.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(cookie.get("domain")) or ".iaai.com"
+            cookie_path = parse_text(cookie.get("path")) or "/"
+            expires = parse_int(cookie.get("expires"))
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+
+    @staticmethod
+    def _wait_until_non_challenge(
+        *,
+        page: Any,
+        target: str,
+        timeout_ms: int,
+        expected_marker: str | None,
+    ) -> None:
+        # Give Incapsula redirect/challenge flow time to settle and verify page really opened.
+        poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
+        navigation_error_count = 0
+        for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
+            try:
+                page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
+            except Exception:  # noqa: BLE001
+                # The challenge page frequently redirects; continue polling.
+                pass
+            page.wait_for_timeout(poll_ms)
+
+            body: str | None = None
+            for _ in range(3):
+                try:
+                    body = page.content()
+                    break
+                except Exception as exc:  # noqa: BLE001
+                    message = str(exc).lower()
+                    if "page.content" not in message or "navigating and changing the content" not in message:
+                        raise
+                    navigation_error_count += 1
+                    page.wait_for_timeout(max(200, poll_ms // 4))
+                    continue
+
+            if body is not None and not is_challenge_response(
+                status_code=200,
+                body_text=body,
+                expected_marker=expected_marker,
+            ):
+                return
+            try:
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+            except Exception as exc:  # noqa: BLE001
+                logger.debug(
+                    "Playwright challenge retry navigation failed poll=%s/%s: %s",
+                    poll_index + 1,
+                    PLAYWRIGHT_REFRESH_POLLS,
+                    exc,
+                )
+
+        raise RuntimeError(
+            "Playwright refresh completed but challenge page is still active "
+            f"(navigation_content_errors={navigation_error_count})"
+        )
+
+    def _try_login(self, page: Any) -> None:
+        if not self._settings.iaai_login or not self._settings.iaai_password:
+            return
+
+        try:
+            login_selectors = (
+                "input[type='email']",
+                "input[name*='email' i]",
+                "input[id*='email' i]",
+                "input[name*='username' i]",
+            )
+            password_selector = "input[type='password']"
+            submit_selector = "button[type='submit'],input[type='submit']"
+
+            email_locator = None
+            for selector in login_selectors:
+                locator = page.locator(selector)
+                if locator.count() > 0:
+                    email_locator = locator.first
+                    break
+            if email_locator is None:
+                return
+
+            email_locator.fill(self._settings.iaai_login)
+            password_locator = page.locator(password_selector)
+            if password_locator.count() == 0:
+                return
+            password_locator.first.fill(self._settings.iaai_password)
+            submit_locator = page.locator(submit_selector)
+            if submit_locator.count() > 0:
+                submit_locator.first.click()
+                page.wait_for_load_state("domcontentloaded", timeout=30_000)
+        except Exception as exc:  # noqa: BLE001
+            logger.warning("Playwright login step failed: %s", exc)
+
+    def _accept_cookie_banner(self, page: Any) -> None:
+        for selector in COOKIE_ACCEPT_SELECTORS:
+            try:
+                locator = page.locator(selector).first
+                if locator.count() == 0:
+                    continue
+                if not locator.is_visible(timeout=500):
+                    continue
+                locator.click(timeout=2_000)
+                page.wait_for_timeout(250)
+                return
+            except Exception:  # noqa: BLE001
+                continue
+
+    def _save_storage_state(self, session: requests.Session) -> None:
+        path = self._settings.iaai_storage_state_path
+        cookies: list[dict[str, Any]] = []
+        for cookie in session.cookies:
+            cookie_payload: dict[str, Any] = {
+                "name": cookie.name,
+                "value": cookie.value,
+                "domain": cookie.domain or ".iaai.com",
+                "path": cookie.path or "/",
+                "httpOnly": False,
+                "secure": bool(cookie.secure),
+                "sameSite": "Lax",
+            }
+            if cookie.expires is not None:
+                cookie_payload["expires"] = int(cookie.expires)
+            cookies.append(cookie_payload)
+
+        payload = {"cookies": cookies, "origins": []}
+        path.parent.mkdir(parents=True, exist_ok=True)
+        path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
+
+    @staticmethod
+    def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None:
+        if retry_backoff_ms <= 0:
+            return
+        delay = retry_backoff_ms * (2**attempt) / 1000
+        time.sleep(delay)
+
+
+class IAAIClient:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._auth = HybridSessionAuth(settings)
+
+    def persist_session_state(self) -> None:
+        self._auth.persist_storage_state()
+
+    def keep_session_alive(self) -> dict[str, Any]:
+        started = time.perf_counter()
+        scope = resolve_listing_scope_paths(
+            listing_start_url=self._settings.iaai_listing_start_url,
+            brands=set(),
+        )[0]
+        page_html = self._fetch_listing_first_page(scope)
+        marker_present = LISTING_MARKER in page_html
+        result_count = parse_int(parse_hidden_input_value(page_html, "ResultCount"))
+        self.persist_session_state()
+        return {
+            "scope": scope,
+            "marker_present": marker_present,
+            "result_count": result_count,
+            "elapsed_ms": int((time.perf_counter() - started) * 1000),
+        }
+
+    def iter_listing_vehicles(self, filters: RuntimeFilters) -> Iterator[ListingVehicle]:
+        seen_inventory_ids: set[str] = set()
+        if self._settings.iaai_listing_start_url.strip() and filters.brands:
+            logger.info(
+                "Using explicit listing start URL; runtime brand scopes in listing step are ignored."
+            )
+        scope_paths = resolve_listing_scope_paths(
+            listing_start_url=self._settings.iaai_listing_start_url,
+            brands=filters.brands,
+        )
+        listing_started_at = time.perf_counter()
+
+        for scope_index, scope_path in enumerate(scope_paths, start=1):
+            logger.info(
+                "IAAI listing scope start scope=%s/%s path=%s",
+                scope_index,
+                len(scope_paths),
+                scope_path,
+            )
+            first_page_html = self._fetch_listing_first_page(scope_path)
+            first_page = parse_listing_page(first_page_html)
+            for vehicle in first_page.vehicles:
+                if vehicle.inventory_id in seen_inventory_ids:
+                    continue
+                seen_inventory_ids.add(vehicle.inventory_id)
+                yield vehicle
+
+            page_size = max(1, first_page.page_size)
+            total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size))
+            gbp_search_query = first_page.gbp_search_query
+            logger.info(
+                "IAAI listing scope loaded scope=%s/%s path=%s result_count=%s page_size=%s total_pages=%s unique_ids=%s elapsed=%.1fs",
+                scope_index,
+                len(scope_paths),
+                scope_path,
+                first_page.result_count,
+                page_size,
+                total_pages,
+                len(seen_inventory_ids),
+                time.perf_counter() - listing_started_at,
+            )
+
+            for page_number in range(2, total_pages + 1):
+                page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size)
+                parsed_page = parse_listing_page(page_html)
+                gbp_search_query = parsed_page.gbp_search_query
+                for vehicle in parsed_page.vehicles:
+                    if vehicle.inventory_id in seen_inventory_ids:
+                        continue
+                    seen_inventory_ids.add(vehicle.inventory_id)
+                    yield vehicle
+                if page_number % 10 == 0 or page_number == total_pages:
+                    logger.info(
+                        "IAAI listing page progress scope=%s/%s path=%s page=%s/%s unique_ids=%s elapsed=%.1fs",
+                        scope_index,
+                        len(scope_paths),
+                        scope_path,
+                        page_number,
+                        total_pages,
+                        len(seen_inventory_ids),
+                        time.perf_counter() - listing_started_at,
+                    )
+
+    def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]:
+        escaped_id = quote(inventory_id, safe="~")
+        url = urljoin(self._settings.iaai_base_url + "/", f"VehicleDetail/{escaped_id}")
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=self._settings.http_timeout,
+            retries=self._settings.http_retries,
+            retry_backoff_ms=self._settings.http_retry_backoff_ms,
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=DETAIL_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
+            return parse_product_details_vm(response.text)
+
+    def _fetch_listing_first_page(self, scope_path: str) -> str:
+        if scope_path.lower().startswith(("http://", "https://")):
+            url = scope_path
+        else:
+            url = urljoin(self._settings.iaai_base_url + "/", scope_path.lstrip("/"))
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=self._settings.http_timeout,
+            retries=self._settings.http_retries,
+            retry_backoff_ms=self._settings.http_retry_backoff_ms,
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=LISTING_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}")
+            return response.text
+
+    def _fetch_listing_page(
+        self,
+        scope_path: str,
+        gbp_search_query: dict[str, Any],
+        page_number: int,
+        page_size: int,
+    ) -> str:
+        query_payload = dict(gbp_search_query)
+        query_payload["CurrentPage"] = page_number
+        query_payload["PageSize"] = page_size
+
+        search_url = urljoin(self._settings.iaai_base_url + "/", "Search")
+        common_headers = {
+            "accept": "text/html,application/xhtml+xml,*/*",
+            "x-requested-with": "XMLHttpRequest",
+        }
+
+        attempts: list[tuple[dict[str, str], Any, Any]] = [
+            ({**common_headers, "content-type": "application/json"}, None, query_payload),
+            ({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}),
+            ({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None),
+            (
+                {**common_headers, "content-type": "application/json"},
+                json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}),
+                None,
+            ),
+        ]
+
+        last_error: Exception | None = None
+        for headers, data, json_body in attempts:
+            try:
+                response = self._auth.request(
+                    "POST",
+                    search_url,
+                    timeout=self._settings.http_timeout,
+                    retries=self._settings.http_retries,
+                    retry_backoff_ms=self._settings.http_retry_backoff_ms,
+                    headers=headers,
+                    data=data,
+                    json_body=json_body,
+                    expected_marker=LISTING_MARKER,
+                )
+                with response:
+                    if response.status_code >= 400:
+                        raise RuntimeError(
+                            f"Listing page request failed status={response.status_code} page={page_number}"
+                        )
+                    body = response.text
+                    if LISTING_MARKER not in body:
+                        raise RuntimeError("Listing page response does not include GBPSearchQuery")
+                    return body
+            except Exception as exc:  # noqa: BLE001
+                last_error = exc
+                continue
+
+        if last_error is not None:
+            raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error
+        raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}")
+
+
+def build_brand_scope_paths(brands: set[str]) -> list[str]:
+    if not brands:
+        return ["/Vehiclelisting/Cars"]
+
+    paths: list[str] = []
+    for brand in sorted(brands):
+        raw = brand.strip()
+        if not raw:
+            continue
+        slug_hyphen = quote(raw.replace(" ", "-"), safe="-")
+        slug_raw = quote(raw, safe="")
+        for slug in (slug_hyphen, slug_raw):
+            path = f"/Vehiclelisting/Cars/{slug}"
+            if path not in paths:
+                paths.append(path)
+    return paths or ["/Vehiclelisting/Cars"]
+
+
+def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
+    explicit_scope = listing_start_url.strip()
+    if explicit_scope:
+        return [explicit_scope]
+    return build_brand_scope_paths(brands)
+
+
+def parse_listing_page(html_text: str) -> ListingPage:
+    gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
+    vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
+    result_count_raw = parse_hidden_input_value(html_text, "ResultCount")
+    page_size_raw = parse_hidden_input_value(html_text, "PageSize")
+    current_page_raw = parse_hidden_input_value(html_text, "CurrentPage")
+
+    if not gbp_raw:
+        raise RuntimeError("Listing page missing GBPSearchQuery")
+    if vehicle_raw is None:
+        raise RuntimeError("Listing page missing VehicleDetails")
+
+    gbp_payload = json.loads(gbp_raw)
+    if not isinstance(gbp_payload, dict):
+        raise RuntimeError("GBPSearchQuery payload is not object")
+
+    vehicle_payload = json.loads(vehicle_raw)
+    if not isinstance(vehicle_payload, list):
+        raise RuntimeError("VehicleDetails payload is not array")
+
+    vehicles: list[ListingVehicle] = []
+    for item in vehicle_payload:
+        if not isinstance(item, dict):
+            continue
+        inventory_id = parse_text(item.get("Id"))
+        if not inventory_id:
+            continue
+        vehicles.append(
+            ListingVehicle(
+                inventory_id=inventory_id,
+                tenant=parse_text(item.get("Tenant")),
+                auction_id=parse_text(item.get("ActnLnId")),
+                auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")),
+                inventory_status=parse_text(item.get("InventoryStatus")),
+                currency=parse_text(item.get("Currency")),
+                timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")),
+                timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")),
+                prebid_indicator=parse_bool(item.get("PreBidIndicator")),
+                buynow_indicator=parse_bool(item.get("BuyNowIndicator")),
+            )
+        )
+
+    return ListingPage(
+        vehicles=vehicles,
+        result_count=parse_int(result_count_raw) or len(vehicles),
+        page_size=parse_int(page_size_raw) or max(1, len(vehicles)),
+        current_page=parse_int(current_page_raw) or 1,
+        gbp_search_query=gbp_payload,
+    )
+
+
+def parse_product_details_vm(html_text: str) -> dict[str, Any]:
+    match = re.search(
+        r"<script[^>]*id=\"ProductDetailsVM\"[^>]*>\s*(\{.*?\})\s*</script>",
+        html_text,
+        flags=re.DOTALL,
+    )
+    if match is None:
+        raise RuntimeError("ProductDetailsVM script not found")
+
+    payload = json.loads(match.group(1))
+    if not isinstance(payload, dict):
+        raise RuntimeError("ProductDetailsVM root is not object")
+    return payload
+
+
+def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
+    escaped_id = re.escape(input_id)
+    patterns = (
+        rf"<input[^>]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"",
+        rf"<input[^>]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, html_text, flags=re.IGNORECASE)
+        if match is not None:
+            return html.unescape(match.group(1))
+    return None
+
+
+def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
+    seen_fullres: set[str] = set()
+    images: list[dict[str, str | int]] = []
+
+    for index, item in enumerate(image_keys):
+        if not isinstance(item, dict):
+            continue
+        key = parse_text(item.get("k"))
+        if key is None:
+            continue
+
+        width = parse_int(item.get("w")) or 1600
+        height = parse_int(item.get("h")) or 1200
+        if width <= 0:
+            width = 1600
+        if height <= 0:
+            height = 1200
+
+        order_index = parse_int(item.get("i"))
+        if order_index is None:
+            order_index = parse_int(item.get("in"))
+        if order_index is None:
+            order_index = index
+
+        preview_width = min(640, width)
+        preview_height = max(1, int(round(height * (preview_width / width))))
+
+        escaped_key = quote(key, safe="~")
+        fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}"
+        preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}"
+
+        if fullres in seen_fullres:
+            continue
+        seen_fullres.add(fullres)
+        images.append(
+            {
+                "order_index": order_index,
+                "fullres_image": fullres,
+                "preview_image": preview,
+            }
+        )
+
+    images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image"))))
+    return images
+
+
+def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool:
+    if status_code in {401, 403}:
+        return True
+
+    if _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        # If expected listing/detail marker is present, this is a valid page even if
+        # Incapsula script references are embedded in the HTML.
+        return False
+
+    lowered = (body_text or "").lower()
+    if any(marker in lowered for marker in CHALLENGE_MARKERS):
+        return True
+
+    if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        # Expected hidden marker/script missing from HTML often means anti-bot interstitial.
+        if "<html" in lowered or "<body" in lowered:
+            return True
+    return False
+
+
+def _expected_marker_present(*, body_text: str, expected_marker: str | None) -> bool:
+    if not expected_marker:
+        return False
+    if expected_marker in body_text:
+        return True
+
+    # Accept quote variants for marker fragments like id="GBPSearchQuery" / id='GBPSearchQuery'.
+    if '"' in expected_marker:
+        single_quoted = expected_marker.replace('"', "'")
+        if single_quoted in body_text:
+            return True
+    if "'" in expected_marker:
+        double_quoted = expected_marker.replace("'", '"')
+        if double_quoted in body_text:
+            return True
+
+    marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker)
+    if marker_match is None:
+        return False
+    marker_id = re.escape(marker_match.group(1))
+    return bool(
+        re.search(
+            rf"id\s*=\s*['\"]{marker_id}['\"]",
+            body_text,
+            flags=re.IGNORECASE,
+        )
+    )
+
+
+def parse_text(value: Any) -> str | None:
+    if isinstance(value, str):
+        text = value.strip()
+        return text if text else None
+    return None
+
+
+def parse_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        return normalized in {"true", "1", "yes", "on"}
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return value != 0
+    return False
+
+
+def parse_int(value: Any) -> int | None:
+    if value is None or isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(round(value))
+    if isinstance(value, str):
+        text = value.strip()
+        if not text:
+            return None
+        normalized = text.replace(",", "").replace(" ", "")
+        try:
+            return int(round(float(normalized)))
+        except ValueError:
+            return None
+    return None

iaai_sync_service/database.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session, sessionmaker
+
+from iaai_sync_service.settings import get_settings
+
+_settings = get_settings()
+
+engine = create_engine(_settings.database_url, pool_pre_ping=True, future=True)
+SessionLocal = sessionmaker(bind=engine, autoflush=False, autocommit=False, expire_on_commit=False)
+
+
+def get_session() -> Session:
+    return SessionLocal()

iaai_sync_service/models.py

@@ -0,0 +1,221 @@
+from __future__ import annotations
+
+from datetime import datetime
+
+from sqlalchemy import (
+    BigInteger,
+    Boolean,
+    DateTime,
+    Enum,
+    ForeignKey,
+    Integer,
+    String,
+    Text,
+    func,
+)
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+CAR_TABLE_NAME = "iaai_cars"
+IMAGE_TABLE_NAME = "iaai_images"
+SYNC_RUN_TABLE_NAME = "iaai_sync_runs"
+
+
+CURRENCY_ENUM_VALUES = ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD")
+DRIVE_ENUM_VALUES = ("FWD", "RWD", "TWO_WD", "FOUR_WD", "2WD", "4WD", "NA")
+GEARBOX_ENUM_VALUES = ("AT", "CVT", "MT", "EV", "NA")
+STEERING_WHEEL_ENUM_VALUES = ("LEFT", "RIGHT", "left", "right", "NA")
+BODY_TYPE_ENUM_VALUES = (
+    "COUPE",
+    "SUV",
+    "HATCHBACK",
+    "MINIVAN",
+    "SEDAN",
+    "NA",
+    "Station Wagon",
+    "Pickup",
+    "Truck",
+    "Open",
+    "RV",
+    "Other",
+    "STATION_WAGON",
+    "PICKUP",
+    "TRUCK",
+    "OPEN",
+    "OTHER",
+)
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
+ORIGIN_ENUM_VALUES = (
+    "TAU",
+    "CARSENSOR",
+    "HANAMARU",
+    "ENCAR",
+    "KURUMA_TRADER",
+    "carsensor",
+    "encar",
+    "kuruma_trader",
+    "asnet",
+    "kababa",
+    "ACV",
+    "COPART",
+    "IAAI",
+    "copart",
+    "iaai",
+    "NA",
+    "ASNET",
+    "KABABA",
+)
+SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "stock", "auction", "tender", "NA")
+
+
+class Car(Base):
+    __tablename__ = CAR_TABLE_NAME
+
+    id: Mapped[int] = mapped_column(Integer, primary_key=True, autoincrement=True)
+    parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
+    brand: Mapped[str] = mapped_column(String(50), nullable=False)
+    model: Mapped[str] = mapped_column(String(50), nullable=False)
+    year: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
+    currency: Mapped[str] = mapped_column(
+        Enum(
+            *CURRENCY_ENUM_VALUES,
+            name="currencyenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="USD",
+    )
+    mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    country: Mapped[str] = mapped_column(
+        Enum(
+            *COUNTRY_ENUM_VALUES,
+            name="countryenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="NA",
+    )
+    is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
+    drive: Mapped[str | None] = mapped_column(
+        Enum(
+            *DRIVE_ENUM_VALUES,
+            name="driveenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=True,
+    )
+    gearbox: Mapped[str | None] = mapped_column(
+        Enum(
+            *GEARBOX_ENUM_VALUES,
+            name="gearboxenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=True,
+    )
+    steering_wheel: Mapped[str | None] = mapped_column(
+        Enum(
+            *STEERING_WHEEL_ENUM_VALUES,
+            name="steeringwheelenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=True,
+    )
+    body_type: Mapped[str] = mapped_column(
+        Enum(
+            *BODY_TYPE_ENUM_VALUES,
+            name="bodytypeenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="OTHER",
+    )
+    engine_volume: Mapped[int | None] = mapped_column(Integer, nullable=True)
+    selling_type: Mapped[str] = mapped_column(
+        Enum(
+            *SELLING_TYPE_ENUM_VALUES,
+            name="sellingtypeenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="NA",
+    )
+    one_owner: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    new_car: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    is_hidden: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    origin: Mapped[str] = mapped_column(
+        Enum(
+            *ORIGIN_ENUM_VALUES,
+            name="originenum",
+            native_enum=True,
+            create_constraint=False,
+        ),
+        nullable=False,
+        default="NA",
+    )
+    origin_url: Mapped[str] = mapped_column(String(), nullable=False)
+    origin_id: Mapped[str] = mapped_column(String(), nullable=False, unique=True)
+    is_damaged: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    evaluation: Mapped[str | None] = mapped_column(String(), nullable=True)
+    non_smoking: Mapped[bool] = mapped_column(Boolean, nullable=False, default=True)
+    rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
+    slug: Mapped[str] = mapped_column(String(), nullable=False)
+    last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+
+    images: Mapped[list["Image"]] = relationship(
+        "Image",
+        back_populates="car",
+        cascade="all, delete-orphan",
+    )
+
+
+class Image(Base):
+    __tablename__ = IMAGE_TABLE_NAME
+
+    id: Mapped[int] = mapped_column(
+        BigInteger().with_variant(Integer, "sqlite"),
+        primary_key=True,
+        autoincrement=True,
+    )
+    fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
+    preview_image: Mapped[str] = mapped_column(String(), nullable=False)
+    order_index: Mapped[int] = mapped_column(Integer, nullable=False)
+    car_id: Mapped[int] = mapped_column(
+        Integer,
+        ForeignKey(f"{CAR_TABLE_NAME}.id", ondelete="CASCADE"),
+        nullable=False,
+    )
+
+    car: Mapped[Car] = relationship("Car", back_populates="images")
+
+
+class SyncRun(Base):
+    __tablename__ = SYNC_RUN_TABLE_NAME
+
+    id: Mapped[int] = mapped_column(
+        BigInteger().with_variant(Integer, "sqlite"),
+        primary_key=True,
+        autoincrement=True,
+    )
+    started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
+    finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+    status: Mapped[str] = mapped_column(Text, nullable=False)
+    lane: Mapped[str] = mapped_column(Text, nullable=False)
+    ids_fetched: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    cars_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+    error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)

iaai_sync_service/runtime_config.py

@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import json
+import logging
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+
+@dataclass(frozen=True)
+class RuntimeFilters:
+    brands: set[str] = field(default_factory=set)
+    models: set[str] = field(default_factory=set)
+    years: set[int] = field(default_factory=set)
+    body_types: set[str] = field(default_factory=set)
+
+    def is_enabled(self) -> bool:
+        return bool(self.brands or self.models or self.years or self.body_types)
+
+
+@dataclass(frozen=True)
+class RuntimeConfig:
+    condition_check_enabled: bool | None = None
+    filters: RuntimeFilters = field(default_factory=RuntimeFilters)
+
+
+def load_runtime_config(path: Path, logger: logging.Logger | None = None) -> RuntimeConfig:
+    if not path.exists():
+        return RuntimeConfig()
+
+    try:
+        payload = json.loads(path.read_text(encoding="utf-8"))
+    except Exception as exc:  # noqa: BLE001
+        if logger is not None:
+            logger.warning("Failed to parse runtime config file '%s': %s", path, exc)
+        return RuntimeConfig()
+
+    if not isinstance(payload, dict):
+        if logger is not None:
+            logger.warning("Runtime config file '%s' root must be object.", path)
+        return RuntimeConfig()
+
+    sync_block = payload.get("sync")
+    filters_block = payload.get("filters")
+    if not isinstance(sync_block, dict):
+        sync_block = {}
+    if not isinstance(filters_block, dict):
+        filters_block = {}
+
+    return RuntimeConfig(
+        condition_check_enabled=_to_optional_bool(sync_block.get("condition_check_enabled")),
+        filters=RuntimeFilters(
+            brands=_to_ci_set(filters_block.get("brands")),
+            models=_to_ci_set(filters_block.get("models")),
+            years=_to_int_set(filters_block.get("years"), min_value=1900),
+            body_types=_to_ci_set(filters_block.get("body_types")),
+        )
+    )
+
+
+def _to_optional_bool(value: Any) -> bool | None:
+    if value is None:
+        return None
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        normalized = value.strip().lower()
+        if normalized in {"true", "1", "yes", "on"}:
+            return True
+        if normalized in {"false", "0", "no", "off"}:
+            return False
+    return None
+
+
+def _to_ci_set(value: Any) -> set[str]:
+    if not isinstance(value, list):
+        return set()
+    result: set[str] = set()
+    for item in value:
+        if not isinstance(item, str):
+            continue
+        text = item.strip().casefold()
+        if text:
+            result.add(text)
+    return result
+
+
+def _to_int_set(value: Any, *, min_value: int) -> set[int]:
+    if not isinstance(value, list):
+        return set()
+    result: set[int] = set()
+    for item in value:
+        if isinstance(item, bool):
+            continue
+        if isinstance(item, int):
+            if item >= min_value:
+                result.add(item)
+            continue
+        if isinstance(item, str) and item.strip().isdigit():
+            parsed = int(item.strip())
+            if parsed >= min_value:
+                result.add(parsed)
+    return result

iaai_sync_service/schema_bootstrap.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from sqlalchemy import text
+from sqlalchemy.orm import Session
+
+from iaai_sync_service.models import (
+    BODY_TYPE_ENUM_VALUES,
+    CAR_TABLE_NAME,
+    COUNTRY_ENUM_VALUES,
+    CURRENCY_ENUM_VALUES,
+    DRIVE_ENUM_VALUES,
+    GEARBOX_ENUM_VALUES,
+    IMAGE_TABLE_NAME,
+    ORIGIN_ENUM_VALUES,
+    SELLING_TYPE_ENUM_VALUES,
+    SYNC_RUN_TABLE_NAME,
+    STEERING_WHEEL_ENUM_VALUES,
+)
+
+SCHEMA_BOOTSTRAP_LOCK_KEY = 901_700_001
+
+
+def _create_enum_if_missing(session: Session, type_name: str, values: tuple[str, ...]) -> None:
+    values_sql = ", ".join("'" + value.replace("'", "''") + "'" for value in values)
+    session.execute(
+        text(
+            f"""
+            DO $$
+            BEGIN
+                IF NOT EXISTS (
+                    SELECT 1
+                    FROM pg_type t
+                    JOIN pg_namespace n ON n.oid = t.typnamespace
+                    WHERE n.nspname = 'public'
+                      AND t.typname = '{type_name}'
+                ) THEN
+                    CREATE TYPE public.{type_name} AS ENUM ({values_sql});
+                END IF;
+            END
+            $$;
+            """
+        )
+    )
+
+
+def _add_enum_value_if_missing(session: Session, type_name: str, value: str) -> None:
+    escaped = value.replace("'", "''")
+    session.execute(text(f"ALTER TYPE public.{type_name} ADD VALUE IF NOT EXISTS '{escaped}'"))
+
+
+def _acquire_schema_bootstrap_lock(session: Session) -> None:
+    session.execute(text("SELECT pg_advisory_xact_lock(:key)"), {"key": SCHEMA_BOOTSTRAP_LOCK_KEY})
+
+
+def ensure_schema(session: Session) -> None:
+    bind = session.get_bind()
+    if bind is None or bind.dialect.name != "postgresql":
+        return
+
+    _acquire_schema_bootstrap_lock(session)
+
+    _create_enum_if_missing(session, "currencyenum", CURRENCY_ENUM_VALUES)
+    _create_enum_if_missing(session, "driveenum", DRIVE_ENUM_VALUES)
+    _create_enum_if_missing(session, "gearboxenum", GEARBOX_ENUM_VALUES)
+    _create_enum_if_missing(session, "steeringwheelenum", STEERING_WHEEL_ENUM_VALUES)
+    _create_enum_if_missing(session, "bodytypeenum", BODY_TYPE_ENUM_VALUES)
+    _create_enum_if_missing(session, "countryenum", COUNTRY_ENUM_VALUES)
+    _create_enum_if_missing(session, "originenum", ORIGIN_ENUM_VALUES)
+    _create_enum_if_missing(session, "sellingtypeenum", SELLING_TYPE_ENUM_VALUES)
+
+    for value in CURRENCY_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "currencyenum", value)
+    for value in DRIVE_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "driveenum", value)
+    for value in GEARBOX_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "gearboxenum", value)
+    for value in STEERING_WHEEL_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "steeringwheelenum", value)
+    for value in BODY_TYPE_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "bodytypeenum", value)
+    for value in COUNTRY_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "countryenum", value)
+    for value in ORIGIN_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "originenum", value)
+    for value in SELLING_TYPE_ENUM_VALUES:
+        _add_enum_value_if_missing(session, "sellingtypeenum", value)
+
+    session.execute(
+        text(
+            f"""
+            CREATE TABLE IF NOT EXISTS public.{CAR_TABLE_NAME} (
+                id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+                parser_id VARCHAR(50) NOT NULL UNIQUE,
+                brand VARCHAR(50) NOT NULL,
+                model VARCHAR(50) NOT NULL,
+                year INTEGER NULL,
+                price BIGINT NULL,
+                currency public.currencyenum NOT NULL DEFAULT 'USD',
+                mileage INTEGER NOT NULL DEFAULT 0,
+                country public.countryenum NOT NULL DEFAULT 'NA',
+                is_sold BOOLEAN NOT NULL DEFAULT FALSE,
+                color VARCHAR NOT NULL DEFAULT 'other',
+                drive public.driveenum NULL,
+                gearbox public.gearboxenum NULL,
+                steering_wheel public.steeringwheelenum NULL,
+                body_type public.bodytypeenum NOT NULL DEFAULT 'OTHER',
+                engine_volume INTEGER NULL,
+                selling_type public.sellingtypeenum NOT NULL DEFAULT 'NA',
+                one_owner BOOLEAN NOT NULL DEFAULT FALSE,
+                new_car BOOLEAN NOT NULL DEFAULT FALSE,
+                is_hidden BOOLEAN NOT NULL DEFAULT FALSE,
+                origin public.originenum NOT NULL DEFAULT 'NA',
+                origin_url VARCHAR NOT NULL,
+                origin_id VARCHAR NOT NULL UNIQUE,
+                is_damaged BOOLEAN NOT NULL DEFAULT FALSE,
+                evaluation VARCHAR NULL,
+                non_smoking BOOLEAN NOT NULL DEFAULT TRUE,
+                rental BOOLEAN NOT NULL DEFAULT FALSE,
+                repair_history BOOLEAN NOT NULL DEFAULT FALSE,
+                slug VARCHAR NOT NULL,
+                last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+            )
+            """
+        )
+    )
+    session.execute(
+        text(
+            f"""
+            CREATE TABLE IF NOT EXISTS public.{IMAGE_TABLE_NAME} (
+                id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+                fullres_image VARCHAR NOT NULL,
+                preview_image VARCHAR NOT NULL,
+                order_index INTEGER NOT NULL,
+                car_id INTEGER NOT NULL REFERENCES public.{CAR_TABLE_NAME}(id) ON DELETE CASCADE
+            )
+            """
+        )
+    )
+    session.execute(
+        text(
+            f"""
+            CREATE TABLE IF NOT EXISTS public.{SYNC_RUN_TABLE_NAME} (
+                id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
+                started_at TIMESTAMPTZ NOT NULL,
+                finished_at TIMESTAMPTZ NULL,
+                status TEXT NOT NULL,
+                lane TEXT NOT NULL,
+                ids_fetched INTEGER NOT NULL DEFAULT 0,
+                cars_upserted INTEGER NOT NULL DEFAULT 0,
+                cars_failed INTEGER NOT NULL DEFAULT 0,
+                images_upserted INTEGER NOT NULL DEFAULT 0,
+                error_summary TEXT NULL
+            )
+            """
+        )
+    )
+
+    session.execute(text(f"CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON public.{IMAGE_TABLE_NAME} (car_id)"))
+    session.execute(
+        text(f"CREATE INDEX IF NOT EXISTS ix_iaai_images_car_order ON public.{IMAGE_TABLE_NAME} (car_id, order_index)")
+    )
+    session.execute(
+        text(
+            f"CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_id ON public.{CAR_TABLE_NAME} "
+            "(last_seen_at DESC, id DESC)"
+        )
+    )
+    session.execute(
+        text(
+            f"CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_feed ON public.{CAR_TABLE_NAME} "
+            "(last_seen_at DESC, id DESC) WHERE is_sold = FALSE AND is_hidden = FALSE"
+        )
+    )
+    session.execute(
+        text(f"CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_started_at ON public.{SYNC_RUN_TABLE_NAME} (started_at)")
+    )
+    session.execute(
+        text(f"CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON public.{SYNC_RUN_TABLE_NAME} (status)")
+    )

iaai_sync_service/settings.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from functools import lru_cache
+from pathlib import Path
+
+
+def _read_env(name: str, default: str | None = None, *, required: bool = False) -> str:
+    value = os.getenv(name, default)
+    if required and (value is None or not value.strip()):
+        raise RuntimeError(f"Missing required environment variable: {name}")
+    if value is None:
+        return ""
+    return value.strip()
+
+
+def _read_int(name: str, default: int) -> int:
+    raw = _read_env(name, str(default))
+    try:
+        return int(raw)
+    except ValueError as exc:
+        raise RuntimeError(f"Environment variable {name} must be integer, got: {raw}") from exc
+
+
+def _read_bool(name: str, default: bool) -> bool:
+    raw = _read_env(name, "true" if default else "false")
+    normalized = raw.strip().lower()
+    if normalized in {"1", "true", "yes", "on"}:
+        return True
+    if normalized in {"0", "false", "no", "off"}:
+        return False
+    raise RuntimeError(f"Environment variable {name} must be boolean, got: {raw}")
+
+
+def _normalize_database_url(raw_url: str) -> str:
+    normalized = raw_url.strip()
+    if normalized.startswith("jdbc:"):
+        normalized = normalized[len("jdbc:") :]
+    if normalized.startswith("postgres://"):
+        normalized = "postgresql+psycopg://" + normalized[len("postgres://") :]
+    elif normalized.startswith("postgresql://"):
+        normalized = "postgresql+psycopg://" + normalized[len("postgresql://") :]
+    return normalized
+
+
+@dataclass(frozen=True)
+class Settings:
+    database_url: str
+    celery_broker_url: str
+    celery_result_backend: str
+    iaai_base_url: str
+    iaai_listing_start_url: str
+    sync_runtime_config_file: Path
+    iaai_session_cookies: str
+    iaai_storage_state_path: Path
+    iaai_login: str
+    iaai_password: str
+    http_timeout: int
+    http_retries: int
+    http_retry_backoff_ms: int
+    fetch_concurrency: int
+    db_commit_batch_size: int
+    session_keepalive_enabled: bool
+    session_keepalive_interval_minutes: int
+    schema_bootstrap_enabled: bool
+    advisory_lock_key: int
+    error_summary_max_len: int
+
+    @classmethod
+    def from_env(cls) -> "Settings":
+        settings = cls(
+            database_url=_normalize_database_url(_read_env("DATABASE_URL", required=True)),
+            celery_broker_url=_read_env("CELERY_BROKER_URL", required=True),
+            celery_result_backend=_read_env("CELERY_RESULT_BACKEND", "redis://redis:6379/1"),
+            iaai_base_url=_read_env("IAAI_BASE_URL", "https://www.iaai.com").rstrip("/"),
+            iaai_listing_start_url=_read_env("IAAI_LISTING_START_URL", ""),
+            sync_runtime_config_file=Path(_read_env("SYNC_RUNTIME_CONFIG_FILE", "sync_runtime_config.json")),
+            iaai_session_cookies=_read_env("IAAI_SESSION_COOKIES", ""),
+            iaai_storage_state_path=Path(_read_env("IAAI_STORAGE_STATE_PATH", "iaai_storage_state.json")),
+            iaai_login=_read_env("IAAI_LOGIN", ""),
+            iaai_password=_read_env("IAAI_PASSWORD", ""),
+            http_timeout=_read_int("IAAI_HTTP_TIMEOUT", 30),
+            http_retries=_read_int("IAAI_HTTP_RETRIES", 2),
+            http_retry_backoff_ms=_read_int("IAAI_HTTP_RETRY_BACKOFF_MS", 700),
+            fetch_concurrency=_read_int("IAAI_FETCH_CONCURRENCY", 8),
+            db_commit_batch_size=_read_int("IAAI_DB_COMMIT_BATCH_SIZE", 20),
+            session_keepalive_enabled=_read_bool("IAAI_SESSION_KEEPALIVE_ENABLED", True),
+            session_keepalive_interval_minutes=_read_int("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES", 15),
+            schema_bootstrap_enabled=_read_bool("IAAI_SCHEMA_BOOTSTRAP_ENABLED", False),
+            advisory_lock_key=_read_int("SYNC_ADVISORY_LOCK_KEY", 7642200),
+            error_summary_max_len=_read_int("SYNC_ERROR_SUMMARY_MAX_LEN", 2000),
+        )
+
+        if settings.http_timeout <= 0:
+            raise RuntimeError("IAAI_HTTP_TIMEOUT must be greater than 0")
+        if settings.http_retries < 0:
+            raise RuntimeError("IAAI_HTTP_RETRIES cannot be negative")
+        if settings.http_retry_backoff_ms < 0:
+            raise RuntimeError("IAAI_HTTP_RETRY_BACKOFF_MS cannot be negative")
+        if settings.fetch_concurrency <= 0:
+            raise RuntimeError("IAAI_FETCH_CONCURRENCY must be greater than 0")
+        if settings.db_commit_batch_size <= 0:
+            raise RuntimeError("IAAI_DB_COMMIT_BATCH_SIZE must be greater than 0")
+        if settings.session_keepalive_interval_minutes <= 0:
+            raise RuntimeError("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES must be greater than 0")
+        if settings.session_keepalive_interval_minutes > 59:
+            raise RuntimeError("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES must be <= 59")
+        if settings.error_summary_max_len <= 0:
+            raise RuntimeError("SYNC_ERROR_SUMMARY_MAX_LEN must be greater than 0")
+        if settings.iaai_base_url.lower().startswith("http") is False:
+            raise RuntimeError("IAAI_BASE_URL must be absolute URL")
+        if settings.iaai_listing_start_url and not (
+            settings.iaai_listing_start_url.lower().startswith("http")
+            or settings.iaai_listing_start_url.startswith("/")
+        ):
+            raise RuntimeError("IAAI_LISTING_START_URL must be absolute URL or path starting with '/'")
+        return settings
+
+
+@lru_cache(maxsize=1)
+def get_settings() -> Settings:
+    return Settings.from_env()

iaai_sync_service/tasks.py

@@ -0,0 +1,213 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Any
+
+from celery.utils.log import get_task_logger
+from sqlalchemy import text
+from sqlalchemy.exc import ProgrammingError
+
+from iaai_sync_service.celery_app import app
+from iaai_sync_service.client import IAAIClient
+from iaai_sync_service.database import get_session
+from iaai_sync_service.models import SYNC_RUN_TABLE_NAME, SyncRun
+from iaai_sync_service.runtime_config import load_runtime_config
+from iaai_sync_service.schema_bootstrap import ensure_schema
+from iaai_sync_service.settings import get_settings
+from iaai_sync_service.sync_engine import run_sync_once
+
+logger = get_task_logger(__name__)
+
+SYNC_LANE = "iaai_cars"
+
+
+def utc_now() -> datetime:
+    return datetime.now(UTC)
+
+
+def trim_error_summary(errors: list[str], max_len: int) -> str | None:
+    if not errors:
+        return None
+    joined = " | ".join(errors)
+    if len(joined) <= max_len:
+        return joined
+    return joined[: max_len - 3] + "..."
+
+
+def _is_missing_sync_runs_error(exc: ProgrammingError) -> bool:
+    message = str(exc).lower()
+    return "does not exist" in message and SYNC_RUN_TABLE_NAME in message
+
+
+def _create_sync_run(session, lane: str) -> SyncRun:  # noqa: ANN001
+    run = SyncRun(
+        status="running",
+        lane=lane,
+        started_at=utc_now(),
+        ids_fetched=0,
+        cars_upserted=0,
+        cars_failed=0,
+        images_upserted=0,
+    )
+    session.add(run)
+    session.commit()
+    return run
+
+
+@app.task(name="iaai.sync_cars_feed", queue="iaai_sync", bind=True)
+def sync_cars_feed(self: Any) -> dict[str, Any]:
+    return execute_sync_job()
+
+
+@app.task(name="iaai.keep_session_alive", queue="iaai_sync", bind=True)
+def keep_session_alive(self: Any) -> dict[str, Any]:
+    return execute_keepalive_job()
+
+
+def execute_sync_job() -> dict[str, Any]:
+    settings = get_settings()
+    session = get_session()
+    run_id: int | None = None
+    lock_acquired = False
+
+    try:
+        logger.info("IAAI sync job started base_url=%s", settings.iaai_base_url)
+        if settings.schema_bootstrap_enabled:
+            ensure_schema(session)
+            session.commit()
+            logger.info("Schema bootstrap is enabled and completed.")
+        else:
+            logger.info("Schema bootstrap is disabled.")
+
+        lock_acquired = bool(
+            session.execute(
+                text("SELECT pg_try_advisory_lock(:key)"),
+                {"key": settings.advisory_lock_key},
+            ).scalar_one()
+        )
+        if not lock_acquired:
+            logger.info("IAAI sync skipped: advisory lock is already held.")
+            return {"status": "skipped_locked"}
+
+        try:
+            run = _create_sync_run(session=session, lane=SYNC_LANE)
+        except ProgrammingError as exc:
+            session.rollback()
+            if not _is_missing_sync_runs_error(exc):
+                raise
+            logger.warning(
+                "Table '%s' is missing. Running one-time schema bootstrap and retrying.",
+                SYNC_RUN_TABLE_NAME,
+            )
+            ensure_schema(session)
+            session.commit()
+            run = _create_sync_run(session=session, lane=SYNC_LANE)
+        run_id = run.id
+        logger.info("IAAI sync run created run_id=%s", run_id)
+
+        client = IAAIClient(settings=settings)
+        runtime_config = load_runtime_config(settings.sync_runtime_config_file, logger=logger)
+        logger.info("Loaded runtime config from '%s'", settings.sync_runtime_config_file)
+
+        stats, errors = run_sync_once(
+            session=session,
+            client=client,
+            settings=settings,
+            runtime_config=runtime_config,
+        )
+
+        run = session.get(SyncRun, run_id)
+        if run is None:
+            raise RuntimeError(f"sync_runs row disappeared for id={run_id}")
+        run.finished_at = utc_now()
+        run.ids_fetched = stats.ids_fetched
+        run.cars_upserted = stats.cars_upserted
+        run.cars_failed = stats.cars_failed
+        run.images_upserted = stats.images_upserted
+        run.status = "partial" if stats.cars_failed > 0 else "success"
+        run.error_summary = trim_error_summary(errors, settings.error_summary_max_len)
+        session.commit()
+
+        logger.info(
+            "IAAI sync completed status=%s ids=%s cars_upserted=%s cars_failed=%s images_upserted=%s",
+            run.status,
+            run.ids_fetched,
+            run.cars_upserted,
+            run.cars_failed,
+            run.images_upserted,
+        )
+        return {
+            "status": run.status,
+            "ids_fetched": run.ids_fetched,
+            "cars_upserted": run.cars_upserted,
+            "cars_failed": run.cars_failed,
+            "images_upserted": run.images_upserted,
+            "run_id": run.id,
+        }
+    except Exception as exc:  # noqa: BLE001
+        session.rollback()
+        if run_id is not None:
+            failed = session.get(SyncRun, run_id)
+            if failed is not None:
+                failed.finished_at = utc_now()
+                failed.status = "failed"
+                failed.error_summary = trim_error_summary([str(exc)], settings.error_summary_max_len)
+                session.commit()
+        logger.exception("IAAI sync failed: %s", exc)
+        raise
+    finally:
+        if lock_acquired:
+            try:
+                session.execute(
+                    text("SELECT pg_advisory_unlock(:key)"),
+                    {"key": settings.advisory_lock_key},
+                )
+                session.commit()
+            except Exception:  # noqa: BLE001
+                session.rollback()
+                logger.exception("Failed to release advisory lock.")
+        session.close()
+
+
+def execute_keepalive_job() -> dict[str, Any]:
+    settings = get_settings()
+    session = get_session()
+    lock_acquired = False
+
+    try:
+        lock_acquired = bool(
+            session.execute(
+                text("SELECT pg_try_advisory_lock(:key)"),
+                {"key": settings.advisory_lock_key},
+            ).scalar_one()
+        )
+        if not lock_acquired:
+            logger.info("IAAI keepalive skipped: advisory lock is already held.")
+            return {"status": "skipped_locked"}
+
+        logger.info("IAAI keepalive job started base_url=%s", settings.iaai_base_url)
+        client = IAAIClient(settings=settings)
+        result = client.keep_session_alive()
+        logger.info(
+            "IAAI keepalive completed scope=%s marker_present=%s result_count=%s elapsed_ms=%s",
+            result.get("scope"),
+            result.get("marker_present"),
+            result.get("result_count"),
+            result.get("elapsed_ms"),
+        )
+        return {"status": "success", **result}
+    except Exception as exc:  # noqa: BLE001
+        logger.exception("IAAI keepalive failed: %s", exc)
+        raise
+    finally:
+        if lock_acquired:
+            try:
+                session.execute(
+                    text("SELECT pg_advisory_unlock(:key)"),
+                    {"key": settings.advisory_lock_key},
+                )
+                session.commit()
+            except Exception:  # noqa: BLE001
+                session.rollback()
+                logger.exception("Failed to release advisory lock for keepalive.")
+        session.close()

pyproject.toml

@@ -0,0 +1,26 @@
+[build-system]
+requires = ["setuptools>=69", "wheel"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "iaai"
+version = "0.1.0"
+description = "IAAI Cars sync service"
+readme = "README.md"
+requires-python = ">=3.13"
+dependencies = [
+    "alembic>=1.17.1",
+    "celery[redis]>=5.5.3",
+    "playwright>=1.58.0",
+    "psycopg[binary]>=3.2.12",
+    "requests>=2.32.5",
+    "sqlalchemy>=2.0.43",
+]
+
+[project.optional-dependencies]
+dev = [
+    "pytest>=8.4.2",
+]
+
+[tool.setuptools.packages.find]
+include = ["iaai_sync_service*"]

sync_runtime_config.json

@@ -0,0 +1,11 @@
+{
+  "sync": {
+    "condition_check_enabled": false
+  },
+  "filters": {
+    "brands": [],
+    "models": [],
+    "years": [],
+    "body_types": []
+  }
+}

tests/conftest.py

@@ -0,0 +1,13 @@
+from __future__ import annotations
+
+import os
+import sys
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parents[1]
+if str(ROOT) not in sys.path:
+    sys.path.insert(0, str(ROOT))
+
+os.environ.setdefault("DATABASE_URL", "sqlite+pysqlite:///:memory:")
+os.environ.setdefault("CELERY_BROKER_URL", "redis://localhost:6379/0")
+os.environ.setdefault("IAAI_BASE_URL", "https://www.iaai.com")

tests/test_client.py

@@ -0,0 +1,231 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from iaai_sync_service.client import (
+    LISTING_MARKER,
+    HybridSessionAuth,
+    build_resizer_images_from_keys,
+    is_challenge_response,
+    parse_listing_page,
+    parse_product_details_vm,
+    resolve_listing_scope_paths,
+)
+from iaai_sync_service.settings import Settings
+
+
+def _catalog_html() -> str:
+    return (Path(__file__).resolve().parents[1] / "html" / "catalog.html").read_text(encoding="utf-8")
+
+
+def _page_html() -> str:
+    return (Path(__file__).resolve().parents[1] / "html" / "page.html").read_text(encoding="utf-8")
+
+
+def test_parse_listing_page_extracts_hidden_payloads() -> None:
+    parsed = parse_listing_page(_catalog_html())
+
+    assert parsed.result_count >= len(parsed.vehicles)
+    assert parsed.page_size == 100
+    assert parsed.current_page == 1
+    assert parsed.vehicles
+    first = parsed.vehicles[0]
+    assert first.inventory_id.endswith("~US")
+    assert first.inventory_status == "RS"
+
+
+def test_parse_product_details_vm_extracts_json_script() -> None:
+    payload = parse_product_details_vm(_page_html())
+    inventory_view = payload["inventoryView"]
+    attrs = inventory_view["attributes"]
+
+    assert attrs["Id"] == "45078011~US"
+    assert attrs["Make"] == "ACURA"
+    assert attrs["Model"] == "RSX"
+
+
+def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None:
+    keys = [
+        {"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
+        {"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
+        {"k": "abc~2", "w": 1800, "h": 1200, "i": 1},
+    ]
+    images = build_resizer_images_from_keys(keys)
+
+    assert len(images) == 2
+    assert images[0]["order_index"] == 1
+    assert "imageKeys=abc~2" in str(images[0]["fullres_image"])
+
+
+def test_is_challenge_response_detects_missing_marker() -> None:
+    html = "<html><body><h1>Blocked</h1></body></html>"
+    assert is_challenge_response(status_code=200, body_text=html, expected_marker='id="GBPSearchQuery"') is True
+
+
+def test_is_challenge_response_detects_incapsula_marker() -> None:
+    html = '<html><script>var a="_Incapsula_Resource";</script></html>'
+    assert is_challenge_response(status_code=200, body_text=html, expected_marker=None) is True
+
+
+def test_is_challenge_response_accepts_expected_marker_even_with_incapsula_script() -> None:
+    html = f'<html><input id="GBPSearchQuery" value="{{}}" /><script src="/_Incapsula_Resource"></script></html>'
+    assert is_challenge_response(status_code=200, body_text=html, expected_marker=LISTING_MARKER) is False
+
+
+def test_catalog_hidden_vehicle_json_is_valid() -> None:
+    parsed = parse_listing_page(_catalog_html())
+    # smoke-check we can serialize parsed query back to JSON
+    dumped = json.dumps(parsed.gbp_search_query)
+    assert "CurrentPage" in dumped
+
+
+def test_resolve_listing_scope_paths_prefers_explicit_url() -> None:
+    paths = resolve_listing_scope_paths(
+        listing_start_url="https://www.iaai.com/Search?url=abc",
+        brands={"acura", "bmw"},
+    )
+    assert paths == ["https://www.iaai.com/Search?url=abc"]
+
+
+class _FakeResponse:
+    def __init__(self, status_code: int, text: str) -> None:
+        self.status_code = status_code
+        self.text = text
+
+    def close(self) -> None:
+        return
+
+
+class _FakeSession:
+    def __init__(self, responses: list[_FakeResponse], *, cookies: list[object] | None = None) -> None:
+        self._responses = responses
+        self.calls = 0
+        self.cookies = cookies or []
+
+    def request(self, *args, **kwargs):  # noqa: ANN002, ANN003, ANN202
+        self.calls += 1
+        return self._responses.pop(0)
+
+
+def _settings() -> Settings:
+    return Settings(
+        database_url="sqlite+pysqlite:///:memory:",
+        celery_broker_url="redis://localhost:6379/0",
+        celery_result_backend="redis://localhost:6379/1",
+        iaai_base_url="https://www.iaai.com",
+        iaai_listing_start_url="",
+        sync_runtime_config_file=Path("sync_runtime_config.json"),
+        iaai_session_cookies="",
+        iaai_storage_state_path=Path("iaai_storage_state.json"),
+        iaai_login="",
+        iaai_password="",
+        http_timeout=10,
+        http_retries=0,
+        http_retry_backoff_ms=1,
+        fetch_concurrency=1,
+        db_commit_batch_size=10,
+        session_keepalive_enabled=True,
+        session_keepalive_interval_minutes=15,
+        schema_bootstrap_enabled=False,
+        advisory_lock_key=1,
+        error_summary_max_len=2000,
+    )
+
+
+def test_hybrid_auth_retries_after_challenge(monkeypatch) -> None:  # noqa: ANN001
+    auth = HybridSessionAuth(_settings())
+    cookie = type("Cookie", (), {"domain": ".iaai.com"})()
+    fake_session = _FakeSession(
+        [
+            _FakeResponse(403, "<html>forbidden</html>"),
+            _FakeResponse(200, f"<html>{LISTING_MARKER}</html>"),
+        ],
+        cookies=[cookie],
+    )
+    auth._thread_local.session = fake_session  # type: ignore[attr-defined]
+    auth._bootstrap_cookies_loaded = True  # type: ignore[attr-defined]
+
+    refreshed = {"count": 0}
+
+    def _fake_refresh(*, expected_marker=None, session=None) -> None:  # noqa: ANN001
+        refreshed["count"] += 1
+
+    monkeypatch.setattr(auth, "_refresh_session_via_playwright", _fake_refresh)
+
+    response = auth.request(
+        "GET",
+        "https://www.iaai.com/Vehiclelisting/Cars",
+        timeout=5,
+        retries=0,
+        retry_backoff_ms=1,
+        expected_marker=LISTING_MARKER,
+    )
+    assert response.status_code == 200
+    assert refreshed["count"] == 1
+    assert fake_session.calls == 2
+
+
+def test_hybrid_auth_bootstraps_anonymous_session_when_no_cookies(monkeypatch) -> None:  # noqa: ANN001
+    auth = HybridSessionAuth(_settings())
+    fake_session = _FakeSession(
+        [
+            _FakeResponse(200, f"<html>{LISTING_MARKER}</html>"),
+        ],
+        cookies=[],
+    )
+    auth._thread_local.session = fake_session  # type: ignore[attr-defined]
+    auth._bootstrap_cookies_loaded = True  # type: ignore[attr-defined]
+
+    refreshed = {"count": 0}
+
+    def _fake_refresh(*, expected_marker=None, session=None) -> None:  # noqa: ANN001
+        refreshed["count"] += 1
+
+    monkeypatch.setattr(auth, "_refresh_session_via_playwright", _fake_refresh)
+
+    response = auth.request(
+        "GET",
+        "https://www.iaai.com/Vehiclelisting/Cars",
+        timeout=5,
+        retries=0,
+        retry_backoff_ms=1,
+        expected_marker=LISTING_MARKER,
+    )
+    assert response.status_code == 200
+    assert refreshed["count"] == 1
+    assert fake_session.calls == 1
+
+
+def test_wait_until_non_challenge_tolerates_transient_navigation_errors() -> None:
+    class _FakePage:
+        def __init__(self) -> None:
+            self.content_calls = 0
+            self.goto_calls = 0
+
+        def wait_for_timeout(self, *_args, **_kwargs) -> None:  # noqa: ANN002, ANN003
+            return
+
+        def wait_for_load_state(self, *_args, **_kwargs) -> None:  # noqa: ANN002, ANN003
+            return
+
+        def content(self) -> str:
+            self.content_calls += 1
+            if self.content_calls == 1:
+                raise RuntimeError(
+                    "Page.content: Unable to retrieve content because the page is navigating and changing the content."
+                )
+            return f"<html>{LISTING_MARKER}</html>"
+
+        def goto(self, *_args, **_kwargs) -> None:  # noqa: ANN002, ANN003
+            self.goto_calls += 1
+
+    page = _FakePage()
+    HybridSessionAuth._wait_until_non_challenge(
+        page=page,
+        target="https://www.iaai.com/Vehiclelisting/Cars",
+        timeout_ms=5_000,
+        expected_marker=LISTING_MARKER,
+    )
+    assert page.content_calls >= 2
+    assert page.goto_calls == 0

tests/test_migrations.py

@@ -0,0 +1,36 @@
+from __future__ import annotations
+
+import importlib.util
+from pathlib import Path
+
+
+def _load_module(path: Path):  # noqa: ANN202
+    spec = importlib.util.spec_from_file_location(path.stem, path)
+    assert spec is not None
+    assert spec.loader is not None
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    return module
+
+
+def test_revision_chain_is_correct() -> None:
+    base = Path(__file__).resolve().parents[1] / "alembic" / "versions"
+    rev1 = _load_module(base / "0001_ensure_sync_schema.py")
+    rev2 = _load_module(base / "0002_extend_origin_enum_for_iaai.py")
+    rev3 = _load_module(base / "0003_create_iaai_source_tables.py")
+
+    assert rev1.revision == "0001_ensure_sync_schema"
+    assert rev1.down_revision is None
+    assert rev2.revision == "0002_extend_origin_enum_for_iaai"
+    assert rev2.down_revision == "0001_ensure_sync_schema"
+    assert rev3.revision == "0003_create_iaai_source_tables"
+    assert rev3.down_revision == "0002_extend_origin_enum_for_iaai"
+
+
+def test_enum_migration_contains_required_values() -> None:
+    migration = (
+        Path(__file__).resolve().parents[1] / "alembic" / "versions" / "0002_extend_origin_enum_for_iaai.py"
+    ).read_text(encoding="utf-8")
+    assert "originenum" in migration
+    assert "IAAI" in migration
+    assert "ADD VALUE IF NOT EXISTS" in migration

tests/test_runtime_config.py

@@ -0,0 +1,40 @@
+from __future__ import annotations
+
+import json
+
+from iaai_sync_service.runtime_config import RuntimeConfig, load_runtime_config
+
+
+def test_load_runtime_config_reads_filters(tmp_path) -> None:  # noqa: ANN001
+    config_path = tmp_path / "sync_runtime_config.json"
+    config_path.write_text(
+        json.dumps(
+            {
+                "sync": {
+                    "condition_check_enabled": "true",
+                },
+                "filters": {
+                    "brands": ["Acura", "BMW"],
+                    "models": ["RSX", "3 SERIES"],
+                    "years": [2021, "2022"],
+                    "body_types": ["SUV"],
+                },
+            }
+        ),
+        encoding="utf-8",
+    )
+
+    runtime = load_runtime_config(config_path)
+    assert runtime.condition_check_enabled is True
+    assert runtime.filters.brands == {"acura", "bmw"}
+    assert runtime.filters.models == {"rsx", "3 series"}
+    assert runtime.filters.years == {2021, 2022}
+    assert runtime.filters.body_types == {"suv"}
+
+
+def test_load_runtime_config_returns_defaults_on_invalid_json(tmp_path) -> None:  # noqa: ANN001
+    config_path = tmp_path / "sync_runtime_config.json"
+    config_path.write_text("{invalid", encoding="utf-8")
+
+    runtime = load_runtime_config(config_path)
+    assert runtime == RuntimeConfig()

tests/test_settings.py

@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import pytest
+
+from iaai_sync_service.settings import Settings
+
+
+def test_settings_from_env_reads_required_values(monkeypatch) -> None:  # noqa: ANN001
+    monkeypatch.setenv("DATABASE_URL", "postgres://user:pass@localhost:5432/db")
+    monkeypatch.setenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
+    monkeypatch.setenv("IAAI_BASE_URL", "https://www.iaai.com")
+    monkeypatch.setenv("IAAI_LISTING_START_URL", "https://www.iaai.com/Search?url=abc")
+    monkeypatch.setenv("IAAI_FETCH_CONCURRENCY", "4")
+    monkeypatch.setenv("IAAI_SESSION_KEEPALIVE_ENABLED", "false")
+    monkeypatch.setenv("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES", "20")
+    monkeypatch.setenv("IAAI_SCHEMA_BOOTSTRAP_ENABLED", "true")
+
+    settings = Settings.from_env()
+
+    assert settings.database_url == "postgresql+psycopg://user:pass@localhost:5432/db"
+    assert settings.celery_broker_url == "redis://localhost:6379/0"
+    assert settings.iaai_base_url == "https://www.iaai.com"
+    assert settings.iaai_listing_start_url == "https://www.iaai.com/Search?url=abc"
+    assert settings.fetch_concurrency == 4
+    assert settings.session_keepalive_enabled is False
+    assert settings.session_keepalive_interval_minutes == 20
+    assert settings.schema_bootstrap_enabled is True
+
+
+def test_settings_rejects_non_positive_concurrency(monkeypatch) -> None:  # noqa: ANN001
+    monkeypatch.setenv("DATABASE_URL", "postgresql://user:pass@localhost:5432/db")
+    monkeypatch.setenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
+    monkeypatch.setenv("IAAI_BASE_URL", "https://www.iaai.com")
+    monkeypatch.setenv("IAAI_FETCH_CONCURRENCY", "0")
+
+    with pytest.raises(RuntimeError, match="IAAI_FETCH_CONCURRENCY"):
+        Settings.from_env()
+
+
+def test_settings_bootstrap_flag_defaults_to_false(monkeypatch) -> None:  # noqa: ANN001
+    monkeypatch.setenv("DATABASE_URL", "postgresql://user:pass@localhost:5432/db")
+    monkeypatch.setenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
+    monkeypatch.setenv("IAAI_BASE_URL", "https://www.iaai.com")
+
+    settings = Settings.from_env()
+    assert settings.schema_bootstrap_enabled is False
+
+
+def test_settings_rejects_invalid_listing_start_url(monkeypatch) -> None:  # noqa: ANN001
+    monkeypatch.setenv("DATABASE_URL", "postgresql://user:pass@localhost:5432/db")
+    monkeypatch.setenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
+    monkeypatch.setenv("IAAI_BASE_URL", "https://www.iaai.com")
+    monkeypatch.setenv("IAAI_LISTING_START_URL", "Search?url=abc")
+
+    with pytest.raises(RuntimeError, match="IAAI_LISTING_START_URL"):
+        Settings.from_env()
+
+
+def test_settings_rejects_invalid_keepalive_interval(monkeypatch) -> None:  # noqa: ANN001
+    monkeypatch.setenv("DATABASE_URL", "postgresql://user:pass@localhost:5432/db")
+    monkeypatch.setenv("CELERY_BROKER_URL", "redis://localhost:6379/0")
+    monkeypatch.setenv("IAAI_BASE_URL", "https://www.iaai.com")
+    monkeypatch.setenv("IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES", "0")
+
+    with pytest.raises(RuntimeError, match="IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES"):
+        Settings.from_env()

tests/test_sync_engine.py

@@ -0,0 +1,288 @@
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from pathlib import Path
+
+from sqlalchemy import create_engine
+from sqlalchemy.orm import Session
+
+from iaai_sync_service.client import ListingVehicle
+from iaai_sync_service.models import Base, Car, Image
+from iaai_sync_service.runtime_config import RuntimeConfig, RuntimeFilters
+from iaai_sync_service.settings import Settings
+from iaai_sync_service.sync_engine import (
+    prepare_car,
+    replace_images_if_changed,
+    run_sync_once,
+)
+
+
+def _session() -> Session:
+    engine = create_engine("sqlite+pysqlite:///:memory:", future=True)
+    Base.metadata.create_all(engine)
+    return Session(bind=engine)
+
+
+def _settings() -> Settings:
+    return Settings(
+        database_url="sqlite+pysqlite:///:memory:",
+        celery_broker_url="redis://localhost:6379/0",
+        celery_result_backend="redis://localhost:6379/1",
+        iaai_base_url="https://www.iaai.com",
+        iaai_listing_start_url="",
+        sync_runtime_config_file=Path("sync_runtime_config.json"),
+        iaai_session_cookies="",
+        iaai_storage_state_path=Path("iaai_storage_state.json"),
+        iaai_login="",
+        iaai_password="",
+        http_timeout=10,
+        http_retries=1,
+        http_retry_backoff_ms=1,
+        fetch_concurrency=2,
+        db_commit_batch_size=20,
+        session_keepalive_enabled=True,
+        session_keepalive_interval_minutes=15,
+        schema_bootstrap_enabled=False,
+        advisory_lock_key=1,
+        error_summary_max_len=2000,
+    )
+
+
+def _listing_vehicle(inventory_id: str, *, status: str = "RS", closed: bool = False) -> ListingVehicle:
+    return ListingVehicle(
+        inventory_id=inventory_id,
+        tenant="US",
+        auction_id="1_1",
+        auction_date="2026-04-08T08:30:00+00:00",
+        inventory_status=status,
+        currency="USD",
+        timed_auction_closed=closed,
+        timed_auction_indicator=False,
+        prebid_indicator=True,
+        buynow_indicator=False,
+    )
+
+
+def _detail_payload(
+    inventory_id: str,
+    *,
+    make: str = "ACURA",
+    model: str = "RSX",
+    series: str = "BASE",
+    year: int = 2002,
+    bid: int = 500,
+) -> dict:
+    return {
+        "inventoryView": {
+            "attributes": {
+                "Id": inventory_id,
+                "Year": str(year),
+                "Make": make,
+                "Model": model,
+                "Series": series,
+                "Currency": "USD",
+                "ODOValue": "219187",
+                "ExteriorColor": "GRAY",
+                "DriveLineTypeDesc": "FWD",
+                "Transmission": "Automatic",
+                "BodyStyleName": "COUPE",
+                "EngineSize": "2.0L I-4",
+                "VehicleGrade": "50",
+                "PrimaryDamageDesc": "NORMAL WEAR & TEAR",
+                "SecondaryDamageDesc": " ",
+            },
+            "imageDimensions": {
+                "keys": {
+                    "$values": [
+                        {"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0},
+                        {"k": f"{inventory_id}~I2", "w": 1600, "h": 1200, "i": 1},
+                    ]
+                }
+            },
+        },
+        "auctionInformation": {
+            "biddingInformation": {
+                "buyNowAmount": 0,
+                "buyNowPrice": "$0",
+            },
+            "prebidInformation": {
+                "decimalHighBidAmount": str(bid),
+                "highBidAmount": f"${bid}",
+                "buyNowPrice": "$0",
+            },
+        },
+    }
+
+
+class FakeClient:
+    def __init__(self, listing: list[ListingVehicle], detail_payloads: dict[str, dict]) -> None:
+        self._listing = listing
+        self._detail_payloads = detail_payloads
+        self.detail_calls = 0
+        self.persist_calls = 0
+
+    def iter_listing_vehicles(self, filters: RuntimeFilters):  # noqa: ANN201, ARG002
+        return iter(self._listing)
+
+    def fetch_vehicle_detail_payload(self, inventory_id: str):  # noqa: ANN201
+        self.detail_calls += 1
+        return self._detail_payloads[inventory_id]
+
+    def persist_session_state(self) -> None:
+        self.persist_calls += 1
+
+
+def test_prepare_car_maps_fields() -> None:
+    row = prepare_car(
+        listing_vehicle=_listing_vehicle("45078011~US"),
+        detail_payload=_detail_payload("45078011~US", bid=600),
+        currency_labels={"USD", "CAD", "NA"},
+        country_labels={"US", "CA", "NA"},
+        drive_labels={"FWD", "RWD", "4WD", "NA"},
+        gearbox_labels={"AT", "MT", "NA"},
+        body_type_labels={"COUPE", "OTHER"},
+        body_type_default="OTHER",
+        steering_left="LEFT",
+        selling_type_auction="AUCTION",
+        origin_code="IAAI",
+    )
+
+    assert row.origin_id == "iaai:45078011~US"
+    assert row.origin_url == "https://www.iaai.com/VehicleDetail/45078011~US"
+    assert row.brand == "ACURA"
+    assert row.model == "RSX BASE"
+    assert row.year == 2002
+    assert row.price == 600
+    assert row.currency == "USD"
+    assert row.country == "US"
+    assert row.drive == "FWD"
+    assert row.gearbox == "AT"
+    assert row.body_type == "COUPE"
+    assert row.engine_volume == 2000
+    assert row.is_damaged is False
+    assert len(row.images) == 2
+
+
+def test_replace_images_if_changed_returns_zero_for_same_set() -> None:
+    session = _session()
+    car = Car(
+        parser_id="car-aaaaaaaaaaaaaaaaaaaaaa",
+        brand="ACURA",
+        model="RSX",
+        origin_id="iaai:45078011~US",
+        origin_url="https://www.iaai.com/VehicleDetail/45078011~US",
+        slug="acura-rsx-2002",
+        is_sold=False,
+        last_seen_at=datetime(2026, 4, 1, 12, 0, tzinfo=UTC),
+    )
+    session.add(car)
+    session.flush()
+    session.add(
+        Image(
+            car_id=car.id,
+            fullres_image="https://vis.iaai.com/resizer?imageKeys=a&width=1600&height=1200",
+            preview_image="https://vis.iaai.com/resizer?imageKeys=a&width=640&height=480",
+            order_index=0,
+        )
+    )
+    session.commit()
+
+    written = replace_images_if_changed(
+        session=session,
+        car_id=car.id,
+        images=[
+            {
+                "order_index": 0,
+                "fullres_image": "https://vis.iaai.com/resizer?imageKeys=a&width=1600&height=1200",
+                "preview_image": "https://vis.iaai.com/resizer?imageKeys=a&width=640&height=480",
+            }
+        ],
+    )
+
+    assert written == 0
+
+
+def test_run_sync_once_upserts_and_marks_missing_as_sold() -> None:
+    session = _session()
+
+    listing_first = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
+    detail_first = {
+        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
+        "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
+    }
+    client_first = FakeClient(listing_first, detail_first)
+
+    stats1, errors1 = run_sync_once(session=session, client=client_first, settings=_settings())
+    assert errors1 == []
+    assert stats1.ids_fetched == 2
+    assert stats1.cars_upserted == 2
+    assert client_first.persist_calls == 1
+
+    listing_second = [_listing_vehicle("45078011~US")]
+    detail_second = {
+        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=650),
+    }
+    client_second = FakeClient(listing_second, detail_second)
+
+    stats2, errors2 = run_sync_once(session=session, client=client_second, settings=_settings())
+    assert errors2 == []
+    assert stats2.ids_fetched == 1
+    assert stats2.cars_upserted == 1
+
+    active = session.query(Car).filter_by(origin_id="iaai:45078011~US").one()
+    sold = session.query(Car).filter_by(origin_id="iaai:45268167~US").one()
+    assert active.is_sold is False
+    assert active.price == 650
+    assert sold.is_sold is True
+
+
+def test_run_sync_once_skips_reconcile_when_runtime_filters_enabled() -> None:
+    session = _session()
+    session.add(
+        Car(
+            parser_id="car-zzzzzzzzzzzzzzzzzzzzzz",
+            brand="BMW",
+            model="X5",
+            origin_id="iaai:99999999~US",
+            origin_url="https://www.iaai.com/VehicleDetail/99999999~US",
+            slug="bmw-x5-2020",
+            is_sold=False,
+            last_seen_at=datetime(2026, 4, 1, 12, 0, tzinfo=UTC),
+        )
+    )
+    session.commit()
+
+    listing = [_listing_vehicle("45078011~US")]
+    details = {"45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500)}
+    runtime = RuntimeConfig(filters=RuntimeFilters(brands={"bmw"}))
+
+    stats, errors = run_sync_once(
+        session=session,
+        client=FakeClient(listing, details),
+        settings=_settings(),
+        runtime_config=runtime,
+    )
+
+    assert errors == []
+    assert stats.ids_fetched == 0
+
+    row = session.query(Car).filter_by(origin_id="iaai:99999999~US").one()
+    assert row.is_sold is False
+
+
+def test_run_sync_once_condition_check_skips_closed_rows() -> None:
+    session = _session()
+    listing = [_listing_vehicle("45078011~US", status="SOLD", closed=True)]
+    details = {"45078011~US": _detail_payload("45078011~US", bid=500)}
+
+    runtime = RuntimeConfig(condition_check_enabled=True)
+    stats, errors = run_sync_once(
+        session=session,
+        client=FakeClient(listing, details),
+        settings=_settings(),
+        runtime_config=runtime,
+    )
+
+    assert errors == []
+    assert stats.ids_fetched == 0
+    assert session.query(Car).count() == 0

tests/test_tasks.py

@@ -0,0 +1,167 @@
+from __future__ import annotations
+
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+
+from iaai_sync_service.models import SyncRun
+from iaai_sync_service.sync_engine import SyncStats
+from iaai_sync_service.tasks import execute_keepalive_job, execute_sync_job
+
+
+class _ScalarResult:
+    def __init__(self, value):  # noqa: ANN001
+        self._value = value
+
+    def scalar_one(self):  # noqa: ANN201
+        return self._value
+
+
+class FakeSession:
+    def __init__(self, *, lock_acquired: bool) -> None:
+        self.lock_acquired = lock_acquired
+        self.runs: dict[int, SyncRun] = {}
+        self.closed = False
+
+    def execute(self, stmt, params=None):  # noqa: ANN001, ANN201
+        sql = str(stmt)
+        if "pg_try_advisory_lock" in sql:
+            return _ScalarResult(self.lock_acquired)
+        if "pg_advisory_unlock" in sql:
+            return _ScalarResult(True)
+        raise AssertionError(f"Unexpected SQL execute call: {sql}")
+
+    def add(self, obj) -> None:  # noqa: ANN001
+        if isinstance(obj, SyncRun):
+            if obj.id is None:
+                obj.id = len(self.runs) + 1
+            self.runs[obj.id] = obj
+
+    def commit(self) -> None:
+        return
+
+    def rollback(self) -> None:
+        return
+
+    def get(self, model, run_id):  # noqa: ANN001, ANN201
+        assert model is SyncRun
+        return self.runs.get(run_id)
+
+    def close(self) -> None:
+        self.closed = True
+
+
+def _settings() -> SimpleNamespace:
+    return SimpleNamespace(
+        iaai_base_url="https://www.iaai.com",
+        sync_runtime_config_file=Path("sync_runtime_config.json"),
+        schema_bootstrap_enabled=True,
+        advisory_lock_key=123,
+        error_summary_max_len=2000,
+    )
+
+
+def test_execute_sync_job_skips_when_lock_not_available(monkeypatch) -> None:  # noqa: ANN001
+    session = FakeSession(lock_acquired=False)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_settings", _settings)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_session", lambda: session)
+    monkeypatch.setattr("iaai_sync_service.tasks.ensure_schema", lambda _session: None)
+
+    result = execute_sync_job()
+
+    assert result == {"status": "skipped_locked"}
+    assert session.closed is True
+
+
+def test_execute_sync_job_success_writes_sync_run(monkeypatch) -> None:  # noqa: ANN001
+    session = FakeSession(lock_acquired=True)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_settings", _settings)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_session", lambda: session)
+    monkeypatch.setattr("iaai_sync_service.tasks.ensure_schema", lambda _session: None)
+    monkeypatch.setattr(
+        "iaai_sync_service.tasks.run_sync_once",
+        lambda **kwargs: (SyncStats(ids_fetched=5, cars_upserted=4, cars_failed=0, images_upserted=12), []),
+    )
+
+    result = execute_sync_job()
+
+    assert result["status"] == "success"
+    assert result["ids_fetched"] == 5
+    run = session.runs[result["run_id"]]
+    assert run.status == "success"
+    assert run.cars_upserted == 4
+
+
+def test_execute_sync_job_marks_failed_when_sync_raises(monkeypatch) -> None:  # noqa: ANN001
+    session = FakeSession(lock_acquired=True)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_settings", _settings)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_session", lambda: session)
+    monkeypatch.setattr("iaai_sync_service.tasks.ensure_schema", lambda _session: None)
+
+    def _raise(**kwargs):  # noqa: ANN003, ANN202
+        raise RuntimeError("boom")
+
+    monkeypatch.setattr("iaai_sync_service.tasks.run_sync_once", _raise)
+
+    with pytest.raises(RuntimeError, match="boom"):
+        execute_sync_job()
+
+    failed = session.runs[1]
+    assert failed.status == "failed"
+    assert failed.error_summary is not None
+
+
+def test_execute_sync_job_skips_schema_bootstrap_when_disabled(monkeypatch) -> None:  # noqa: ANN001
+    session = FakeSession(lock_acquired=False)
+    settings = _settings()
+    settings.schema_bootstrap_enabled = False
+
+    monkeypatch.setattr("iaai_sync_service.tasks.get_settings", lambda: settings)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_session", lambda: session)
+
+    def _fail_if_called(_session):  # noqa: ANN001, ANN202
+        raise AssertionError("ensure_schema should not be called")
+
+    monkeypatch.setattr("iaai_sync_service.tasks.ensure_schema", _fail_if_called)
+
+    result = execute_sync_job()
+    assert result == {"status": "skipped_locked"}
+
+
+def test_execute_keepalive_job_skips_when_lock_not_available(monkeypatch) -> None:  # noqa: ANN001
+    session = FakeSession(lock_acquired=False)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_settings", _settings)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_session", lambda: session)
+
+    result = execute_keepalive_job()
+
+    assert result == {"status": "skipped_locked"}
+    assert session.closed is True
+
+
+def test_execute_keepalive_job_success(monkeypatch) -> None:  # noqa: ANN001
+    session = FakeSession(lock_acquired=True)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_settings", _settings)
+    monkeypatch.setattr("iaai_sync_service.tasks.get_session", lambda: session)
+
+    class _FakeClient:
+        def __init__(self, settings):  # noqa: ANN001
+            self._settings = settings
+
+        def keep_session_alive(self):  # noqa: ANN201
+            return {
+                "scope": "https://www.iaai.com/Search?url=abc",
+                "marker_present": True,
+                "result_count": 1234,
+                "elapsed_ms": 456,
+            }
+
+    monkeypatch.setattr("iaai_sync_service.tasks.IAAIClient", _FakeClient)
+
+    result = execute_keepalive_job()
+
+    assert result["status"] == "success"
+    assert result["marker_present"] is True
+    assert result["result_count"] == 1234
+    assert session.closed is True