Add tools

Reviewed-on: #1

.env.example

@@ -6,7 +6,9 @@ IAAI_MAX_CAPTURED_REQUESTS=40
 IAAI_MAX_CAPTURED_JSON_RESPONSES=20
 
 IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
-IAAI_LISTING_SEGMENTS=auto
+IAAI_FILTERED_SEARCH_URL=
+IAAI_FILTERED_SEARCH_URLS=
+IAAI_LISTING_SEGMENTS=runtime
 IAAI_MAX_PAGES_PER_RUN=999999
 IAAI_MAX_VEHICLES_PER_RUN=999999
 IAAI_PAGE_LINK_LIMIT=999999
@@ -55,3 +57,6 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=1
 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
 CELERY_BEAT_SYNC_LIMIT=0
+IAAI_ALWAYS_FULL_SCAN=true
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600

.gitignore

@@ -19,4 +19,6 @@ build/
 artifacts/
 celerybeat-schedule*
 tokens_data/
-tokens.json
+tokens.json
+iaai_deploy.tar.gz
+.tmp_iaai_fast_ref/

Dockerfile

@@ -13,20 +13,20 @@ RUN pip install --no-cache-dir uv \
     && pip install --no-cache-dir -r /tmp/requirements.txt \
     && pip install --no-cache-dir playwright-stealth
 
-# Install both Chromium and Firefox. Chromium is the default engine in Docker
+# Ставим Chromium и Firefox.
-# because it works more reliably with the current IAAI listing page.
+# По умолчанию используем Chromium.
 RUN python -m playwright install chromium firefox
 
 COPY . .
 RUN pip install --no-cache-dir -e .
 RUN python -m compileall -q iaai_scraper
 RUN chmod +x entrypoint.sh
-RUN chown -R app:app /app
+RUN mkdir -p /data && chown -R app:app /app /data
 
 STOPSIGNAL SIGINT
 
 USER app
 
-# По умолчанию  API, но worker/beat переопределяют CMD в docker-compose
+# По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
 CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

README.md

@@ -124,9 +124,10 @@ docker compose ps
 - `worker` имеет healthcheck через `celery inspect ping`
 - `beat` имеет healthcheck по файлу `celerybeat-schedule`
 
+
 ## API
 
-**Здоровье и статистика:**
+**Статистика:**
 - `GET /health` — статус сервиса и подключения к БД
 - `GET /api/v1/stats` — сколько машин и картинок в базе, топ брендов
 
@@ -242,11 +243,6 @@ pytest -q
 
 ## `pyproject.toml` + `uv.lock`
 
-Проект переведён на современную схему зависимостей:
-
-- `pyproject.toml` — декларация зависимостей и метаданных проекта
-- `uv.lock` — зафиксированные версии для воспроизводимых установок
-
 Локально можно использовать:
 
 ```bash
@@ -254,10 +250,6 @@ uv sync
 uv run pytest -q
 ```
 
-## Runtime-фильтры
-
-Файл `runtime_config.json` управляет runtime-поведением sync и фильтрацией автомобилей.
-
 ### Секция `sync`
 
 Поддерживаются поля:

alembic/env.py

@@ -14,6 +14,7 @@ from iaai_scraper.core.config import Settings
 from iaai_scraper.storage.models import Base
 
 config = context.config
+VERSION_TABLE = "iaai_parser_alembic_version"
 
 # Logging
 if config.config_file_name is not None:
@@ -32,6 +33,7 @@ def run_migrations_offline() -> None:
     context.configure(
         url=url,
         target_metadata=target_metadata,
+        version_table=VERSION_TABLE,
         literal_binds=True,
         dialect_opts={"paramstyle": "named"},
     )
@@ -47,7 +49,11 @@ def run_migrations_online() -> None:
         poolclass=pool.NullPool,
     )
     with connectable.connect() as connection:
-        context.configure(connection=connection, target_metadata=target_metadata)
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table=VERSION_TABLE,
+        )
         with context.begin_transaction():
             context.run_migrations()
 

alembic/versions/001_initial.py

@@ -1,4 +1,4 @@
-# Начальная схема: cars, images, sync_runs
+# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs
 from typing import Sequence, Union
 
 from alembic import op
@@ -10,70 +10,94 @@ branch_labels: Union[str, Sequence[str], None] = None
 depends_on: Union[str, Sequence[str], None] = None
 
 
+def _schema_name() -> str | None:
+    bind = op.get_bind()
+    return "public" if bind.dialect.name == "postgresql" else None
+
+
+def _table_exists(table_name: str) -> bool:
+    inspector = sa.inspect(op.get_bind())
+    return table_name in inspector.get_table_names(schema=_schema_name())
+
+
+def _index_exists(table_name: str, index_name: str) -> bool:
+    if not _table_exists(table_name):
+        return False
+    inspector = sa.inspect(op.get_bind())
+    indexes = inspector.get_indexes(table_name, schema=_schema_name())
+    return any(index.get("name") == index_name for index in indexes)
+
+
 def upgrade() -> None:
-    # Таблица cars — аукционные машины с IAAI
+    # Таблица iaai_cars — аукционные машины с IAAI
-    op.create_table(
+    if not _table_exists("iaai_cars"):
-        "cars",
+        op.create_table(
-        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+            "iaai_cars",
-        sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
+            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
-        sa.Column("brand", sa.String(50), nullable=False),
+            sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
-        sa.Column("model", sa.String(50), nullable=False),
+            sa.Column("brand", sa.String(50), nullable=False),
-        sa.Column("year", sa.Integer, nullable=True),
+            sa.Column("model", sa.String(50), nullable=False),
-        sa.Column("price", sa.BigInteger, nullable=True),
+            sa.Column("year", sa.Integer, nullable=True),
-        sa.Column("currency", sa.String(10), nullable=False, server_default="USD"),
+            sa.Column("price", sa.BigInteger, nullable=True),
-        sa.Column("mileage", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("currency", sa.String(10), nullable=False, server_default="USD"),
-        sa.Column("country", sa.String(10), nullable=False, server_default="NA"),
+            sa.Column("mileage", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("country", sa.String(10), nullable=False, server_default="NA"),
-        sa.Column("color", sa.String, nullable=False, server_default="other"),
+            sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("drive", sa.String(10), nullable=True),
+            sa.Column("color", sa.String, nullable=False, server_default="other"),
-        sa.Column("gearbox", sa.String(10), nullable=True),
+            sa.Column("drive", sa.String(10), nullable=True),
-        sa.Column("steering_wheel", sa.String(10), nullable=True),
+            sa.Column("gearbox", sa.String(10), nullable=True),
-        sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"),
+            sa.Column("steering_wheel", sa.String(10), nullable=True),
-        sa.Column("engine_volume", sa.Integer, nullable=True),
+            sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"),
-        sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"),
+            sa.Column("engine_volume", sa.Integer, nullable=True),
-        sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"),
-        sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("origin", sa.String(20), nullable=False, server_default="NA"),
+            sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("origin_url", sa.String, nullable=False),
+            sa.Column("origin", sa.String(20), nullable=False, server_default="NA"),
-        sa.Column("origin_id", sa.String, nullable=False, unique=True),
+            sa.Column("origin_url", sa.String, nullable=False),
-        sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("origin_id", sa.String, nullable=False, unique=True),
-        sa.Column("evaluation", sa.String, nullable=True),
+            sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")),
+            sa.Column("evaluation", sa.String, nullable=True),
-        sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")),
-        sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("slug", sa.String, nullable=False),
+            sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
-        sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+            sa.Column("slug", sa.String, nullable=False),
-    )
+            sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
-    op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
+        )
-    op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
 
-    # Таблица images — изображения машин
+    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"):
-    op.create_table(
+        op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"])
-        "images",
+    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"):
-        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+        op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True)
-        sa.Column("fullres_image", sa.String, nullable=False),
+
-        sa.Column("preview_image", sa.String, nullable=False),
+    # Таблица iaai_images — изображения машин
-        sa.Column("order_index", sa.Integer, nullable=False),
+    if not _table_exists("iaai_images"):
-        sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
+        op.create_table(
-    )
+            "iaai_images",
+            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+            sa.Column("fullres_image", sa.String, nullable=False),
+            sa.Column("preview_image", sa.String, nullable=False),
+            sa.Column("order_index", sa.Integer, nullable=False),
+            sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False),
+        )
+
+    # Таблица iaai_sync_runs — логирование синхронизаций
+    if not _table_exists("iaai_sync_runs"):
+        op.create_table(
+            "iaai_sync_runs",
+            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+            sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+            sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("status", sa.Text, nullable=False),
+            sa.Column("lane", sa.Text, nullable=False),
+            sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("error_summary", sa.Text, nullable=True),
+        )
 
-    # Таблица sync_runs — логирование синхронизаций
-    op.create_table(
-        "sync_runs",
-        sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
-        sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
-        sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
-        sa.Column("status", sa.Text, nullable=False),
-        sa.Column("lane", sa.Text, nullable=False),
-        sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"),
-        sa.Column("error_summary", sa.Text, nullable=True),
-    )
 
 def downgrade() -> None:
-    op.drop_table("sync_runs")
+    # Compatibility-only migration for shared production databases.
-    op.drop_table("images")
+    pass
-    op.drop_table("cars")

alembic/versions/002_add_indexes.py

@@ -10,20 +10,15 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    op.create_index("ix_cars_brand", "cars", ["brand"])
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)")
-    op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)")
-    op.create_index("ix_cars_year", "cars", ["year"])
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)")
-    op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)")
-    op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)")
-    op.create_index("ix_images_car_id", "images", ["car_id"])
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)")
-    op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)")
 
 
 def downgrade() -> None:
-    op.drop_index("ix_sync_runs_status", table_name="sync_runs")
+    # Compatibility-only migration for shared production databases.
-    op.drop_index("ix_images_car_id", table_name="images")
+    pass
-    op.drop_index("ix_cars_last_seen_at", table_name="cars")
-    op.drop_index("ix_cars_is_sold", table_name="cars")
-    op.drop_index("ix_cars_year", table_name="cars")
-    op.drop_index("ix_cars_brand_model", table_name="cars")
-    op.drop_index("ix_cars_brand", table_name="cars")

alembic/versions/003_add_composite_indexes.py

@@ -13,26 +13,24 @@ def upgrade() -> None:
     # Partial index для активных машин IAAI (is_sold = FALSE)
     # Ускоряет поиск при mark_sold операциях
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
+        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai "
-        "ON cars (origin_url) "
+        "ON iaai_cars (origin_url) "
         "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
     )
 
     # Composite index для проверки дубликатов изображений
-    op.create_index(
+    op.execute(
-        "ix_images_car_id_fullres",
+        "CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres "
-        "images",
+        "ON iaai_images (car_id, fullres_image)"
-        ["car_id", "fullres_image"],
     )
 
     # Index для быстрого поиска existing машин по origin_url
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
+        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id "
-        "ON cars (origin_url, origin_id)"
+        "ON iaai_cars (origin_url, origin_id)"
     )
 
 
 def downgrade() -> None:
-    op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
+    # Compatibility-only migration for shared production databases.
-    op.drop_index("ix_images_car_id_fullres", table_name="images")
+    pass
-    op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")

alembic/versions/004_add_ingestion_tables.py

@@ -0,0 +1,17 @@
+# Placeholder migration (ingestion tables not yet needed)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "004_add_ingestion_tables"
+down_revision: Union[str, None] = "003_add_composite_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass

check_vps.py

@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+import os
+import paramiko
+
+HOST = "2.26.123.84"
+USER = "root"
+PASSWORD = os.environ["VPS_PASSWORD"]
+
+cmds = [
+    "cd /root/iaai-parser && docker compose ps",
+    "docker logs --since 3m iaai-parser-worker-1 2>&1 | tail -n 120",
+    "docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c \"SELECT now() AS ts, count(*) AS cars FROM iaai_cars; SELECT count(*) AS runs FROM iaai_sync_runs;\"",
+]
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect(
+    HOST,
+    username=USER,
+    password=PASSWORD,
+    look_for_keys=False,
+    allow_agent=False,
+    timeout=30,
+    banner_timeout=30,
+    auth_timeout=30,
+)
+try:
+    for cmd in cmds:
+        print(f"REMOTE_RUN {cmd}", flush=True)
+        stdin, stdout, stderr = client.exec_command(cmd, timeout=120)
+        code = stdout.channel.recv_exit_status()
+        print(stdout.read().decode("utf-8", "replace"), end="")
+        print(stderr.read().decode("utf-8", "replace"), end="")
+        print(f"EXIT {code}", flush=True)
+finally:
+    client.close()

clean_vps_db.py

@@ -0,0 +1,41 @@
+from __future__ import annotations
+
+import os
+import paramiko
+
+HOST = "2.26.123.84"
+USER = "root"
+PASSWORD = os.environ["VPS_PASSWORD"]
+
+commands = [
+    "cd /root/iaai-parser && docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c 'TRUNCATE TABLE iaai_images, iaai_cars, iaai_sync_runs RESTART IDENTITY CASCADE;'",
+    "docker exec -i iaai-redis redis-cli FLUSHALL",
+    "docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c 'SELECT count(*) AS cars FROM iaai_cars; SELECT count(*) AS runs FROM iaai_sync_runs;'",
+    "cd /root/iaai-parser && docker compose ps",
+    "docker logs --since 2m iaai-parser-worker-1 2>&1 | tail -n 100",
+]
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect(
+    HOST,
+    username=USER,
+    password=PASSWORD,
+    look_for_keys=False,
+    allow_agent=False,
+    timeout=30,
+    banner_timeout=30,
+    auth_timeout=30,
+)
+try:
+    for command in commands:
+        print(f"REMOTE_RUN {command}", flush=True)
+        stdin, stdout, stderr = client.exec_command(command, timeout=180)
+        exit_code = stdout.channel.recv_exit_status()
+        print(stdout.read().decode("utf-8", "replace"), end="")
+        print(stderr.read().decode("utf-8", "replace"), end="")
+        print(f"EXIT {exit_code}", flush=True)
+        if exit_code != 0:
+            raise SystemExit(exit_code)
+finally:
+    client.close()

deploy_vps.py

@@ -0,0 +1,135 @@
+from __future__ import annotations
+
+import os
+import posixpath
+import stat
+import tarfile
+import time
+import socket
+from pathlib import Path
+
+import paramiko
+
+HOST = "2.26.123.84"
+USER = "root"
+PASSWORD = os.environ["VPS_PASSWORD"]
+LOCAL_ROOT = Path(__file__).resolve().parent
+ARCHIVE = LOCAL_ROOT / "iaai_deploy.tar.gz"
+REMOTE_DIR = "/root/iaai-parser"
+REMOTE_ARCHIVE = "/root/iaai_deploy.tar.gz"
+
+EXCLUDE_DIRS = {
+    ".git",
+    ".venv",
+    "__pycache__",
+    ".pytest_cache",
+    ".mypy_cache",
+    ".ruff_cache",
+    "iaai_scraper.egg-info",
+}
+EXCLUDE_FILES = {"iaai_deploy.tar.gz", "deploy_vps.py"}
+
+
+def _include(path: Path) -> bool:
+    rel = path.relative_to(LOCAL_ROOT)
+    parts = set(rel.parts)
+    if parts & EXCLUDE_DIRS:
+        return False
+    if path.name in EXCLUDE_FILES:
+        return False
+    if path.suffix in {".pyc", ".pyo"}:
+        return False
+    return True
+
+
+def make_archive() -> None:
+    if ARCHIVE.exists():
+        ARCHIVE.unlink()
+    with tarfile.open(ARCHIVE, "w:gz") as tf:
+        for path in LOCAL_ROOT.rglob("*"):
+            if not _include(path):
+                continue
+            tf.add(path, arcname=str(path.relative_to(LOCAL_ROOT)))
+    print(f"ARCHIVE_READY {ARCHIVE} {ARCHIVE.stat().st_size} bytes", flush=True)
+
+
+def connect() -> paramiko.SSHClient:
+    last_exc: BaseException | None = None
+    for attempt in range(1, 6):
+        client = paramiko.SSHClient()
+        client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+        try:
+            print(f"SSH_CONNECT attempt={attempt}", flush=True)
+            client.connect(
+                HOST,
+                username=USER,
+                password=PASSWORD,
+                look_for_keys=False,
+                allow_agent=False,
+                timeout=45,
+                banner_timeout=60,
+                auth_timeout=45,
+            )
+            return client
+        except (paramiko.SSHException, socket.timeout, OSError) as exc:
+            last_exc = exc
+            client.close()
+            print(f"SSH_CONNECT_RETRY attempt={attempt} error={type(exc).__name__}: {exc}", flush=True)
+            time.sleep(3 * attempt)
+    raise SystemExit(f"SSH_CONNECT_FAILED: {last_exc}")
+
+
+def run(client: paramiko.SSHClient, cmd: str, timeout: int | None = None) -> None:
+    print(f"REMOTE_RUN {cmd}", flush=True)
+    stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
+    exit_code = stdout.channel.recv_exit_status()
+    out = stdout.read().decode("utf-8", "replace")
+    err = stderr.read().decode("utf-8", "replace")
+    if out:
+        print(out, end="", flush=True)
+    if err:
+        print(err, end="", flush=True)
+    if exit_code != 0:
+        raise SystemExit(f"REMOTE_FAILED code={exit_code}: {cmd}")
+
+
+def upload_file(sftp: paramiko.SFTPClient, local: Path, remote: str) -> None:
+    total = local.stat().st_size
+    last = 0.0
+
+    def cb(done: int, _total: int) -> None:
+        nonlocal last
+        now = time.time()
+        if now - last >= 2 or done == total:
+            print(f"UPLOAD {done}/{total}", flush=True)
+            last = now
+
+    sftp.put(str(local), remote, callback=cb)
+
+
+def main() -> None:
+    make_archive()
+    client = connect()
+    try:
+        run(client, "echo VPS_OK && hostname && docker --version && docker compose version")
+        sftp = client.open_sftp()
+        try:
+            upload_file(sftp, ARCHIVE, REMOTE_ARCHIVE)
+        finally:
+            sftp.close()
+        run(
+            client,
+            f"mkdir -p {REMOTE_DIR} && cd {REMOTE_DIR} && docker compose down || true && "
+            f"find {REMOTE_DIR} -mindepth 1 -maxdepth 1 ! -name '.env' -exec rm -rf {{}} + && "
+            f"tar -xzf {REMOTE_ARCHIVE} -C {REMOTE_DIR} && rm -f {REMOTE_ARCHIVE}",
+            timeout=300,
+        )
+        run(client, f"cd {REMOTE_DIR} && docker compose up -d --build", timeout=1800)
+        run(client, f"cd {REMOTE_DIR} && docker compose ps", timeout=120)
+        run(client, "docker logs --since 2m iaai-parser-worker-1 2>&1 | tail -n 120 || docker compose -f /root/iaai-parser/docker-compose.yml logs --since 2m worker | tail -n 120", timeout=120)
+    finally:
+        client.close()
+
+
+if __name__ == "__main__":
+    main()

docker-compose.vps.yml

@@ -1,76 +0,0 @@
-# Overlay для слабого VPS (4 vCPU, 4 GB RAM).
-# Применять как: docker compose -f docker-compose.yml -f docker-compose.vps.yml up -d --build
-#
-# Главное:
-#  - worker concurrency=1, max-tasks-per-child=3 (Chromium быстро течёт)
-#  - IAAI_PARALLEL_TABS=4 (важно: каждая вкладка ~80-150 MB)
-#  - IAAI_MAX_PAGES_PER_RUN=200, IAAI_MAX_VEHICLES_PER_RUN=2000 — короткие тестовые прогоны
-#  - hard memory limits на каждый сервис
-
-x-vps-env: &vps-env
-  CELERY_WORKER_CONCURRENCY: "1"
-  CELERY_WORKER_MAX_TASKS_PER_CHILD: "3"
-  CELERY_WORKER_MAX_MEMORY_PER_CHILD_KB: "350000"   # ~350 MB → перезапуск процесса
-  IAAI_PARALLEL_TABS: "4"
-  IAAI_BLOCK_RESOURCES: "true"
-  IAAI_MAX_PAGES_PER_RUN: "200"
-  IAAI_MAX_VEHICLES_PER_RUN: "2000"
-  IAAI_LISTING_SEGMENTS: "auto"
-  IAAI_HUMAN_PACE_ENABLED: "true"
-  CELERY_BATCH_SIZE: "100"
-  CELERY_TASK_SOFT_TIME_LIMIT: "1500"   # 25 мин — короче бутстрап-сегмент
-  CELERY_TASK_TIME_LIMIT: "1800"        # 30 мин
-  CELERY_BROKER_VISIBILITY_TIMEOUT: "3600"
-  IAAI_DATABASE_POOL_SIZE: "3"
-  IAAI_DATABASE_MAX_OVERFLOW: "2"
-
-services:
-  postgres:
-    mem_limit: 512m
-    mem_reservation: 256m
-    command:
-      - "postgres"
-      - "-c"
-      - "shared_buffers=128MB"
-      - "-c"
-      - "effective_cache_size=384MB"
-      - "-c"
-      - "work_mem=8MB"
-      - "-c"
-      - "maintenance_work_mem=64MB"
-      - "-c"
-      - "max_connections=50"
-
-  redis:
-    mem_limit: 192m
-    mem_reservation: 64m
-    command: >
-      redis-server
-      --appendonly yes
-      --save 60 1000
-      --maxmemory 128mb
-      --maxmemory-policy allkeys-lru
-
-  api:
-    mem_limit: 384m
-    mem_reservation: 128m
-    environment:
-      <<: *vps-env
-
-  worker:
-    mem_limit: 1800m
-    mem_reservation: 512m
-    environment:
-      <<: *vps-env
-    # Переопределяем, чтобы concurrency=1 и max-tasks-per-child=3 точно применились
-    command: >
-      celery -A iaai_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=1 --pool=prefork
-      --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=3
-
-  beat:
-    mem_limit: 256m
-    mem_reservation: 64m
-    environment:
-      <<: *vps-env

docker-compose.yml

@@ -1,27 +1,69 @@
 x-app-env: &app-env
-  # NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
+  # Всегда используем Postgres.
   IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
   IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
   CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
   IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300}
+  IAAI_DATABASE_POOL_SIZE: ${IAAI_DATABASE_POOL_SIZE:-20}
-  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600}
+  IAAI_DATABASE_MAX_OVERFLOW: ${IAAI_DATABASE_MAX_OVERFLOW:-40}
-  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
+  # Для больших Search-выборок (десятки тысяч лотов) run должен жить дольше одного батча.
-  # 0 => без лимита (в коде интерпретируется как None).
+  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-7200}
-  # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
+  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-7500}
+  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-10800}
+  IAAI_PROFILE: ${IAAI_PROFILE:-fast}
+  IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}}
+  IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true}
+  IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false}
+  IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false}
+  IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1}
+  IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-1}
+  IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0.03}
+  IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-2}
+  IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
+  # 0 = без лимита.
   CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
+  # Следующий плановый прогон — не раньше чем через час после предыдущего tick beat.
+  CELERY_BEAT_SYNC_INTERVAL_MINUTES: ${CELERY_BEAT_SYNC_INTERVAL_MINUTES:-60}
+  # Любой внутренний follow-up после неполного bootstrap тоже не стартует сразу.
+  IAAI_SYNC_FOLLOWUP_MIN_DELAY_SECONDS: ${IAAI_SYNC_FOLLOWUP_MIN_DELAY_SECONDS:-3600}
+  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
+  CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
   CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
+  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1500}
-  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
+  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
+  # Стабильный fast-профиль: не душим IAAI слишком большим числом HTTPS detail-соединений.
+  IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-96}
+  IAAI_MAX_RETRIES: ${IAAI_MAX_RETRIES:-2}
+  IAAI_RETRY_DELAY_SECONDS: ${IAAI_RETRY_DELAY_SECONDS:-0.35}
+  CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-false}
+  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
+  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
   IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
-  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
+  IAAI_FILTERED_SEARCH_URL: ${IAAI_FILTERED_SEARCH_URL:-}
+  IAAI_FILTERED_SEARCH_URLS: ${IAAI_FILTERED_SEARCH_URLS:-}
+  IAAI_FAST_PATH_TIMEOUT_MS: ${IAAI_FAST_PATH_TIMEOUT_MS:-10000}
+  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-[]}
+  IAAI_DISCOVERY_MODE: ${IAAI_DISCOVERY_MODE:-listing}
+  IAAI_HOURLY_MODE: ${IAAI_HOURLY_MODE:-rolling_refresh}
+  IAAI_HOURLY_REFRESH_BATCH_SIZE: ${IAAI_HOURLY_REFRESH_BATCH_SIZE:-500}
   IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
   IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
-  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
+  IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-false}
+  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-false}
+  # Первый sync после clean restart стартует сразу; следующий запуск — только через hourly guard/beat.
+  IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
   IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
   IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
   IAAI_BROWSER_ENGINE: chromium
+  # Self-heal для worker.
+  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
+  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
+  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
+  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
+  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
+  # Если в Postgres нет записей дольше 60 минут при активной работе — полный restart с segment 1.
+  IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -40,10 +82,9 @@ x-worker-service: &worker-service
   volumes:
     - ./runtime_config.json:/app/runtime_config.json:ro
     - tokens_data:/data
-    - beat_data:/var/lib/celery
 
 services:
-  # PostgreSQL
+  # PostgreSQL.
   postgres:
     image: postgres:16-alpine
     container_name: iaai-postgres
@@ -67,7 +108,7 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Redis (Celery broker)
+  # Redis.
   redis:
     image: redis:7-alpine
     container_name: iaai-redis
@@ -91,7 +132,7 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # DB migrations
+  # Миграции.
   migrate:
     <<: *app-service
     container_name: iaai-migrate
@@ -101,7 +142,7 @@ services:
         condition: service_healthy
     command: alembic upgrade head
 
-  # FastAPI
+  # API.
   api:
     <<: *app-service
     container_name: iaai-api
@@ -129,10 +170,9 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  # Celery Worker
+  # Worker.
   worker:
     <<: *worker-service
-    container_name: iaai-worker
     restart: unless-stopped
     init: true
     depends_on:
@@ -143,22 +183,25 @@ services:
     stop_grace_period: 60s
     command: >
       celery -A iaai_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=prefork
+      --loglevel=info
+      --concurrency=${CELERY_WORKER_CONCURRENCY:-4}
+      --pool=${CELERY_WORKER_POOL:-prefork}
       --pidfile=/tmp/celery-worker.pid
-      -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+      -Q iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
     healthcheck:
-      test: ["CMD-SHELL", "celery -A iaai_scraper.worker.celery_app inspect ping -d celery@$$HOSTNAME -t 15 >/dev/null 2>&1 || exit 1"]
+      # Проверка worker.
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
       interval: 60s
-      timeout: 20s
+      timeout: 10s
       retries: 3
-      start_period: 120s
+      start_period: 90s
     logging:
       driver: json-file
       options:
         max-size: "10m"
         max-file: "5"
 
-  # Celery Beat (периодический планировщик)
+  # Beat.
   beat:
     <<: *worker-service
     container_name: iaai-beat
@@ -173,7 +216,7 @@ services:
       celery -A iaai_scraper.worker.celery_app beat
       --loglevel=info
       --pidfile=/tmp/celery-beat.pid
-      --schedule=/var/lib/celery/celerybeat-schedule
+      --schedule=/tmp/celerybeat-schedule
     healthcheck:
       test: ["CMD-SHELL", "test -f /tmp/celery-beat.pid && kill -0 $(cat /tmp/celery-beat.pid)"]
       interval: 60s
@@ -190,4 +233,3 @@ volumes:
   pgdata:
   redisdata:
   tokens_data:
-  beat_data:

entrypoint.sh

@@ -43,39 +43,47 @@ start_proxy_bridge_if_needed() {
   fi
 
   echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
-
+  python -m iaai_scraper.proxy_bridge &
-  # Watchdog: автоматически рестартует bridge при падении.
+  BRIDGE_PID=$!
-  # Нужно для долгоиграющих контейнеров (месяцы аптайма): без watchdog
-  # упавший bridge тихо ломает весь скрапинг и IAAI начинает банить реальный IP VPS.
-  (
-    while true; do
-      python -m iaai_scraper.proxy_bridge
-      EXIT_CODE=$?
-      echo "[entrypoint] proxy_bridge exited with code ${EXIT_CODE}; restarting in 3s..."
-      sleep 3
-    done
-  ) &
-  BRIDGE_WATCHDOG_PID=$!
   sleep 1
 
-  if ! kill -0 "${BRIDGE_WATCHDOG_PID}" 2>/dev/null; then
+  if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
-    echo "[entrypoint] ERROR: proxy_bridge watchdog failed to start"
+    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
     exit 1
   fi
 
-  # Проверяем что bridge действительно слушает порт.
+  echo "[entrypoint] Proxy bridge started (PID ${BRIDGE_PID})"
-  for i in 1 2 3 4 5; do
+}
-    if python -c "import socket,sys; s=socket.socket(); s.settimeout(2); sys.exit(0 if s.connect_ex(('127.0.0.1', 8899))==0 else 1)" 2>/dev/null; then
-      echo "[entrypoint] Proxy bridge listening on :8899 (watchdog PID ${BRIDGE_WATCHDOG_PID})"
-      return 0
-    fi
-    sleep 1
-  done
 
-  echo "[entrypoint] ERROR: proxy_bridge did not start listening on :8899"
+start_self_heal_watchdog_if_worker() {
-  exit 1
+  if ! is_worker_command "$@"; then
+    return 0
+  fi
+
+  # После reboot/restart контейнера файловая система контейнера сохраняется,
+  # поэтому stale pidfile может остаться от прошлого запуска и блокировать старт Celery.
+  # Удаляем его перед запуском worker-процесса.
+  rm -f /tmp/celery-worker.pid
+
+  if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then
+    echo "[entrypoint] Self-heal watchdog disabled"
+    return 0
+  fi
+
+  echo "[entrypoint] Starting self-heal watchdog for worker..."
+  python -m iaai_scraper.worker.self_heal &
+  SELF_HEAL_PID=$!
+  sleep 1
+
+  if ! kill -0 "${SELF_HEAL_PID}" 2>/dev/null; then
+    echo "[entrypoint] ERROR: self-heal watchdog failed to start"
+    exit 1
+  fi
+
+  echo "[entrypoint] Self-heal watchdog started (PID ${SELF_HEAL_PID})"
 }
 
 start_proxy_bridge_if_needed "$@"
+start_self_heal_watchdog_if_worker "$@"
 
 exec "$@"

iaai_scraper/api/routes/tasks.py

@@ -7,7 +7,7 @@ from sqlalchemy import select, func
 from ..deps import get_persistence
 from ...storage.db import PersistenceService
 from ...storage.models import SyncRun
-from ...worker.celery_app import celery_app
+from ...worker.celery_app import IAAI_SYNC_QUEUE, celery_app
 from ...worker.tasks import sync_vehicle_task, sync_listing_task
 
 router = APIRouter()
@@ -33,7 +33,7 @@ def start_sync_vehicle(
     # Запустить задачу скрапинга одного автомобиля через Celery
     result = sync_vehicle_task.apply_async(
         kwargs={"vehicle_url": body.vehicle_url, "lane": body.lane},
-        queue="scraping",
+        queue=IAAI_SYNC_QUEUE,
     )
 
     return {
@@ -56,7 +56,7 @@ def start_sync_listing(
             "limit": body.limit,
             "only_new": body.only_new,
         },
-        queue="scraping",
+        queue=IAAI_SYNC_QUEUE,
     )
 
     return {

iaai_scraper/browser/factory.py

@@ -15,7 +15,7 @@ logger = logging.getLogger("iaai_scraper.browser")
 
 
 def _build_init_script() -> str:
-    # Патч признаков автоматизации.
+    # Маскировка браузера.
     hardware_concurrency = random.choice([4, 8, 12, 16])
     device_memory = random.choice([4, 8, 16])
     languages = ["en-US", "en"]
@@ -69,11 +69,10 @@ class BrowserFactory:
         self.settings = settings
 
     def _resolve_engine(self) -> str:
-        # Выбор движка браузера.
+        # Выбор движка.
         engine = self.settings.browser_engine.strip().lower()
         if engine == "auto":
-            # Для IAAI в headless-режиме Chromium со stealth-скриптами
+            # Для IAAI стабильнее Chromium.
-            # значительно стабильнее Firefox по anti-bot.
             return "chromium"
         if engine in ("firefox", "chromium"):
             return engine
@@ -90,11 +89,11 @@ class BrowserFactory:
             if proxy_dict:
                 launch_kwargs["proxy"] = proxy_dict
                 logger.info("Using proxy: %s", self.settings.proxy.server)
-            # Параметры Firefox для headless-режима.
+            # Настройки Firefox.
             launch_kwargs["firefox_user_prefs"] = {
                 "dom.webdriver.enabled": False,
                 "useAutomationExtension": False,
-                # Базовые оптимизации для VPS.
+                # Базовые оптимизации.
                 "media.autoplay.default": 5,
                 "media.volume_scale": "0.0",
                 "media.audio.playback.standalone": False,
@@ -111,7 +110,7 @@ class BrowserFactory:
             logger.info("Launching Firefox (headless=%s)", self.settings.headless)
             return playwright.firefox.launch(**launch_kwargs)
 
-        # Путь запуска Chromium.
+        # Запуск Chromium.
         args = [
             "--disable-blink-features=AutomationControlled",
             "--no-default-browser-check",
@@ -154,7 +153,7 @@ class BrowserFactory:
         }
 
         if is_firefox:
-            # Заголовки и user-agent для Firefox.
+            # Заголовки Firefox.
             ctx_kwargs["user_agent"] = (
                 "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) "
                 "Gecko/20100101 Firefox/128.0"
@@ -183,7 +182,7 @@ class BrowserFactory:
         context.set_default_navigation_timeout(self.settings.default_timeout_ms)
 
         if not is_firefox:
-            # Патчи маскировки для Chromium.
+            # Маскировка Chromium.
             context.add_init_script(_build_init_script())
             if stealth_sync:
                 context.on("page", lambda page: stealth_sync(page))
@@ -193,7 +192,7 @@ class BrowserFactory:
 
     @staticmethod
     def enable_resource_blocking(page) -> None:
-        # Блокируем тяжёлые ресурсы для ускорения загрузки страниц.
+        # Блокируем тяжёлые ресурсы.
         BLOCKED_TYPES = {"image", "stylesheet", "font", "media"}
         BLOCKED_URL_PATTERNS = (
             "google-analytics", "googletagmanager", "facebook.net",

iaai_scraper/browser/fast_client.py

@@ -0,0 +1,873 @@
+from __future__ import annotations
+
+import html
+import json
+import logging
+import math
+import re
+import threading
+import time
+from dataclasses import dataclass
+from typing import Any, Iterator
+from urllib.parse import quote, urljoin
+
+import requests
+from requests.adapters import HTTPAdapter
+
+from ..core.config import Settings
+
+logger = logging.getLogger("iaai_scraper.fast_client")
+
+TRANSIENT_HTTP_CODES = {408, 425, 429, 500, 502, 503, 504}
+CHALLENGE_MARKERS = (
+    "_incapsula_resource",
+    "incapsula",
+    "incident id",
+    "request unsuccessful",
+    "access denied",
+)
+COOKIE_ACCEPT_SELECTORS = (
+    "button:has-text('Accept All')",
+    "button:has-text('Accept all')",
+    "button:has-text('I Agree')",
+    "button:has-text('Agree')",
+    "button:has-text('Only necessary')",
+    "button:has-text('Только необходимые')",
+    "button:has-text('Принять все')",
+    "[id*='accept']",
+    "[class*='accept']",
+)
+LISTING_MARKER = 'id="GBPSearchQuery"'
+DETAIL_MARKER = 'id="ProductDetailsVM"'
+RESIZER_URL = "https://vis.iaai.com/resizer"
+BRAND_SCOPE_OVERRIDES = {
+    # IAAI does not resolve every rare make through /Vehiclelisting/Cars/{make}.
+    # CUPRA is available through a saved Search scope URL from the site UI.
+    "CUPRA": "/Search?url=Ck7mLZr7Vc2sWBshBCBOx9WhRn%2fOPJoWOhUHRQ7JNhQ%3d",
+}
+DEFAULT_USER_AGENT = (
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+    "AppleWebKit/537.36 (KHTML, like Gecko) "
+    "Chrome/124.0.0.0 Safari/537.36"
+)
+PLAYWRIGHT_REFRESH_POLLS = 8
+
+
+@dataclass(frozen=True)
+class FastListingVehicle:
+    inventory_id: str
+    tenant: str | None
+    auction_id: str | None
+    auction_date: str | None
+    inventory_status: str | None
+    currency: str | None
+    timed_auction_closed: bool
+    timed_auction_indicator: bool
+    prebid_indicator: bool
+    buynow_indicator: bool
+
+
+@dataclass(frozen=True)
+class FastListingPage:
+    vehicles: list[FastListingVehicle]
+    result_count: int
+    page_size: int
+    current_page: int
+    gbp_search_query: dict[str, Any]
+
+
+class HybridSessionAuth:
+    """Requests session with Playwright cookie refresh fallback.
+
+    Fast path is direct HTTP. Playwright is used only to obtain/refresh anti-bot
+    cookies when IAAI returns a challenge or an expected hidden payload is absent.
+    """
+
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._thread_local = threading.local()
+        self._lock = threading.Lock()
+        self._refresh_lock = threading.Lock()
+        self._bootstrap_cookies_loaded = False
+        self._anonymous_bootstrap_attempted = False
+        self._refresh_generation = 0
+        self._latest_refresh_cookies: list[dict[str, Any]] = []
+
+    def request(
+        self,
+        method: str,
+        url: str,
+        *,
+        timeout: int,
+        retries: int,
+        retry_backoff_ms: int,
+        headers: dict[str, str] | None = None,
+        data: Any | None = None,
+        json_body: Any | None = None,
+        expected_marker: str | None = None,
+    ) -> requests.Response:
+        session = self._get_session()
+        self._ensure_anonymous_session_bootstrap(session=session)
+        self._sync_session_with_latest_refresh(session)
+        last_error: Exception | None = None
+        refresh_attempts = 0
+        attempt = 0
+        max_refresh_attempts = max(0, int(self._settings.scraping_profile.challenge_refresh_attempts))
+
+        while attempt <= retries:
+            try:
+                request_started_at = time.perf_counter()
+                if self._settings.scraping_profile.verbose_http_logs:
+                    logger.debug(
+                        "HTTP request started method=%s url=%s attempt=%s/%s timeout=%s marker=%s",
+                        method,
+                        url,
+                        attempt + 1,
+                        retries + 1,
+                        timeout,
+                        expected_marker,
+                    )
+                response = session.request(
+                    method=method,
+                    url=url,
+                    headers=headers,
+                    data=data,
+                    json=json_body,
+                    timeout=(min(10, max(1, timeout)), max(1, timeout)),
+                )
+                if self._settings.scraping_profile.verbose_http_logs or response.status_code >= 400:
+                    logger.debug(
+                        "HTTP request completed method=%s url=%s status=%s elapsed=%.1fs marker=%s",
+                        method,
+                        url,
+                        response.status_code,
+                        time.perf_counter() - request_started_at,
+                        expected_marker,
+                    )
+            except requests.RequestException as exc:
+                last_error = exc
+                if attempt >= retries:
+                    break
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if response.status_code in TRANSIENT_HTTP_CODES and attempt < retries:
+                response.close()
+                self._sleep_backoff(retry_backoff_ms, attempt)
+                attempt += 1
+                continue
+
+            if is_challenge_response(
+                status_code=response.status_code,
+                body_text=response.text,
+                expected_marker=expected_marker,
+            ):
+                response.close()
+                if not self._settings.scraping_profile.challenge_refresh_enabled or refresh_attempts >= max_refresh_attempts:
+                    raise RuntimeError(
+                        "IAAI challenge persisted after "
+                        f"{refresh_attempts} Playwright refresh attempts for url={url}"
+                    )
+                refresh_attempts += 1
+                logger.info(
+                    "Challenge detected for url=%s status=%s marker=%s refresh_attempt=%s/%s",
+                    url,
+                    response.status_code,
+                    expected_marker,
+                    refresh_attempts,
+                    max_refresh_attempts,
+                )
+                self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+                logger.info("Retrying HTTP request after Playwright refresh url=%s", url)
+                if refresh_attempts > 1:
+                    self._sleep_backoff(retry_backoff_ms, refresh_attempts - 1)
+                continue
+
+            return response
+
+        if last_error is not None:
+            raise RuntimeError(f"Request failed url={url}: {last_error}") from last_error
+        raise RuntimeError(f"Request failed url={url} after retries")
+
+    def persist_storage_state(self) -> None:
+        session = self._get_session()
+        with self._lock:
+            self._save_storage_state(session)
+
+    def _get_session(self) -> requests.Session:
+        session = getattr(self._thread_local, "session", None)
+        if session is None:
+            session = requests.Session()
+            pool_size = max(20, int(self._settings.fetch_concurrency) * 2)
+            adapter = HTTPAdapter(pool_connections=pool_size, pool_maxsize=pool_size)
+            session.mount("http://", adapter)
+            session.mount("https://", adapter)
+            session.headers.update(
+                {
+                    "user-agent": DEFAULT_USER_AGENT,
+                    "accept-language": "en-US,en;q=0.9",
+                    "cache-control": "no-cache",
+                    "pragma": "no-cache",
+                }
+            )
+            if self._settings.proxy.enabled:
+                proxies = self._settings.proxy.to_requests_proxies()
+                if proxies:
+                    session.proxies.update(proxies)
+            with self._lock:
+                self._bootstrap_session_cookies(session)
+            self._thread_local.session = session
+            self._thread_local.session_generation = 0
+            self._sync_session_with_latest_refresh(session)
+        return session
+
+    def _sync_session_with_latest_refresh(self, session: requests.Session) -> None:
+        with self._lock:
+            latest_generation = self._refresh_generation
+            session_generation = getattr(self._thread_local, "session_generation", 0)
+            if latest_generation <= session_generation or not self._latest_refresh_cookies:
+                return
+            cookies = list(self._latest_refresh_cookies)
+        self._apply_cookies_to_session(session, cookies)
+        self._thread_local.session_generation = latest_generation
+
+    def _ensure_anonymous_session_bootstrap(self, *, session: requests.Session) -> None:
+        if self._anonymous_bootstrap_attempted or not self._settings.scraping_profile.anonymous_bootstrap_enabled:
+            return
+        if self._session_has_iaai_cookies(session):
+            self._anonymous_bootstrap_attempted = True
+            return
+        with self._lock:
+            if self._anonymous_bootstrap_attempted:
+                return
+            self._anonymous_bootstrap_attempted = True
+        logger.info("No IAAI cookies preloaded. Attempting anonymous session bootstrap via Playwright.")
+        try:
+            self._refresh_session_via_playwright(expected_marker=LISTING_MARKER, session=session)
+        except Exception as exc:
+            logger.warning("Anonymous session bootstrap via Playwright failed; continuing with direct HTTP flow: %s", exc)
+
+    @staticmethod
+    def _session_has_iaai_cookies(session: requests.Session) -> bool:
+        for item in session.cookies:
+            domain = str(getattr(item, "domain", "") or "")
+            if not domain or "iaai.com" in domain.lower():
+                return True
+        return False
+
+    def _bootstrap_session_cookies(self, session: requests.Session) -> None:
+        if self._bootstrap_cookies_loaded:
+            return
+        self._load_storage_state_cookies(session)
+        self._bootstrap_cookies_loaded = True
+
+    def _load_storage_state_cookies(self, session: requests.Session) -> None:
+        tokens_file = self._settings.tokens_file
+        if not tokens_file:
+            return
+        path = __import__("pathlib").Path(tokens_file)
+        if not path.exists():
+            return
+        try:
+            payload = json.loads(path.read_text(encoding="utf-8"))
+        except Exception as exc:
+            logger.warning("Failed to read storage state file '%s': %s", path, exc)
+            return
+        cookies = payload.get("cookies") if isinstance(payload, dict) else None
+        if not isinstance(cookies, list):
+            return
+        applied = 0
+        for item in cookies:
+            if not isinstance(item, dict):
+                continue
+            name = parse_text(item.get("name"))
+            value = parse_text(item.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(item.get("domain")) or ".iaai.com"
+            cookie_path = parse_text(item.get("path")) or "/"
+            expires = parse_int(item.get("expires"))
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+            applied += 1
+        if applied:
+            logger.info("Loaded %s cookies from storage state", applied)
+
+    def _refresh_session_via_playwright(
+        self,
+        *,
+        expected_marker: str | None = None,
+        session: requests.Session | None = None,
+    ) -> None:
+        target_session = session or self._get_session()
+        with self._lock:
+            baseline_generation = self._refresh_generation
+
+        with self._refresh_lock:
+            with self._lock:
+                if self._refresh_generation > baseline_generation and self._latest_refresh_cookies:
+                    self._apply_cookies_to_session(target_session, self._latest_refresh_cookies)
+                    self._thread_local.session_generation = self._refresh_generation
+                    return
+
+            logger.info("IAAI session challenge detected. Refreshing session via Playwright.")
+            cookies = self._fetch_cookies_via_playwright(expected_marker=expected_marker)
+            logger.info("Playwright refresh returned %d cookies", len(cookies))
+            self._apply_cookies_to_session(target_session, cookies)
+            logger.info("Playwright cookies applied to requests session")
+
+            with self._lock:
+                self._refresh_generation += 1
+                self._latest_refresh_cookies = list(cookies)
+                self._anonymous_bootstrap_attempted = True
+                refreshed_generation = self._refresh_generation
+            self._thread_local.session_generation = refreshed_generation
+            logger.info("Playwright refresh completed generation=%s", refreshed_generation)
+
+    def _fetch_cookies_via_playwright(self, *, expected_marker: str | None = None) -> list[dict[str, Any]]:
+        from playwright.sync_api import TimeoutError as PlaywrightTimeoutError
+        from playwright.sync_api import sync_playwright
+
+        with sync_playwright() as playwright:
+            browser = playwright.chromium.launch(headless=self._settings.headless)
+            try:
+                context = browser.new_context(
+                    locale=self._settings.fingerprint.locale,
+                    viewport={"width": 1366, "height": 768},
+                    user_agent=DEFAULT_USER_AGENT,
+                    proxy=self._settings.proxy.to_playwright_dict(),
+                )
+                page = context.new_page()
+                home_target = self._settings.home_url
+                filtered_urls = self._settings.listing.filtered_search_urls
+                target = filtered_urls[0] if filtered_urls else urljoin(self._settings.home_url, "Vehiclelisting/Cars")
+                timeout_ms = max(30_000, self._settings.default_timeout_ms)
+                logger.info("Playwright session refresh opening target=%s marker=%s", target, expected_marker or LISTING_MARKER)
+                page.goto(home_target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+                self._accept_cookie_banner(page)
+                self._wait_until_non_challenge(
+                    page=page,
+                    target=target,
+                    timeout_ms=timeout_ms,
+                    expected_marker=expected_marker or LISTING_MARKER,
+                )
+                cookies = context.cookies()
+                logger.info("Playwright context returned %d cookies", len(cookies))
+            except PlaywrightTimeoutError as exc:
+                raise RuntimeError(f"Playwright refresh timed out: {exc}") from exc
+            finally:
+                try:
+                    browser.close()
+                except Exception as exc:
+                    logger.info("Playwright browser close failed after cookie refresh: %s", exc)
+
+        if not isinstance(cookies, list) or not cookies:
+            raise RuntimeError("Playwright refresh did not return cookies")
+        return [cookie for cookie in cookies if isinstance(cookie, dict)]
+
+    @staticmethod
+    def _apply_cookies_to_session(session: requests.Session, cookies: list[dict[str, Any]]) -> None:
+        session.cookies.clear()
+        for cookie in cookies:
+            name = parse_text(cookie.get("name"))
+            value = parse_text(cookie.get("value"))
+            if not name or value is None:
+                continue
+            domain = parse_text(cookie.get("domain")) or ".iaai.com"
+            cookie_path = parse_text(cookie.get("path")) or "/"
+            expires = parse_int(cookie.get("expires"))
+            session.cookies.set(name, value, domain=domain, path=cookie_path, expires=expires)
+
+    @staticmethod
+    def _wait_until_non_challenge(*, page: Any, target: str, timeout_ms: int, expected_marker: str | None) -> None:
+        poll_ms = max(1000, min(5000, timeout_ms // PLAYWRIGHT_REFRESH_POLLS))
+        navigation_error_count = 0
+        for poll_index in range(PLAYWRIGHT_REFRESH_POLLS):
+            try:
+                page.wait_for_load_state("domcontentloaded", timeout=poll_ms)
+            except Exception:
+                pass
+            page.wait_for_timeout(poll_ms)
+            body: str | None = None
+            for _ in range(3):
+                try:
+                    body = page.content()
+                    break
+                except Exception as exc:
+                    message = str(exc).lower()
+                    if "page.content" not in message or "navigating and changing the content" not in message:
+                        raise
+                    navigation_error_count += 1
+                    page.wait_for_timeout(max(200, poll_ms // 4))
+            if body is not None and not is_challenge_response(
+                status_code=200,
+                body_text=body,
+                expected_marker=expected_marker,
+            ):
+                logger.info(
+                    "Playwright session refresh passed challenge target=%s poll=%s/%s marker=%s",
+                    target,
+                    poll_index + 1,
+                    PLAYWRIGHT_REFRESH_POLLS,
+                    expected_marker,
+                )
+                return
+            try:
+                logger.info(
+                    "Playwright session refresh still waiting target=%s poll=%s/%s marker=%s",
+                    target,
+                    poll_index + 1,
+                    PLAYWRIGHT_REFRESH_POLLS,
+                    expected_marker,
+                )
+                page.goto(target, wait_until="domcontentloaded", timeout=timeout_ms)
+            except Exception:
+                pass
+        raise RuntimeError(
+            "Playwright refresh completed but challenge page is still active "
+            f"(navigation_content_errors={navigation_error_count})"
+        )
+
+    @staticmethod
+    def _accept_cookie_banner(page: Any) -> None:
+        for selector in COOKIE_ACCEPT_SELECTORS:
+            try:
+                locator = page.locator(selector).first
+                if locator.count() == 0 or not locator.is_visible(timeout=500):
+                    continue
+                locator.click(timeout=2_000)
+                page.wait_for_timeout(250)
+                return
+            except Exception:
+                continue
+
+    def _save_storage_state(self, session: requests.Session) -> None:
+        tokens_file = self._settings.tokens_file
+        if not tokens_file:
+            return
+        path = __import__("pathlib").Path(tokens_file)
+        cookies: list[dict[str, Any]] = []
+        for cookie in session.cookies:
+            payload: dict[str, Any] = {
+                "name": cookie.name,
+                "value": cookie.value,
+                "domain": cookie.domain or ".iaai.com",
+                "path": cookie.path or "/",
+                "httpOnly": False,
+                "secure": bool(cookie.secure),
+                "sameSite": "Lax",
+            }
+            if cookie.expires is not None:
+                payload["expires"] = int(cookie.expires)
+            cookies.append(payload)
+        try:
+            path.parent.mkdir(parents=True, exist_ok=True)
+            path.write_text(json.dumps({"cookies": cookies, "origins": []}, ensure_ascii=False, indent=2), encoding="utf-8")
+        except PermissionError as exc:
+            logger.info("Cannot persist IAAI storage state to '%s': %s", path, exc)
+        except OSError as exc:
+            logger.info("Failed to persist IAAI storage state to '%s': %s", path, exc)
+
+    @staticmethod
+    def _sleep_backoff(retry_backoff_ms: int, attempt: int) -> None:
+        if retry_backoff_ms <= 0:
+            return
+        time.sleep(retry_backoff_ms * (2**attempt) / 1000)
+
+
+class IAAIFastClient:
+    def __init__(self, settings: Settings) -> None:
+        self._settings = settings
+        self._auth = HybridSessionAuth(settings)
+
+    def persist_session_state(self) -> None:
+        self._auth.persist_storage_state()
+
+    def iter_listing_vehicles(
+        self,
+        *,
+        listing_start_url: str | None = None,
+        make: str | None = None,
+        max_pages: int | None = None,
+    ) -> Iterator[FastListingVehicle]:
+        seen_inventory_ids: set[str] = set()
+        scope_paths = resolve_listing_scope_paths(
+            listing_start_url=listing_start_url or "",
+            brands={make} if make else set(),
+        )
+        for scope_path in scope_paths:
+            first_page_html = self._fetch_listing_first_page(scope_path)
+            search_scope_path = build_search_scope_path_from_html(first_page_html)
+            if search_scope_path and search_scope_path != scope_path:
+                logger.info(
+                    "Resolved listing scope to fast Search URL: scope=%s search_scope=%s",
+                    scope_path,
+                    search_scope_path,
+                )
+                scope_path = search_scope_path
+            first_page = parse_listing_page(first_page_html)
+            for vehicle in first_page.vehicles:
+                if vehicle.inventory_id in seen_inventory_ids:
+                    continue
+                seen_inventory_ids.add(vehicle.inventory_id)
+                yield vehicle
+
+            page_size = max(1, first_page.page_size)
+            total_pages = max(1, math.ceil(max(first_page.result_count, len(first_page.vehicles)) / page_size))
+            if max_pages is not None and max_pages > 0:
+                total_pages = min(total_pages, max_pages)
+            gbp_search_query = first_page.gbp_search_query
+            logger.info(
+                "Fast listing first page parsed: scope=%s result_count=%s page_size=%s total_pages=%s first_page_vehicles=%s",
+                scope_path,
+                first_page.result_count,
+                page_size,
+                total_pages,
+                len(first_page.vehicles),
+            )
+            for page_number in range(2, total_pages + 1):
+                if page_number == 2 or page_number % 25 == 0 or page_number == total_pages:
+                    logger.info("Fast listing page fetch progress: page=%s/%s", page_number, total_pages)
+                page_html = self._fetch_listing_page(scope_path, gbp_search_query, page_number, page_size)
+                parsed_page = parse_listing_page(page_html)
+                gbp_search_query = parsed_page.gbp_search_query
+                for vehicle in parsed_page.vehicles:
+                    if vehicle.inventory_id in seen_inventory_ids:
+                        continue
+                    seen_inventory_ids.add(vehicle.inventory_id)
+                    yield vehicle
+
+    def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict[str, Any]:
+        escaped_id = quote(inventory_id, safe="~")
+        url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+            retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
+            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=DETAIL_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
+            return parse_product_details_vm(response.text)
+
+    def fetch_vehicle_detail_html(self, inventory_id: str) -> str:
+        escaped_id = quote(inventory_id, safe="~")
+        url = urljoin(self._settings.home_url, f"VehicleDetail/{escaped_id}")
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+            retries=max(0, self._settings.scraping_profile.detail_retries if self._settings.scraping_profile.detail_retries is not None else self._settings.max_retries),
+            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=DETAIL_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Vehicle detail request failed id={inventory_id} status={response.status_code}")
+            return response.text
+
+    def _fetch_listing_first_page(self, scope_path: str) -> str:
+        url = scope_path if scope_path.lower().startswith(("http://", "https://")) else urljoin(self._settings.home_url, scope_path.lstrip("/"))
+        response = self._auth.request(
+            "GET",
+            url,
+            timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+            retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
+            retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+            headers={"accept": "text/html,application/xhtml+xml"},
+            expected_marker=LISTING_MARKER,
+        )
+        with response:
+            if response.status_code >= 400:
+                raise RuntimeError(f"Listing request failed path={scope_path} status={response.status_code}")
+            return response.text
+
+    def _fetch_listing_page(self, scope_path: str, gbp_search_query: dict[str, Any], page_number: int, page_size: int) -> str:
+        query_payload = dict(gbp_search_query)
+        query_payload["CurrentPage"] = page_number
+        query_payload["PageSize"] = page_size
+        search_url = urljoin(self._settings.home_url, "Search")
+        common_headers = {
+            "accept": "text/html,application/xhtml+xml,*/*",
+            "x-requested-with": "XMLHttpRequest",
+        }
+        attempts: list[tuple[dict[str, str], Any, Any]] = [
+            ({**common_headers, "content-type": "application/json"}, None, query_payload),
+            ({**common_headers, "content-type": "application/json"}, None, {"GBPSearchQuery": query_payload}),
+            ({**common_headers}, {"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}, None),
+            ({**common_headers, "content-type": "application/json"}, json.dumps({"GBPSearchQuery": json.dumps(query_payload, separators=(",", ":"))}), None),
+        ]
+        attempts = attempts[:max(1, min(len(attempts), int(self._settings.scraping_profile.listing_post_attempts)))]
+        last_error: Exception | None = None
+        for headers, data, json_body in attempts:
+            try:
+                response = self._auth.request(
+                    "POST",
+                    search_url,
+                    timeout=max(1, self._settings.fast_path_timeout_ms // 1000),
+                    retries=max(0, self._settings.scraping_profile.listing_retries if self._settings.scraping_profile.listing_retries is not None else self._settings.max_retries),
+                    retry_backoff_ms=int(max(0, self._settings.retry_delay_seconds * 1000)),
+                    headers=headers,
+                    data=data,
+                    json_body=json_body,
+                    expected_marker=LISTING_MARKER,
+                )
+                with response:
+                    if response.status_code >= 400:
+                        raise RuntimeError(f"Listing page request failed status={response.status_code} page={page_number}")
+                    body = response.text
+                    if LISTING_MARKER not in body:
+                        raise RuntimeError("Listing page response does not include GBPSearchQuery")
+                    return body
+            except Exception as exc:
+                last_error = exc
+                continue
+        if last_error is not None:
+            raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}: {last_error}") from last_error
+        raise RuntimeError(f"Failed to load listing page={page_number} for {scope_path}")
+
+
+def build_brand_scope_paths(brands: set[str]) -> list[str]:
+    if not brands:
+        return ["/Vehiclelisting/Cars"]
+    paths: list[str] = []
+    for brand in sorted(brands):
+        raw = brand.strip()
+        if not raw:
+            continue
+        override = BRAND_SCOPE_OVERRIDES.get(raw.upper())
+        if override:
+            if override not in paths:
+                paths.append(override)
+            continue
+        slug_hyphen = quote(raw.replace(" ", "-"), safe="-")
+        slug_raw = quote(raw, safe="")
+        for slug in (slug_hyphen, slug_raw):
+            path = f"/Vehiclelisting/Cars/{slug}"
+            if path not in paths:
+                paths.append(path)
+    return paths or ["/Vehiclelisting/Cars"]
+
+
+def resolve_listing_scope_paths(*, listing_start_url: str, brands: set[str]) -> list[str]:
+    explicit_scope = listing_start_url.strip()
+    if explicit_scope:
+        if explicit_scope.lower().startswith(("http://", "https://", "/")):
+            return [explicit_scope]
+        return [f"/Search?url={explicit_scope}"]
+    return build_brand_scope_paths(brands)
+
+
+def build_search_scope_path_from_html(html_text: str) -> str | None:
+    tiny_url = parse_attribute_value(html_text, "data-tinyurl")
+    if tiny_url:
+        return f"/Search?url={tiny_url}"
+
+    data_query_raw = parse_attribute_value(html_text, "data-query")
+    if data_query_raw:
+        try:
+            data_query = json.loads(data_query_raw)
+        except (json.JSONDecodeError, ValueError, TypeError):
+            data_query = None
+        if isinstance(data_query, dict):
+            url_value = parse_text(data_query.get("Url"))
+            if url_value:
+                return f"/Search?url={url_value}"
+    return None
+
+
+def parse_listing_page(html_text: str) -> FastListingPage:
+    gbp_raw = parse_hidden_input_value(html_text, "GBPSearchQuery")
+    vehicle_raw = parse_hidden_input_value(html_text, "VehicleDetails")
+    result_count_raw = parse_hidden_input_value(html_text, "ResultCount")
+    page_size_raw = parse_hidden_input_value(html_text, "PageSize")
+    current_page_raw = parse_hidden_input_value(html_text, "CurrentPage")
+    if not gbp_raw:
+        raise RuntimeError("Listing page missing GBPSearchQuery")
+    if vehicle_raw is None:
+        raise RuntimeError("Listing page missing VehicleDetails")
+    gbp_payload = json.loads(gbp_raw)
+    if not isinstance(gbp_payload, dict):
+        raise RuntimeError("GBPSearchQuery payload is not object")
+    vehicle_payload = json.loads(vehicle_raw)
+    if not isinstance(vehicle_payload, list):
+        raise RuntimeError("VehicleDetails payload is not array")
+
+    vehicles: list[FastListingVehicle] = []
+    for item in vehicle_payload:
+        if not isinstance(item, dict):
+            continue
+        inventory_id = parse_text(item.get("Id"))
+        if not inventory_id:
+            continue
+        vehicles.append(
+            FastListingVehicle(
+                inventory_id=inventory_id,
+                tenant=parse_text(item.get("Tenant")),
+                auction_id=parse_text(item.get("ActnLnId")),
+                auction_date=parse_text(item.get("AuctionDate")) or parse_text(item.get("ActnDtTm")),
+                inventory_status=parse_text(item.get("InventoryStatus")),
+                currency=parse_text(item.get("Currency")),
+                timed_auction_closed=parse_bool(item.get("TimedAuctionClosedIndicator")),
+                timed_auction_indicator=parse_bool(item.get("TimedAuctionIndicator")),
+                prebid_indicator=parse_bool(item.get("PreBidIndicator")),
+                buynow_indicator=parse_bool(item.get("BuyNowIndicator")),
+            )
+        )
+    return FastListingPage(
+        vehicles=vehicles,
+        result_count=parse_int(result_count_raw) or len(vehicles),
+        page_size=parse_int(page_size_raw) or max(1, len(vehicles)),
+        current_page=parse_int(current_page_raw) or 1,
+        gbp_search_query=gbp_payload,
+    )
+
+
+def parse_product_details_vm(html_text: str) -> dict[str, Any]:
+    match = re.search(
+        r"<script[^>]*id=[\"']ProductDetailsVM[\"'][^>]*>\s*(\{.*?\})\s*</script>",
+        html_text,
+        flags=re.DOTALL | re.IGNORECASE,
+    )
+    if match is None:
+        raise RuntimeError("ProductDetailsVM script not found")
+    payload = json.loads(match.group(1))
+    if not isinstance(payload, dict):
+        raise RuntimeError("ProductDetailsVM root is not object")
+    return payload
+
+
+def parse_hidden_input_value(html_text: str, input_id: str) -> str | None:
+    escaped_id = re.escape(input_id)
+    patterns = (
+        rf"<input[^>]*\bid=\"{escaped_id}\"[^>]*\bvalue=\"([^\"]*)\"",
+        rf"<input[^>]*\bid='{escaped_id}'[^>]*\bvalue='([^']*)'",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, html_text, flags=re.IGNORECASE)
+        if match is not None:
+            return html.unescape(match.group(1))
+    return None
+
+
+def parse_attribute_value(html_text: str, attribute_name: str) -> str | None:
+    escaped_name = re.escape(attribute_name)
+    patterns = (
+        rf"\b{escaped_name}=\"([^\"]*)\"",
+        rf"\b{escaped_name}='([^']*)'",
+    )
+    for pattern in patterns:
+        match = re.search(pattern, html_text, flags=re.IGNORECASE)
+        if match is not None:
+            value = html.unescape(match.group(1)).strip()
+            return value or None
+    return None
+
+
+def build_resizer_images_from_keys(image_keys: list[dict[str, Any]]) -> list[dict[str, str | int]]:
+    seen_fullres: set[str] = set()
+    images: list[dict[str, str | int]] = []
+    for index, item in enumerate(image_keys):
+        if not isinstance(item, dict):
+            continue
+        key = parse_text(item.get("k"))
+        if key is None:
+            continue
+        width = parse_int(item.get("w")) or 1600
+        height = parse_int(item.get("h")) or 1200
+        if width <= 0:
+            width = 1600
+        if height <= 0:
+            height = 1200
+        order_index = parse_int(item.get("i"))
+        if order_index is None:
+            order_index = parse_int(item.get("in"))
+        if order_index is None:
+            order_index = index
+        preview_width = min(640, width)
+        preview_height = max(1, int(round(height * (preview_width / width))))
+        escaped_key = quote(key, safe="~")
+        fullres = f"{RESIZER_URL}?imageKeys={escaped_key}&width={width}&height={height}"
+        preview = f"{RESIZER_URL}?imageKeys={escaped_key}&width={preview_width}&height={preview_height}"
+        if fullres in seen_fullres:
+            continue
+        seen_fullres.add(fullres)
+        images.append({"order_index": order_index, "fullres_image": fullres, "preview_image": preview})
+    images.sort(key=lambda row: (parse_int(row.get("order_index")) or 0, str(row.get("fullres_image"))))
+    return images
+
+
+def is_challenge_response(*, status_code: int, body_text: str, expected_marker: str | None = None) -> bool:
+    if status_code in {401, 403}:
+        return True
+    if _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        return False
+    lowered = (body_text or "").lower()
+    if any(marker in lowered for marker in CHALLENGE_MARKERS):
+        return True
+    if expected_marker and not _expected_marker_present(body_text=body_text, expected_marker=expected_marker):
+        if "<html" in lowered or "<body" in lowered:
+            return True
+    return False
+
+
+def _expected_marker_present(*, body_text: str, expected_marker: str | None) -> bool:
+    if not expected_marker:
+        return False
+    if expected_marker in body_text:
+        return True
+    if '"' in expected_marker and expected_marker.replace('"', "'") in body_text:
+        return True
+    if "'" in expected_marker and expected_marker.replace("'", '"') in body_text:
+        return True
+    marker_match = re.search(r"id=['\"]([^'\"]+)['\"]", expected_marker)
+    if marker_match is None:
+        return False
+    marker_id = re.escape(marker_match.group(1))
+    return bool(re.search(rf"id\s*=\s*['\"]{marker_id}['\"]", body_text, flags=re.IGNORECASE))
+
+
+def parse_text(value: Any) -> str | None:
+    if isinstance(value, str):
+        text = value.strip()
+        return text if text else None
+    return None
+
+
+def parse_bool(value: Any) -> bool:
+    if isinstance(value, bool):
+        return value
+    if isinstance(value, str):
+        return value.strip().lower() in {"true", "1", "yes", "on"}
+    if isinstance(value, (int, float)) and not isinstance(value, bool):
+        return value != 0
+    return False
+
+
+def parse_int(value: Any) -> int | None:
+    if value is None or isinstance(value, bool):
+        return None
+    if isinstance(value, int):
+        return value
+    if isinstance(value, float):
+        return int(round(value))
+    if isinstance(value, str):
+        text = value.strip()
+        if not text:
+            return None
+        normalized = text.replace(",", "").replace(" ", "").replace("$", "")
+        match = re.search(r"-?\d+(?:\.\d+)?", normalized)
+        if match is None:
+            return None
+        try:
+            return int(round(float(match.group(0))))
+        except ValueError:
+            return None
+    return None

iaai_scraper/browser/listing.py

@@ -133,6 +133,28 @@ class ListingCollector:
 
 		return not old_first_href
 
+	@staticmethod
+	def has_page_number(page: Page, target_page_number: int) -> bool:
+		try:
+			return bool(page.evaluate(
+				"""
+				(targetPageNumber) => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
+					return controls.some((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						const disabled = el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+						return visible(el) && !disabled && /^\d+$/.test(text) && parseInt(text, 10) === targetPageNumber;
+					});
+				}
+				""",
+				target_page_number,
+			))
+		except Exception:
+			return False
+
 	def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
 		url = url_override or self.settings.listing.cars_url
 		logger.info("Opening cars listing page: %s", url)
@@ -625,8 +647,24 @@ class ListingCollector:
 	@staticmethod
 	def _has_next_page(page: Page) -> bool:
 		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
-			if page.locator(selector).count() > 0:
+			locator = page.locator(selector)
+			count = locator.count()
+			if count == 0:
+				continue
+			if not hasattr(locator, "nth"):
 				return True
+			# Проверяем, что хотя бы один элемент не disabled.
+			# Disabled "Next" на последней странице не означает наличия следующей.
+			for i in range(min(count, 3)):
+				try:
+					el = locator.nth(i)
+					disabled_attr = el.get_attribute("disabled", timeout=300)
+					aria_disabled = el.get_attribute("aria-disabled", timeout=300)
+					cls = (el.get_attribute("class", timeout=300) or "").lower()
+					if disabled_attr is None and aria_disabled != "true" and "disabled" not in cls:
+						return True
+				except Exception:
+					continue
 		try:
 			return bool(page.evaluate(
 				"""

iaai_scraper/core/config.py

@@ -2,6 +2,7 @@ import json
 import os
 from dataclasses import dataclass, field
 from pathlib import Path
+from urllib.parse import quote, urlsplit, urlunsplit
 
 from dotenv import load_dotenv
 
@@ -104,6 +105,8 @@ class HumanPaceConfig:
 @dataclass(slots=True)
 class ListingConfig:
     cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
+    filtered_search_url: str | None = _env_optional_str("IAAI_FILTERED_SEARCH_URL")
+    filtered_search_urls_raw: str | None = _env_optional_str("IAAI_FILTERED_SEARCH_URLS")
     max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
     max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
     page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
@@ -113,9 +116,31 @@ class ListingConfig:
     # прекращаем листать — все новые машины уже найдены. 0 = отключено.
     early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
     # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
-    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка
+    # или "runtime" для построения по runtime_config.filters.brands.
     # Пустая строка = без сегментации (backward compatible).
     listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
+    fast_segment_year_splits: bool = _env_bool("IAAI_FAST_SEGMENT_YEAR_SPLITS", True)
+
+    @property
+    def filtered_search_urls(self) -> list[str]:
+        urls: list[str] = []
+        if self.filtered_search_url and self.filtered_search_url.strip():
+            urls.append(self.filtered_search_url.strip())
+        if self.filtered_search_urls_raw and self.filtered_search_urls_raw.strip():
+            raw = self.filtered_search_urls_raw.strip()
+            try:
+                parsed = json.loads(raw)
+            except (json.JSONDecodeError, ValueError):
+                parsed = None
+            if isinstance(parsed, list):
+                candidates = [str(item or "").strip() for item in parsed]
+            else:
+                candidates = [part.strip() for part in raw.replace("\n", ",").split(",")]
+            for candidate in candidates:
+                if candidate and candidate not in urls:
+                    urls.append(candidate)
+        return urls
 
 
 # Список брендов IAAI для автоматической сегментации.
@@ -149,6 +174,43 @@ _YEAR_SPLITS: tuple[tuple[int, int], ...] = (
 )
 
 
+def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
+    """Строит сегменты по переданному списку брендов.
+
+    Крупные бренды режутся по годовым диапазонам так же, как в ``auto``.
+    """
+    segments: list[dict[str, str | int | None]] = []
+    seen: set[str] = set()
+    for raw_make in makes:
+        make = str(raw_make or "").strip().upper()
+        if not make or make in seen:
+            continue
+        seen.add(make)
+        if make in _LARGE_MAKES:
+            for yr_min, yr_max in _YEAR_SPLITS:
+                segments.append({"make": make, "year_min": yr_min, "year_max": yr_max})
+        else:
+            segments.append({"make": make, "year_min": None, "year_max": None})
+    return segments
+
+
+def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
+    """Строит HTTP-first сегменты без UI-фильтров годов.
+
+    Это ближе к iaai-fast: один бренд = один hidden-payload HTTP обход.
+    Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля.
+    """
+    segments: list[dict[str, str | int | None]] = []
+    seen: set[str] = set()
+    for raw_make in makes:
+        make = str(raw_make or "").strip().upper()
+        if not make or make in seen:
+            continue
+        seen.add(make)
+        segments.append({"make": make, "year_min": None, "year_max": None})
+    return segments
+
+
 def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
     """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
 
@@ -160,14 +222,7 @@ def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
     if not raw:
         return []
     if raw.lower() == "auto":
-        segments: list[dict[str, str | int | None]] = []
+        return build_listing_segments_for_makes(list(IAAI_DEFAULT_MAKES))
-        for m in IAAI_DEFAULT_MAKES:
-            if m in _LARGE_MAKES:
-                for yr_min, yr_max in _YEAR_SPLITS:
-                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
-            else:
-                segments.append({"make": m, "year_min": None, "year_max": None})
-        return segments
     try:
         data = json.loads(raw)
     except (json.JSONDecodeError, ValueError):
@@ -209,6 +264,16 @@ class RedisConfig:
     health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
 
 
+# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
+    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
+    always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", False)
+
+
 # --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
 
 @dataclass(slots=True)
@@ -217,18 +282,58 @@ class CeleryConfig:
     result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
     task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
     task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
     worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
     worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
-    # KB; worker перезапустит child при превышении (защита от утечек Chromium/Playwright).
-    worker_max_memory_per_child_kb: int = _env_int("CELERY_WORKER_MAX_MEMORY_PER_CHILD_KB", 500_000)
     broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
     beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
     beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
     batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
     parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
+    fetch_concurrency: int = _env_int("IAAI_FETCH_CONCURRENCY", _env_int("IAAI_PARALLEL_TABS", 8))
+    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
     block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
 
 
+@dataclass(slots=True)
+class ScrapingProfileConfig:
+    name: str = _env_str("IAAI_SCRAPING_PROFILE", _env_str("IAAI_PROFILE", "stable")).strip().lower()
+    http_first: bool = _env_bool("IAAI_HTTP_FIRST", True)
+    browser_fallback_enabled: bool = _env_bool("IAAI_BROWSER_FALLBACK_ENABLED", True)
+    anonymous_bootstrap_enabled: bool = _env_bool("IAAI_ANONYMOUS_BOOTSTRAP_ENABLED", True)
+    verbose_http_logs: bool = _env_bool("IAAI_VERBOSE_HTTP_LOGS", False)
+    verbose_progress_logs: bool = _env_bool("IAAI_VERBOSE_PROGRESS_LOGS", False)
+    challenge_refresh_enabled: bool = _env_bool("IAAI_CHALLENGE_REFRESH_ENABLED", True)
+    challenge_refresh_attempts: int = _env_int("IAAI_CHALLENGE_REFRESH_ATTEMPTS", 3)
+    listing_post_attempts: int = _env_int("IAAI_LISTING_POST_ATTEMPTS", 4)
+    request_jitter_max_s: float = _env_float("IAAI_REQUEST_JITTER_MAX_S", 0.15)
+    detail_retries: int | None = _env_int("IAAI_DETAIL_RETRIES", -1)
+    listing_retries: int | None = _env_int("IAAI_LISTING_RETRIES", -1)
+
+    def __post_init__(self) -> None:
+        if self.name == "fast":
+            if "IAAI_BROWSER_FALLBACK_ENABLED" not in os.environ:
+                self.browser_fallback_enabled = False
+            if "IAAI_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
+                self.anonymous_bootstrap_enabled = False
+            if "IAAI_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
+                self.challenge_refresh_attempts = 1
+            if "IAAI_LISTING_POST_ATTEMPTS" not in os.environ:
+                self.listing_post_attempts = 2
+            if "IAAI_REQUEST_JITTER_MAX_S" not in os.environ:
+                self.request_jitter_max_s = 0.0
+            if "IAAI_DETAIL_RETRIES" not in os.environ:
+                self.detail_retries = 1
+            if "IAAI_LISTING_RETRIES" not in os.environ:
+                self.listing_retries = 1
+        else:
+            if self.detail_retries is not None and self.detail_retries < 0:
+                self.detail_retries = None
+            if self.listing_retries is not None and self.listing_retries < 0:
+                self.listing_retries = None
+
+
 # --- Конфиг прокси (server, username, password) ---
 
 @dataclass(slots=True)
@@ -251,6 +356,32 @@ class ProxyConfig:
             result["password"] = self.password
         return result
 
+    def to_requests_proxy_url(self) -> str | None:
+        if not self.server:
+            return None
+        if not self.username:
+            return self.server
+
+        parts = urlsplit(self.server)
+        if not parts.scheme or not parts.hostname:
+            return self.server
+
+        username = quote(self.username, safe="")
+        password = quote(self.password or "", safe="")
+        host = parts.hostname
+        if ":" in host and not host.startswith("["):
+            host = f"[{host}]"
+        if parts.port is not None:
+            host = f"{host}:{parts.port}"
+        netloc = f"{username}:{password}@{host}"
+        return urlunsplit((parts.scheme, netloc, parts.path, parts.query, parts.fragment))
+
+    def to_requests_proxies(self) -> dict[str, str] | None:
+        proxy_url = self.to_requests_proxy_url()
+        if not proxy_url:
+            return None
+        return {"http": proxy_url, "https": proxy_url}
+
 
 #  Главный объект настроек: собирает все блоки конфигурации 
 
@@ -259,7 +390,7 @@ class Settings:
     home_url: str = "https://www.iaai.com/"
     default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
     network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
-    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
+    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 15000)
     fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
     fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
     max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
@@ -284,11 +415,17 @@ class Settings:
     redis: RedisConfig = field(default_factory=RedisConfig)
     celery: CeleryConfig = field(default_factory=CeleryConfig)
     proxy: ProxyConfig = field(default_factory=ProxyConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
+    scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
 
     @property
     def parallel_tabs(self) -> int:
         return self.celery.parallel_tabs
 
+    @property
+    def fetch_concurrency(self) -> int:
+        return self.celery.fetch_concurrency
+
     @property
     def block_resources(self) -> bool:
         return self.celery.block_resources

iaai_scraper/core/exceptions.py

@@ -12,7 +12,3 @@ class SiteStructureChangedError(ScraperError):
 
 class ListingResumeError(ScraperError):
     """Вызывается, когда resume по checkpoint больше недостижим."""
-
-
-class ScraperAbortedError(ScraperError):
-    """Вызывается при внешнем прерывании (например, потеря Celery lock'а)."""

iaai_scraper/core/logs.py

@@ -1,7 +1,6 @@
 import logging
 import sys
 from contextvars import ContextVar
-from logging.handlers import RotatingFileHandler
 
 # Храним trace_id текущего потока/корутины.
 TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
@@ -19,19 +18,11 @@ def set_trace_id(trace_id: str) -> None:
 
 
 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
-    # stderr — Docker и Celery prefork корректно его подхватывают.
+    # stdout — основной поток для `docker logs`, Docker Desktop и compose logs.
-    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
+    # stderr в PowerShell часто выглядит как NativeCommandError, хотя это обычные логи.
+    handlers: list[logging.Handler] = [logging.StreamHandler(sys.stdout)]
     if log_file:
-        # Ротация файлового лога: 50MB × 5 файлов, чтобы диск VPS не переполнился
+        handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
-        # при многомесячной работе.
-        handlers.append(
-            RotatingFileHandler(
-                log_file,
-                maxBytes=50 * 1024 * 1024,
-                backupCount=5,
-                encoding="utf-8",
-            )
-        )
     trace_filter = TraceIdFilter()
     for handler in handlers:
         handler.addFilter(trace_filter)
@@ -39,9 +30,15 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
     root = logging.getLogger()
     root.setLevel(getattr(logging, level.upper(), logging.INFO))
     # Убираем старые хендлеры, чтобы не дублировать после fork.
+    for old_handler in list(root.handlers):
+        try:
+            old_handler.close()
+        except Exception:
+            pass
     root.handlers.clear()
     for handler in handlers:
         root.addHandler(handler)
+    root.propagate = False
     fmt = logging.Formatter(
         "%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
     )

iaai_scraper/discovery/__init__.py

@@ -0,0 +1,15 @@
+from .sitemap import (
+    SitemapDiscoveryError,
+    SitemapDiscoveryResult,
+    SitemapDiscoveryStats,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+__all__ = [
+    "SitemapDiscoveryError",
+    "SitemapDiscoveryResult",
+    "SitemapDiscoveryStats",
+    "discover_vehicle_urls_from_sitemap",
+    "discover_vehicle_urls_from_sitemap_with_stats",
+]

iaai_scraper/discovery/sitemap.py

@@ -0,0 +1,210 @@
+from __future__ import annotations
+
+import gzip
+import io
+import logging
+import re
+from dataclasses import dataclass, field
+from typing import Iterable
+from urllib.parse import urlsplit, urlunsplit
+from urllib.request import Request, urlopen, ProxyHandler, build_opener
+import xml.etree.ElementTree as ET
+
+logger = logging.getLogger("iaai_scraper.discovery.sitemap")
+
+DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+_SITEMAP_TIMEOUT_SECONDS = 30
+_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
+
+
+class SitemapDiscoveryError(RuntimeError):
+    pass
+
+
+def _is_vehicle_sitemap_url(url: str) -> bool:
+    lowered = url.strip().lower()
+    # Берём только sitemap с авто.
+    if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
+        return False
+    return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
+
+
+def _normalize_vehicle_url(url: str) -> str:
+    parts = urlsplit(url.strip())
+    return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
+
+
+def _download_bytes(url: str, proxy_url: str | None = None) -> bytes:
+    request = Request(
+        url,
+        headers={
+            "User-Agent": (
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
+            ),
+            "Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
+            "Accept-Encoding": "gzip",
+        },
+    )
+    if proxy_url:
+        handler = ProxyHandler({"http": proxy_url, "https": proxy_url})
+        opener = build_opener(handler)
+        response = opener.open(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    else:
+        response = urlopen(request, timeout=_SITEMAP_TIMEOUT_SECONDS)
+    with response:
+        payload = response.read()
+        encoding = str(response.headers.get("Content-Encoding") or "").lower()
+        if encoding == "gzip" or url.lower().endswith(".gz"):
+            return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
+        return payload
+
+
+def _local_name(tag: str) -> str:
+    if "}" in tag:
+        return tag.rsplit("}", 1)[1]
+    return tag
+
+
+def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
+    for match in _LOC_TAG_RE.finditer(xml_bytes):
+        try:
+            value = match.group(1).decode("utf-8", errors="ignore").strip()
+        except Exception:
+            continue
+        if value:
+            yield value
+
+
+def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
+    try:
+        root = ET.fromstring(xml_bytes)
+    except ET.ParseError as exc:
+        fallback_values = list(_iter_loc_values_fallback(xml_bytes))
+        if fallback_values:
+            logger.warning(
+                "Falling back to regex sitemap loc extraction after XML parse error: %s",
+                exc,
+            )
+            yield from fallback_values
+            return
+        raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
+
+    for element in root.iter():
+        if _local_name(element.tag) != "loc":
+            continue
+        if not element.text:
+            continue
+        value = element.text.strip()
+        if value:
+            yield value
+
+
+def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
+    result: list[str] = []
+    seen: set[str] = set()
+    for url in urls:
+        normalized = _normalize_vehicle_url(url)
+        if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
+            continue
+        if normalized in seen:
+            continue
+        seen.add(normalized)
+        result.append(normalized)
+    return result
+
+
+def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
+    return any("/vehicledetail/" in url.lower() for url in urls)
+
+
+def discover_vehicle_urls_from_sitemap(index_url: str = DEFAULT_SITEMAP_INDEX_URL, proxy_url: str | None = None) -> list[str]:
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return deduped
+
+
+@dataclass
+class SitemapDiscoveryStats:
+    transport: str = "http"
+    fetched_sitemaps: int = 0
+    blocked_sitemaps: int = 0
+    malformed_sitemaps: int = 0
+    direct_probe_hits: int = 0
+    direct_probe_misses: int = 0
+
+
+@dataclass
+class SitemapDiscoveryResult:
+    vehicle_urls: list[str] = field(default_factory=list)
+    stats: SitemapDiscoveryStats = field(default_factory=SitemapDiscoveryStats)
+
+
+def discover_vehicle_urls_from_sitemap_with_stats(
+    settings=None,
+    index_url: str = DEFAULT_SITEMAP_INDEX_URL,
+) -> SitemapDiscoveryResult:
+    """Возвращает URL и статистику."""
+    proxy_url = None
+    if settings is not None and hasattr(settings, 'proxy') and settings.proxy.server:
+        proxy_url = settings.proxy.server
+    stats = SitemapDiscoveryStats(transport="http")
+    logger.info("Downloading sitemap index: %s", index_url)
+    index_xml = _download_bytes(index_url, proxy_url=proxy_url)
+    sitemap_urls = [url for url in _iter_loc_values(index_xml) if _is_vehicle_sitemap_url(url)]
+    if not sitemap_urls:
+        raise SitemapDiscoveryError("Sitemap index returned no sitemap URLs")
+
+    all_vehicle_urls: list[str] = []
+    for sitemap_url in sitemap_urls:
+        logger.info("Downloading sitemap: %s", sitemap_url)
+        try:
+            sitemap_xml = _download_bytes(sitemap_url, proxy_url=proxy_url)
+            raw_urls = list(_iter_loc_values(sitemap_xml))
+            stats.fetched_sitemaps += 1
+        except SitemapDiscoveryError as exc:
+            logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
+            stats.malformed_sitemaps += 1
+            continue
+        except Exception as exc:
+            logger.warning("Blocked/failed sitemap %s: %s", sitemap_url, exc)
+            stats.blocked_sitemaps += 1
+            continue
+
+        vehicle_urls = _filter_vehicle_urls(raw_urls)
+        if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
+            logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
+            continue
+        logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
+        all_vehicle_urls.extend(vehicle_urls)
+
+    deduped = _filter_vehicle_urls(all_vehicle_urls)
+    if not deduped:
+        raise SitemapDiscoveryError("No vehicle detail URLs discovered from vehicle sitemaps")
+    logger.info("Sitemap discovery done: %d vehicle URLs", len(deduped))
+    return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

iaai_scraper/fast_sync.py

@@ -0,0 +1,486 @@
+from __future__ import annotations
+
+import concurrent.futures
+import logging
+import random
+import time
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from .browser.fast_client import FastListingVehicle, IAAIFastClient
+from .core.runtime_config import RuntimeConfig
+from .parsing.fast_mapper import INACTIVE_STATUS_VALUES, FastCarMapper
+from .storage.db import PersistenceService
+from .storage.schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.fast_sync")
+
+
+@dataclass(slots=True)
+class FastSyncStats:
+    ids_fetched: int = 0
+    cars_upserted: int = 0
+    cars_failed: int = 0
+    cars_filtered: int = 0
+    images_upserted: int = 0
+    skipped_existing: int = 0
+    protection_events: int = 0
+
+
+def passes_condition_check(row: FastListingVehicle) -> bool:
+    if row.timed_auction_closed:
+        return False
+    status = (row.inventory_status or "").strip().upper()
+    return status not in INACTIVE_STATUS_VALUES
+
+
+class FastSyncEngine:
+    """Full iaai-fast style sync pipeline adapted to this project's storage.
+
+    Flow: hidden listing payloads -> concurrent ProductDetailsVM HTTP fetch ->
+    CarRecord preparation in memory -> single batch DB upsert. Browser is used
+    only inside IAAIFastClient to refresh cookies when IAAI challenge appears.
+    """
+
+    def __init__(
+        self,
+        *,
+        client: IAAIFastClient,
+        mapper: FastCarMapper,
+        persistence: PersistenceService,
+        batch_size: int,
+        fetch_concurrency: int,
+        report_progress: Callable[[str, Any], None] | None = None,
+    ) -> None:
+        self.client = client
+        self.mapper = mapper
+        self.persistence = persistence
+        self.batch_size = max(1, int(batch_size))
+        self.fetch_concurrency = max(1, int(fetch_concurrency))
+        self.report_progress = report_progress
+
+    @staticmethod
+    def _is_transient_detail_error(exc: Exception) -> bool:
+        text = str(exc).lower()
+        return any(
+            marker in text
+            for marker in (
+                "sslerror",
+                "ssleoferror",
+                "unexpected_eof_while_reading",
+                "eof occurred in violation of protocol",
+                "max retries exceeded",
+                "read timed out",
+                "readtimeout",
+                "connection reset",
+                "connection aborted",
+                "connection closed",
+                "temporarily unavailable",
+                "too many requests",
+                "status=429",
+                "status=500",
+                "status=502",
+                "status=503",
+                "status=504",
+            )
+        )
+
+    def sync_listing(
+        self,
+        *,
+        runtime_config: RuntimeConfig,
+        make: str | None = None,
+        model: str | None = None,
+        lane: str = "iaai_cars",
+        limit: int | None = None,
+        only_new: bool = False,
+        listing_url: str | None = None,
+        max_pages: int | None = None,
+        skip_mark_sold: bool = False,
+    ) -> dict[str, Any]:
+        del lane
+        started_at = time.perf_counter()
+        stats = FastSyncStats()
+        errors: list[dict[str, str]] = []
+        selected: dict[str, FastListingVehicle] = {}
+        rows_seen = 0
+        rows_skipped_condition = 0
+
+        filters = runtime_config.filters
+        logger.info(
+            "Fast HTTP-first listing started: make=%s listing_url=%s max_pages=%s concurrency=%s batch_size=%s",
+            make or "ALL",
+            listing_url or "default",
+            max_pages,
+            self.fetch_concurrency,
+            self.batch_size,
+        )
+        for vehicle in self.client.iter_listing_vehicles(
+            listing_start_url=listing_url,
+            make=make,
+            max_pages=max_pages,
+        ):
+            rows_seen += 1
+            if runtime_config.sync.condition_check_enabled and not passes_condition_check(vehicle):
+                rows_skipped_condition += 1
+                continue
+            if vehicle.inventory_id in selected:
+                continue
+            selected[vehicle.inventory_id] = vehicle
+            if limit is not None and limit > 0 and len(selected) >= limit:
+                break
+
+        candidates = list(selected.values())
+        all_listing_origin_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates}
+        if only_new and candidates:
+            origin_urls = [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates]
+            origin_ids = [f"iaai:{v.inventory_id}" for v in candidates]
+            existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(origin_urls, origin_ids)
+            fresh: list[FastListingVehicle] = []
+            for vehicle in candidates:
+                if f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}" in existing_urls or f"iaai:{vehicle.inventory_id}" in existing_ids:
+                    stats.skipped_existing += 1
+                    continue
+                fresh.append(vehicle)
+            candidates = fresh
+
+        self._progress(
+            "fast_listing_collected",
+            rows_seen=rows_seen,
+            rows_filtered_condition=rows_skipped_condition,
+            rows_selected=len(candidates),
+            skipped_existing=stats.skipped_existing,
+        )
+        logger.info(
+            "Fast HTTP-first listing collected: rows_seen=%d selected=%d skipped_existing=%d filtered_condition=%d only_new=%s",
+            rows_seen,
+            len(candidates),
+            stats.skipped_existing,
+            rows_skipped_condition,
+            only_new,
+        )
+
+        scan_completed = not (limit is not None and limit > 0) and not errors
+
+        if not candidates:
+            return self._result(
+                started_at=started_at,
+                stats=stats,
+                failures=errors,
+                listing={
+                    "mode": "fast_hidden_payload",
+                    "vehicles_collected": 0,
+                    "vehicle_urls": [],
+                    "early_stopped": False,
+                    "truncated_by_time_budget": False,
+                    "rows_seen": rows_seen,
+                    "rows_filtered_condition": rows_skipped_condition,
+                },
+                all_listing_origin_urls=all_listing_origin_urls,
+                full_scan_completed=scan_completed,
+            )
+
+        prepared_rows: list[CarRecord] = []
+        db_processed = 0
+        retry_candidates: list[FastListingVehicle] = []
+        transient_failure_log_count = 0
+        started_details = time.perf_counter()
+        with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor:
+            future_to_vehicle = {
+                executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
+                for vehicle in candidates
+            }
+            for index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
+                vehicle = future_to_vehicle[future]
+                try:
+                    payload = future.result()
+                    record = self.mapper.map_payload_to_record(
+                        detail_payload=payload,
+                        vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
+                        listing_vehicle=vehicle,
+                    )
+                    if model and model.casefold() not in record.model.casefold():
+                        stats.cars_filtered += 1
+                        continue
+                    if not filters.matches({
+                        "brand": record.brand,
+                        "model": record.model,
+                        "year": record.year,
+                        "body_type": record.body_type,
+                        "color": record.color,
+                        "drive": record.drive,
+                        "gearbox": record.gearbox,
+                        "price": record.price,
+                        "mileage": record.mileage,
+                    }):
+                        stats.cars_filtered += 1
+                        continue
+                    prepared_rows.append(record)
+                    stats.ids_fetched += 1
+                    if len(prepared_rows) >= self.batch_size:
+                        db_processed += self._flush_db_records(
+                            rows=prepared_rows,
+                            stats=stats,
+                            errors=errors,
+                            processed=db_processed + len(prepared_rows),
+                            total=len(candidates),
+                        )
+                        prepared_rows.clear()
+                except Exception as exc:
+                    stats.cars_failed += 1
+                    errors.append({"vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}", "error": str(exc)})
+                    if _looks_like_protection(exc):
+                        stats.protection_events += 1
+                    if self._is_transient_detail_error(exc):
+                        retry_candidates.append(vehicle)
+                        transient_failure_log_count += 1
+                        if transient_failure_log_count % 100 == 0:
+                            logger.warning(
+                                "Fast detail transient failures queued for retry: count=%d latest_inventory_id=%s",
+                                transient_failure_log_count,
+                                vehicle.inventory_id,
+                            )
+                    else:
+                        logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc)
+
+                if index % 100 == 0 or index == len(candidates):
+                    elapsed = max(0.001, time.perf_counter() - started_details)
+                    verbose_progress_logs = bool(
+                        getattr(
+                            getattr(getattr(self.client, "_settings", None), "scraping_profile", None),
+                            "verbose_progress_logs",
+                            False,
+                        )
+                    )
+                    if verbose_progress_logs:
+                        logger.info(
+                            "Fast HTTP-first details progress: processed=%d/%d ok=%d failed=%d queued_db=%d rate=%.2f/s",
+                            index,
+                            len(candidates),
+                            stats.ids_fetched,
+                            stats.cars_failed,
+                            len(prepared_rows),
+                            index / elapsed,
+                        )
+                    self._progress(
+                        "fast_detail_progress",
+                        processed=index,
+                        total=len(candidates),
+                        ids_fetched=stats.ids_fetched,
+                        cars_failed=stats.cars_failed,
+                        queued_for_db=len(prepared_rows),
+                        throughput=round(index / elapsed, 2),
+                    )
+
+        if retry_candidates:
+            retry_started = time.perf_counter()
+            retry_workers = max(1, min(8, self.fetch_concurrency // 2, len(retry_candidates)))
+            retry_errors: list[dict[str, str]] = []
+            logger.info(
+                "Fast HTTP-first retrying transient detail failures: total=%d workers=%d",
+                len(retry_candidates),
+                retry_workers,
+            )
+            with concurrent.futures.ThreadPoolExecutor(max_workers=retry_workers) as executor:
+                future_to_vehicle = {
+                    executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
+                    for vehicle in retry_candidates
+                }
+                for retry_index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
+                    vehicle = future_to_vehicle[future]
+                    try:
+                        payload = future.result()
+                        record = self.mapper.map_payload_to_record(
+                            detail_payload=payload,
+                            vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
+                            listing_vehicle=vehicle,
+                        )
+                        if model and model.casefold() not in record.model.casefold():
+                            stats.cars_filtered += 1
+                            continue
+                        if not filters.matches({
+                            "brand": record.brand,
+                            "model": record.model,
+                            "year": record.year,
+                            "body_type": record.body_type,
+                            "color": record.color,
+                            "drive": record.drive,
+                            "gearbox": record.gearbox,
+                            "price": record.price,
+                            "mileage": record.mileage,
+                        }):
+                            stats.cars_filtered += 1
+                            continue
+                        prepared_rows.append(record)
+                        stats.ids_fetched += 1
+                        stats.cars_failed = max(0, stats.cars_failed - 1)
+                        if len(prepared_rows) >= self.batch_size:
+                            db_processed += self._flush_db_records(
+                                rows=prepared_rows,
+                                stats=stats,
+                                errors=errors,
+                                processed=db_processed + len(prepared_rows),
+                                total=len(candidates),
+                            )
+                            prepared_rows.clear()
+                    except Exception as exc:
+                        retry_errors.append({
+                            "vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
+                            "error": str(exc),
+                        })
+                        if _looks_like_protection(exc):
+                            stats.protection_events += 1
+                    if retry_index % 100 == 0 or retry_index == len(retry_candidates):
+                        elapsed = max(0.001, time.perf_counter() - retry_started)
+                        logger.info(
+                            "Fast HTTP-first retry progress: processed=%d/%d recovered=%d remaining_failed=%d rate=%.2f/s",
+                            retry_index,
+                            len(retry_candidates),
+                            len(retry_candidates) - len(retry_errors),
+                            len(retry_errors),
+                            retry_index / elapsed,
+                        )
+            transient_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in retry_candidates}
+            errors = [error for error in errors if error.get("vehicle_url") not in transient_urls]
+            errors.extend(retry_errors)
+
+        if prepared_rows:
+            db_processed += self._flush_db_records(
+                rows=prepared_rows,
+                stats=stats,
+                errors=errors,
+                processed=db_processed + len(prepared_rows),
+                total=len(candidates),
+            )
+            prepared_rows.clear()
+
+        self.client.persist_session_state()
+
+        scan_completed = not (limit is not None and limit > 0) and not errors
+        mark_sold_scope_partial = bool(
+            (limit is not None and limit > 0)
+            or make
+            or model
+            or listing_url
+            or only_new
+        )
+        if all_listing_origin_urls and not mark_sold_scope_partial and not skip_mark_sold and scan_completed:
+            try:
+                sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls, lane="iaai")
+            except Exception as exc:
+                sold_count = 0
+                logger.warning("Fast sold reconcile failed: %s", exc)
+        else:
+            sold_count = 0
+
+        listing = {
+            "mode": "fast_hidden_payload",
+            "vehicles_collected": len(candidates),
+            "vehicle_urls": [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates],
+            "early_stopped": False,
+            "truncated_by_time_budget": False,
+            "rows_seen": rows_seen,
+            "rows_filtered_condition": rows_skipped_condition,
+            "sold_marked": sold_count,
+        }
+        return self._result(
+            started_at=started_at,
+            stats=stats,
+            failures=errors,
+            listing=listing,
+            all_listing_origin_urls=all_listing_origin_urls,
+            full_scan_completed=scan_completed,
+        )
+
+    def _result(
+        self,
+        *,
+        started_at: float,
+        stats: FastSyncStats,
+        failures: list[dict[str, str]],
+        listing: dict[str, Any],
+        all_listing_origin_urls: set[str],
+        full_scan_completed: bool,
+    ) -> dict[str, Any]:
+        status = "success" if not failures else ("partial_success" if stats.cars_upserted else "failed")
+        total = int(listing.get("vehicles_collected") or 0)
+        fail_ratio = (stats.cars_failed / total) if total > 0 else 0.0
+        protection_ratio = (stats.protection_events / total) if total > 0 else 0.0
+        anti_bot_detected = total > 0 and ((stats.protection_events >= 30 and protection_ratio >= 0.10) or fail_ratio >= 0.30)
+        return {
+            "status": status,
+            "listing": listing,
+            "total": total,
+            "total_discovered": total,
+            "skipped_existing": stats.skipped_existing,
+            "cars_upserted": stats.cars_upserted,
+            "cars_failed": stats.cars_failed,
+            "cars_filtered": stats.cars_filtered,
+            "images_upserted": stats.images_upserted,
+            "protection_events": stats.protection_events,
+            "failures": failures,
+            "all_listing_origin_urls": all_listing_origin_urls,
+            "full_scan_completed": full_scan_completed and not anti_bot_detected,
+            "anti_bot_detected": anti_bot_detected,
+            "fail_ratio": round(fail_ratio, 4),
+            "protection_ratio": round(protection_ratio, 4),
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+        }
+
+    def _progress(self, stage: str, **meta: Any) -> None:
+        if self.report_progress is None:
+            return
+        try:
+            self.report_progress(stage, **meta)
+        except Exception:
+            pass
+
+    def _fetch_detail_payload(self, inventory_id: str) -> dict[str, Any]:
+        jitter = float(self.client._settings.scraping_profile.request_jitter_max_s)
+        if jitter > 0:
+            time.sleep(random.uniform(0.0, jitter))
+        return self.client.fetch_vehicle_detail_payload(inventory_id)
+
+    def _flush_db_records(
+        self,
+        *,
+        rows: list[CarRecord],
+        stats: FastSyncStats,
+        errors: list[dict[str, str]],
+        processed: int,
+        total: int,
+    ) -> int:
+        if not rows:
+            return 0
+        batch = list(rows)
+        try:
+            upsert = self.persistence.upsert_cars_batch(batch)
+            stats.cars_upserted += int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0))
+            stats.images_upserted += int(upsert.get("images_upserted", 0))
+        except Exception as exc:
+            stats.cars_failed += len(batch)
+            errors.append({"vehicle_url": f"db_batch_{processed - len(batch)}", "error": str(exc)})
+            logger.exception("Fast DB apply failed processed=%s size=%s: %s", processed, len(batch), exc)
+        self._progress(
+            "fast_db_progress",
+            processed=processed,
+            total=total,
+            cars_upserted=stats.cars_upserted,
+            cars_failed=stats.cars_failed,
+            images_upserted=stats.images_upserted,
+        )
+        logger.info(
+            "Fast HTTP-first DB batch: processed=%d/%d batch=%d upserted=%d failed=%d images=%d",
+            processed,
+            total,
+            len(batch),
+            stats.cars_upserted,
+            stats.cars_failed,
+            stats.images_upserted,
+        )
+        return len(batch)
+
+
+def _looks_like_protection(exc: Exception) -> bool:
+    message = str(exc).lower()
+    return any(token in message for token in ("captcha", "antibot", "challenge", "blocked", "403", "429", "incapsula"))

iaai_scraper/parsing/fast_mapper.py

@@ -0,0 +1,368 @@
+from __future__ import annotations
+
+import re
+from datetime import datetime, timezone
+from typing import Any
+from urllib.parse import urlparse
+
+from ..browser.fast_client import FastListingVehicle, build_resizer_images_from_keys, parse_int, parse_text
+from ..storage.enums import BODY_TYPE_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES
+from ..storage.schemas import CarRecord, ImageRecord
+
+ORIGIN_PREFIX = "iaai:"
+ORIGIN_URL_BASE = "https://www.iaai.com/VehicleDetail"
+DAMAGE_NEUTRAL_VALUES = {
+    "NORMAL WEAR & TEAR",
+    "NORMAL WEAR",
+    "NORMALWEAR&TEAR",
+    "NONE",
+    "NO DAMAGE",
+    "NO VISIBLE DAMAGE",
+    "MINOR DENT/SCRATCHES",
+}
+INACTIVE_STATUS_VALUES = {"SOLD", "SO", "CLOSED", "CN", "DELIVERED", "WITHDRAWN", "WDR", "COMPLETE", "COMPLETED"}
+
+
+class FastCarMapper:
+    """Maps IAAI ProductDetailsVM payloads directly to project CarRecord."""
+
+    def map_payload_to_record(
+        self,
+        *,
+        detail_payload: dict[str, Any],
+        vehicle_url: str | None = None,
+        listing_vehicle: FastListingVehicle | None = None,
+    ) -> CarRecord:
+        inventory_view = detail_payload.get("inventoryView")
+        if not isinstance(inventory_view, dict):
+            raise RuntimeError("detail payload missing inventoryView")
+        attributes = inventory_view.get("attributes")
+        if not isinstance(attributes, dict):
+            raise RuntimeError("detail payload missing inventoryView.attributes")
+
+        inventory_id = parse_text(attributes.get("Id"))
+        if not inventory_id and listing_vehicle is not None:
+            inventory_id = listing_vehicle.inventory_id
+        if not inventory_id and vehicle_url:
+            inventory_id = self._inventory_id_from_url(vehicle_url)
+        if not inventory_id:
+            raise RuntimeError("missing inventory id")
+
+        brand = self._limit_text(parse_text(attributes.get("Make")) or "UNKNOWN", 50)
+        model = self._build_model_name(parse_text(attributes.get("Model")), parse_text(attributes.get("Series"))) or "UNKNOWN"
+        model = self._limit_text(model, 50)
+        year = self._parse_year(attributes.get("Year"))
+
+        auction_info = detail_payload.get("auctionInformation")
+        auction_info = auction_info if isinstance(auction_info, dict) else {}
+        bidding_info = auction_info.get("biddingInformation")
+        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
+        prebid_info = auction_info.get("prebidInformation")
+        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
+
+        high_bid = self._first_positive_int(
+            prebid_info.get("decimalHighBidAmount"),
+            prebid_info.get("highBidAmount"),
+            bidding_info.get("highBidAmount"),
+        )
+        buy_now = self._first_positive_int(
+            bidding_info.get("buyNowAmount"),
+            prebid_info.get("buyNowPrice"),
+            bidding_info.get("buyNowPrice"),
+        )
+        price = high_bid if high_bid is not None else buy_now
+
+        image_dimensions = inventory_view.get("imageDimensions")
+        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
+        keys_container = image_dimensions.get("keys")
+        keys_container = keys_container if isinstance(keys_container, dict) else {}
+        image_keys = keys_container.get("$values")
+        image_keys = image_keys if isinstance(image_keys, list) else []
+        images = [ImageRecord.model_validate(row) for row in build_resizer_images_from_keys(image_keys)]
+
+        primary_damage = parse_text(attributes.get("PrimaryDamageDesc"))
+        secondary_damage = parse_text(attributes.get("SecondaryDamageDesc"))
+        origin_url = vehicle_url or f"{ORIGIN_URL_BASE}/{inventory_id}"
+        normalized_origin_id = self._normalize_origin_inventory_id(inventory_id)
+        origin_id = f"{ORIGIN_PREFIX}{normalized_origin_id}"
+
+        return CarRecord(
+            parser_id=self._generate_parser_id(origin_id),
+            brand=brand,
+            model=model,
+            year=year,
+            price=price,
+            currency=self._map_currency(parse_text(attributes.get("Currency")) or (listing_vehicle.currency if listing_vehicle else None)),
+            mileage=self._parse_non_negative_int(attributes.get("ODOValue")) or 0,
+            country=self._map_country(inventory_id),
+            is_sold=self._is_sold(listing_vehicle),
+            color=self._normalize_color(parse_text(attributes.get("ExteriorColor")) or parse_text(attributes.get("ColorDesc"))),
+            drive=self._map_drive(parse_text(attributes.get("DriveLineTypeDesc"))),
+            gearbox=self._map_gearbox(parse_text(attributes.get("Transmission"))),
+            steering_wheel="LEFT",
+            body_type=self._map_body_type(parse_text(attributes.get("BodyStyleName")) or parse_text(attributes.get("VehicleClass"))),
+            engine_volume=self._parse_engine_volume(parse_text(attributes.get("EngineSize")) or parse_text(attributes.get("EngineInformation"))),
+            selling_type="AUCTION",
+            one_owner=False,
+            new_car=False,
+            is_hidden=not bool(images),
+            origin="IAAI",
+            origin_url=origin_url,
+            origin_id=origin_id,
+            is_damaged=self._derive_is_damaged(primary_damage=primary_damage, secondary_damage=secondary_damage),
+            evaluation=parse_text(attributes.get("VehicleGrade")),
+            non_smoking=True,
+            rental=False,
+            repair_history=False,
+            slug=self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))),
+            last_seen_at=datetime.now(timezone.utc),
+            images=images,
+        )
+
+    def payload_to_summary(self, detail_payload: dict[str, Any], vehicle_url: str) -> dict[str, Any]:
+        inventory_view = detail_payload.get("inventoryView") if isinstance(detail_payload, dict) else {}
+        inventory_view = inventory_view if isinstance(inventory_view, dict) else {}
+        attr = inventory_view.get("attributes")
+        attr = attr if isinstance(attr, dict) else {}
+        auction_info = detail_payload.get("auctionInformation") if isinstance(detail_payload, dict) else {}
+        auction_info = auction_info if isinstance(auction_info, dict) else {}
+        bidding_info = auction_info.get("biddingInformation")
+        bidding_info = bidding_info if isinstance(bidding_info, dict) else {}
+        prebid_info = auction_info.get("prebidInformation")
+        prebid_info = prebid_info if isinstance(prebid_info, dict) else {}
+        image_dimensions = inventory_view.get("imageDimensions")
+        image_dimensions = image_dimensions if isinstance(image_dimensions, dict) else {}
+        keys_container = image_dimensions.get("keys")
+        keys_container = keys_container if isinstance(keys_container, dict) else {}
+        image_keys = keys_container.get("$values")
+        image_keys = image_keys if isinstance(image_keys, list) else []
+        image_urls = [str(row["fullres_image"]) for row in build_resizer_images_from_keys(image_keys)]
+        return {
+            "source_url": vehicle_url,
+            "lot_number": attr.get("Id") or attr.get("StockNumber") or attr.get("SalvageId"),
+            "year": attr.get("Year"),
+            "make": attr.get("Make"),
+            "model": attr.get("Model"),
+            "trim": attr.get("Series"),
+            "body_type": attr.get("BodyStyleName"),
+            "drive": attr.get("DriveLineTypeDesc"),
+            "engine": attr.get("EngineInformation") or attr.get("EngineSize"),
+            "fuel_type": attr.get("FuelTypeCode"),
+            "gearbox": attr.get("Transmission"),
+            "color": attr.get("ExteriorColor"),
+            "primary_damage": attr.get("PrimaryDamageDesc"),
+            "secondary_damage": attr.get("SecondaryDamageDesc"),
+            "odometer": attr.get("ODOValue"),
+            "location": attr.get("BranchName"),
+            "auction_date": attr.get("AuctionDateTime"),
+            "title": attr.get("Title"),
+            "current_bid": prebid_info.get("highBidAmount") or bidding_info.get("highBidAmount"),
+            "buy_now": prebid_info.get("buyNowPrice") or bidding_info.get("buyNowPrice"),
+            "actual_cash_value": attr.get("ProviderACV"),
+            "estimated_repair_cost": attr.get("EstRepairCost"),
+            "image_urls": image_urls,
+        }
+
+    @staticmethod
+    def _inventory_id_from_url(vehicle_url: str) -> str | None:
+        tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
+        return tail or None
+
+    @staticmethod
+    def _normalize_origin_inventory_id(inventory_id: str) -> str:
+        return inventory_id.strip().split("~", 1)[0]
+
+    @staticmethod
+    def _build_model_name(model: str | None, series: str | None) -> str:
+        unique_parts: list[str] = []
+        seen: set[str] = set()
+        for value in (model, series):
+            if not value:
+                continue
+            normalized = " ".join(value.split())
+            key = normalized.casefold()
+            if key in seen:
+                continue
+            seen.add(key)
+            unique_parts.append(normalized)
+        return " ".join(unique_parts).strip()
+
+    @staticmethod
+    def _parse_year(value: Any) -> int | None:
+        parsed = parse_int(value)
+        if parsed is None or parsed < 1900 or parsed > 2100:
+            return None
+        return parsed
+
+    @staticmethod
+    def _parse_non_negative_int(value: Any) -> int | None:
+        parsed = parse_int(value)
+        if parsed is None or parsed < 0:
+            return None
+        return parsed
+
+    @classmethod
+    def _first_positive_int(cls, *values: Any) -> int | None:
+        for value in values:
+            parsed = cls._parse_non_negative_int(value)
+            if parsed is not None and parsed > 0:
+                return parsed
+        return None
+
+    @staticmethod
+    def _normalize_color(value: str | None) -> str:
+        if not value:
+            return "other"
+        normalized = value.strip().lower()
+        if "/" in normalized:
+            normalized = normalized.split("/", 1)[0].strip()
+        return normalized or "other"
+
+    @staticmethod
+    def _map_currency(value: str | None) -> str:
+        normalized = (value or "USD").strip().upper()
+        return normalized if normalized in {"USD", "CAD", "EUR", "JPY", "RUB", "KRW", "AED", "GBP"} else "USD"
+
+    @staticmethod
+    def _map_country(inventory_id: str) -> str:
+        upper_id = inventory_id.strip().upper()
+        if upper_id.endswith("~CA"):
+            return "CA"
+        if upper_id.endswith("~US"):
+            return "US"
+        return "NA"
+
+    @staticmethod
+    def _map_drive(value: str | None) -> str | None:
+        if not value:
+            return None
+        normalized = value.strip().lower()
+        candidates: tuple[str, ...] | None = None
+        if "front" in normalized or normalized == "fwd":
+            candidates = ("FWD",)
+        elif "all wheel" in normalized or "4x4" in normalized or normalized == "awd" or "four wheel" in normalized:
+            candidates = ("4WD", "FOUR_WD")
+        elif "rear" in normalized or normalized == "rwd":
+            candidates = ("RWD",)
+        elif "2wd" in normalized or "two wheel" in normalized:
+            candidates = ("2WD", "TWO_WD")
+        elif "unknown" in normalized or normalized in {"na", "n/a"}:
+            candidates = ("NA",)
+        return FastCarMapper._select_allowed(candidates, DRIVE_ENUM_VALUES) if candidates else None
+
+    @staticmethod
+    def _map_gearbox(value: str | None) -> str | None:
+        if not value:
+            return None
+        normalized = value.strip().lower()
+        candidates: tuple[str, ...] | None = None
+        if "cvt" in normalized:
+            candidates = ("CVT",)
+        elif "manual" in normalized or normalized == "mt":
+            candidates = ("MT",)
+        elif "electric" in normalized or normalized == "ev":
+            candidates = ("EV",)
+        elif "auto" in normalized or normalized == "at":
+            candidates = ("AT",)
+        elif "unknown" in normalized or normalized in {"na", "n/a"}:
+            candidates = ("NA",)
+        return FastCarMapper._select_allowed(candidates, GEARBOX_ENUM_VALUES) if candidates else None
+
+    @staticmethod
+    def _map_body_type(value: str | None) -> str:
+        if not value:
+            return "OTHER"
+        normalized = value.strip().lower()
+        candidates: tuple[str, ...] | None = None
+        if "sedan" in normalized:
+            candidates = ("SEDAN",)
+        elif "sport utility" in normalized or normalized == "suv" or "crossover" in normalized:
+            candidates = ("SUV",)
+        elif "hatch" in normalized:
+            candidates = ("HATCHBACK",)
+        elif "wagon" in normalized:
+            candidates = ("STATION_WAGON", "Station Wagon")
+        elif "coupe" in normalized:
+            candidates = ("COUPE",)
+        elif "pickup" in normalized or ("crew" in normalized and "cab" in normalized):
+            candidates = ("PICKUP", "Pickup")
+        elif "convertible" in normalized or "roadster" in normalized or "cabrio" in normalized:
+            candidates = ("OPEN", "Open")
+        elif "van" in normalized:
+            candidates = ("MINIVAN",)
+        elif "truck" in normalized or "chassis" in normalized:
+            candidates = ("TRUCK", "Truck")
+        elif "rv" in normalized or "motorized" in normalized:
+            candidates = ("RV",)
+        elif normalized in {"other", "unknown"}:
+            candidates = ("OTHER", "Other")
+        return FastCarMapper._select_allowed(candidates, BODY_TYPE_ENUM_VALUES, fallback="OTHER") or "OTHER"
+
+    @staticmethod
+    def _parse_engine_volume(value: str | None) -> int | None:
+        if not value:
+            return None
+        match = re.search(r"(\d+(?:\.\d+)?)\s*[lL]\b", value)
+        if not match:
+            return None
+        try:
+            liters = float(match.group(1))
+        except ValueError:
+            return None
+        cc = int(round(liters * 1000))
+        if cc <= 0 or cc > 10000:
+            return None
+        return cc
+
+    @staticmethod
+    def _derive_is_damaged(*, primary_damage: str | None, secondary_damage: str | None) -> bool:
+        neutral = {item.replace(" ", "").strip().upper() for item in DAMAGE_NEUTRAL_VALUES}
+        for value in (primary_damage, secondary_damage):
+            if not value:
+                continue
+            normalized = value.replace(" ", "").strip().upper()
+            if normalized and normalized not in neutral:
+                return True
+        return False
+
+    @staticmethod
+    def _is_sold(listing_vehicle: FastListingVehicle | None) -> bool:
+        if listing_vehicle is None:
+            return False
+        if listing_vehicle.timed_auction_closed:
+            return True
+        status = (listing_vehicle.inventory_status or "").strip().upper()
+        return status in INACTIVE_STATUS_VALUES
+
+    @staticmethod
+    def _select_allowed(candidates: tuple[str, ...] | None, allowed: tuple[str, ...], fallback: str | None = None) -> str | None:
+        if not candidates:
+            return fallback if fallback in allowed else None
+        allowed_set = set(allowed)
+        for candidate in candidates:
+            if candidate in allowed_set:
+                return candidate
+        return fallback if fallback in allowed_set else None
+
+    @staticmethod
+    def _limit_text(value: str, max_length: int) -> str:
+        return value if len(value) <= max_length else value[:max_length].rstrip()
+
+    @staticmethod
+    def _slugify(value: str) -> str:
+        return re.sub(r"[^a-z0-9]+", "-", value.lower()).strip("-") or "car"
+
+    @staticmethod
+    def _generate_parser_id(origin_id: str) -> str:
+        import hashlib
+        from string import ascii_letters, digits
+
+        digest = hashlib.sha256(origin_id.encode()).digest()
+        alphabet = ascii_letters + digits
+        base = len(alphabet)
+        num = int.from_bytes(digest[:17], "big")
+        chars: list[str] = []
+        for _ in range(22):
+            num, idx = divmod(num, base)
+            chars.append(alphabet[idx])
+        return "car-" + "".join(chars)

iaai_scraper/parsing/mapper.py

@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
 
 
 class CarMapper:
-	# Преобразование данных IAAI в CarRecord.
+	# Маппер IAAI в CarRecord.
 
 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -48,7 +48,7 @@ class CarMapper:
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
 
 	def map_to_car_record(self, vehicle_url: str, vehicle_summary: dict[str, Any], payload_insights: dict[str, Any]) -> CarRecord:
-		# Собираем нормализованную DB-модель.
+		# Собираем DB-модель.
 		vehicle_summary = vehicle_summary or {}
 		payload_insights = payload_insights or {}
 		core = payload_insights.get("vehicle_core", {})
@@ -77,7 +77,7 @@ class CarMapper:
 		)
 		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
 		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
-		# Пытаемся определить привод из строки двигателя.
+		# Пробуем взять привод из двигателя.
 		if not drive or drive == "NA":
 			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
 			if engine_text:
@@ -144,7 +144,7 @@ class CarMapper:
 
 	@classmethod
 	def _to_money_int(cls, value: Any) -> int | None:
-		# Нормализация стоимости из разных форматов.
+		# Нормализация цены.
 		if value is None:
 			return None
 		if isinstance(value, bool):
@@ -168,14 +168,14 @@ class CarMapper:
 		for number in numbers:
 			clean = number.replace(" ", "")
 			if "," in clean and "." in clean:
-					# Поддержка 1,234.56 и 1.234,56.
+					# Поддержка двух форматов.
 				if clean.rfind(",") > clean.rfind("."):
 					clean = clean.replace(".", "").replace(",", ".")
 				else:
 					clean = clean.replace(",", "")
 			elif "," in clean:
 				parts = clean.split(",")
-				# 123,45 -> 123.45, иначе разделитель тысяч.
+				# Десятичный или тысячный разделитель.
 				if len(parts[-1]) in {1, 2} and len(parts) == 2:
 					clean = clean.replace(",", ".")
 				else:
@@ -214,7 +214,7 @@ class CarMapper:
 
 	@staticmethod
 	def _parse_odometer(value: Any) -> int:
-		# Разбор пробега из строк IAAI.
+		# Разбор пробега.
 		if value is None:
 			return 0
 		text = str(value).strip()
@@ -223,7 +223,7 @@ class CarMapper:
 		lowered = text.lower()
 		if any(skip in lowered for skip in ["not required", "exempt", "n/a", "na", "unknown", "tbd"]):
 			return 0
-		# Извлекаем число из строкового формата одометра.
+		# Ищем число.
 		numbers = re.findall(r"[\d,]+", text)
 		for num_str in numbers:
 			clean = num_str.replace(",", "")
@@ -294,7 +294,7 @@ class CarMapper:
 		empty_default: str | None,
 		fallback: str | None,
 	) -> str | None:
-		# Общий helper для enum-нормализации.
+		# Общая нормализация enum.
 		text = self._as_str(value).lower()
 		if not text:
 			return empty_default
@@ -329,7 +329,7 @@ class CarMapper:
 		return any(token in self._as_str(auction.get("sale_status")).lower() for token in ["sold", "closed", "ended"])
 
 	def _build_images(self, urls: list[Any]) -> list[ImageRecord]:
-		# Для imageKeys берем самый большой размер.
+		# Для imageKeys берём самый большой размер.
 		best_by_key: dict[str, str] = {}
 		key_order: list[str] = []
 		non_keyed: list[str] = []
@@ -375,11 +375,11 @@ class CarMapper:
 
 	@classmethod
 	def _generate_parser_id(cls, origin_id: str) -> str:
-		# Стабильный parser_id по origin_id.
+		# Стабильный parser_id.
 		digest = hashlib.sha256(origin_id.encode()).digest()
 		alphabet = cls._PARSER_ID_ALPHABET
 		base = len(alphabet)
-		num = int.from_bytes(digest[:17], "big")  # 17 байт = 136 бит, хватает на 22 символа
+		num = int.from_bytes(digest[:17], "big")  # Хватает на 22 символа.
 		chars: list[str] = []
 		for _ in range(22):
 			num, idx = divmod(num, base)
@@ -387,7 +387,7 @@ class CarMapper:
 		return "car-" + "".join(chars)
 
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
-		# Формат: iaai:{lot_number} (аналог copart:94323985).
+		# Формат: iaai:{lot_number}.
 		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
 			text = self._as_str(value)
 			if text:

iaai_scraper/parsing/parser.py

@@ -10,9 +10,9 @@ logger = logging.getLogger("iaai_scraper.parsers")
 
 
 class VehicleParser:
-	# Парсер данных страницы автомобиля.
+	# Парсер страницы авто.
 
-	# Регулярные выражения для парсинга и детекции защиты.
+	# Регулярки парсинга и защиты.
 	_BUY_NOW_RE = re.compile(r"Buy\s+Now[:\s]*\$\s*([\d,]+(?:\.\d{1,2})?)", re.IGNORECASE)
 	_CAPTCHA_TOKENS = frozenset(["captcha", "verify you are human", "i am human", "recaptcha", "cloudflare"])
 	_ANTIBOT_TOKENS = frozenset(["incapsula", "access denied", "request unsuccessful", "bot detection"])
@@ -101,11 +101,11 @@ class VehicleParser:
 		max_fields = len(set(self.DOM_LABEL_MAP.values()))
 
 		for i, line in enumerate(lines):
-			# Ранний выход, когда уже нашли все поля.
+			# Ранний выход.
 			if len(result) >= max_fields:
 				break
 
-			# Сценарий 1: "метка: значение" в одной строке.
+			# Метка и значение в одной строке.
 			colon_pos = line.find(":")
 			if colon_pos > 0:
 				label_part = line[:colon_pos].strip().lower()
@@ -116,7 +116,7 @@ class VehicleParser:
 						result[field_name] = value_part
 						continue
 
-			# Сценарий 2: метка и значение на соседних строках.
+			# Метка и значение в соседних строках.
 			clean = line.rstrip(":").strip().lower()
 			clean_alt = clean.rstrip("#").strip()
 			matched_label = None
@@ -164,7 +164,7 @@ class VehicleParser:
 			page_title = title_match.group(1).strip()
 		title_parsed = self._parse_title_for_year_make_model(page_title, dom_text)
 
-		# Один проход по payload для всех полей.
+		# Один проход по payload.
 		all_found = deep_find_all_keys(payloads, self.SUMMARY_KEY_MAP)
 
 		summary: dict[str, Any] = {"source_url": vehicle_url}
@@ -199,7 +199,7 @@ class VehicleParser:
 			p = item.get("payload")
 			if isinstance(p, (dict, list)):
 				payloads.append(p)
-				# Дополнительный проход по встроенному JSON.
+				# Доп. проход по JSON.
 				extra = deep_find_all_keys([p], self.SUMMARY_KEY_MAP)
 				for field, vals in extra.items():
 					if not summary.get(field):
@@ -207,7 +207,7 @@ class VehicleParser:
 						if v:
 							summary[field] = v
 
-		# Передаём image_urls без повторного извлечения.
+		# Передаём готовые image_urls.
 		image_urls = summary.get("image_urls") or []
 		return {
 			"vehicle_summary": summary,
@@ -325,7 +325,7 @@ class VehicleParser:
 		for script_text in scripts:
 			if "{" not in script_text and "[" not in script_text:
 				continue
-			# Пропускаем слишком большие минифицированные блоки.
+			# Пропускаем большие блоки.
 			if len(script_text) > 51_200:
 				continue
 			try:

iaai_scraper/storage/db.py

@@ -20,6 +20,7 @@ CAR_DB_FIELDS = {
 }
 
 _IN_CHUNK_SIZE = 5000
+CAR_TABLE_NAME = Car.__tablename__
 
 
 class PersistenceService:
@@ -35,11 +36,16 @@ class PersistenceService:
 			engine_kwargs["max_overflow"] = settings.database.max_overflow
 			engine_kwargs["pool_pre_ping"] = True
 			engine_kwargs["pool_recycle"] = settings.database.pool_recycle_seconds
+			engine_kwargs["pool_timeout"] = 30
+			# Таймауты запросов и блокировок.
+			engine_kwargs["connect_args"] = {
+				"options": "-c statement_timeout=120000 -c lock_timeout=30000"
+			}
 		self.engine = create_engine(settings.database.url, **engine_kwargs)
 		self.session_factory = sessionmaker(bind=self.engine, expire_on_commit=False, future=True)
 
 	def create_tables(self) -> None:
-		# В тестах/локально на SQLite разрешаем create_all; для non-SQLite в проде — только через миграции.
+		# Для SQLite можно create_all.
 		is_sqlite = self.settings.database.url.startswith("sqlite")
 		if not is_sqlite and not self.settings.database.auto_create_tables:
 			return
@@ -105,7 +111,7 @@ class PersistenceService:
 	def get_existing_urls_and_ids(
 		self, origin_urls: list[str], origin_ids: list[str],
 	) -> tuple[set[str], set[str]]:
-		# Загрузка существующих URL и origin_id.
+		# Загрузка URL и origin_id.
 		if not origin_urls and not origin_ids:
 			return set(), set()
 		urls: set[str] = set()
@@ -314,7 +320,7 @@ class PersistenceService:
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def upsert_car(self, record: CarRecord):
-		# Вставка или обновление автомобиля по origin_id/origin_url.
+		# Вставка или обновление авто.
 		payload = self._car_payload(record)
 		images = [image.model_dump(mode="python") for image in record.images]
 		with self.session_scope() as session:
@@ -375,7 +381,7 @@ class PersistenceService:
 		- Один DELETE по car_id IN (...) вместо удаления по одному.
 		- Fallback на по-одному upsert если batch commit упал.
 		"""
-		# ── Дедупликация записей внутри батча ──
+		# Дедупликация батча.
 		seen_ids: dict[str, int] = {}
 		unique_records: list[CarRecord] = []
 		for idx, r in enumerate(records):
@@ -406,20 +412,20 @@ class PersistenceService:
 		images_total = 0
 
 		with self.session_scope() as session:
-			# Получаем существующие записи chunked-запросами.
+			# Загружаем существующие записи.
 			origin_ids = [r.origin_id for r in records if r.origin_id]
 			origin_urls = [r.origin_url for r in records if r.origin_url]
 
 			existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
 
-			# Предзагружаем ВСЕ изображения для обновляемых машин одним запросом.
+			# Предзагружаем изображения.
 			existing_car_ids = set()
 			for record in records:
 				car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
 				if car is not None:
 					existing_car_ids.add(int(car.id))
 
-			# Строим маппинг car_id → set(image_urls) для сравнения.
+			# Готовим map car_id -> image_urls.
 			existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
 
 			new_cars: list[tuple[Car, list[dict]]] = []
@@ -441,7 +447,7 @@ class PersistenceService:
 					car.last_seen_at = record.last_seen_at
 					updated += 1
 
-					# Проверяем, изменились ли изображения.
+					# Проверяем изменения картинок.
 					new_image_urls = {img.get("fullres_image", "") for img in images}
 					old_image_urls = existing_images_map.get(int(car.id), set())
 					if new_image_urls != old_image_urls:
@@ -449,15 +455,15 @@ class PersistenceService:
 					else:
 						images_total += len(old_image_urls)
 
-			# Один flush для всех вставок.
+			# Один flush.
 			session.flush()
 
-			# Добавляем изображения для новых автомобилей.
+			# Добавляем картинки новым авто.
 			for car, images in new_cars:
 				self._add_images(session, int(car.id), images)
 				images_total += len(images)
 
-			# Массово обновляем изображения только для машин с изменёнными картинками.
+			# Обновляем только изменённые картинки.
 			if update_cars_needing_images:
 				update_ids = [int(car.id) for car, _ in update_cars_needing_images]
 				for i in range(0, len(update_ids), _IN_CHUNK_SIZE):
@@ -470,7 +476,7 @@ class PersistenceService:
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
 	def _upsert_cars_individually(self, records: list[CarRecord]) -> dict[str, int]:
-		# Fallback на поштучный upsert.
+		# Запасной поштучный upsert.
 		inserted = 0
 		updated = 0
 		images_total = 0
@@ -510,55 +516,107 @@ class PersistenceService:
 
 		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
 		что кардинально быстрее при больших объёмах (100K+ URLs).
+		Встроенная защита: нормализация URL (тильда/дефис) + safety-check на аномальный процент.
 		"""
 		if not active_origin_urls:
 			return 0
 
+		# Нормализация URL.
+		def _norm(url: str) -> str:
+			return url.replace("~", "-")
+
+		normalized_urls = {_norm(u) for u in active_origin_urls}
+
 		is_postgres = "postgresql" in self.settings.database.url
 
 		with self.session_scope() as session:
 			if is_postgres:
-				# Создаём временную таблицу с активными URL.
+				# Считаем кандидатов на sold.
+				total_active = session.execute(
+					text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
+				).scalar() or 0
+
+				if total_active == 0:
+					return 0
+
+				# Временная таблица URL.
 				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
 				session.execute(text("TRUNCATE _active_urls"))
 
-				# Вставляем активные URL чанками.
+				# Вставляем URL чанками.
-				url_list = list(active_origin_urls)
+				url_list = list(normalized_urls)
 				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
 					chunk = url_list[i:i + _IN_CHUNK_SIZE]
 					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
 					params = {f"u{j}": url for j, url in enumerate(chunk)}
 					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
 
-				# Создаём индекс на временной таблице для ускорения JOIN.
+				# Индекс для JOIN.
 				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
 
-				# Массовая пометка проданных в PostgreSQL.
+				# Считаем будущие sold.
+				would_mark = session.execute(text("""
+					SELECT count(*)
+					FROM {car_table} c
+					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
+					WHERE a.url IS NULL
+					AND c.is_sold = FALSE
+					AND c.origin_id LIKE 'iaai:%%'
+				""".format(car_table=CAR_TABLE_NAME))).scalar() or 0
+
+				# Защита от аномалии.
+				if total_active > 100 and would_mark > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						would_mark, total_active, would_mark / total_active * 100,
+					)
+					return 0
+
+				# Массовая пометка sold.
 				result = session.execute(text("""
-					UPDATE cars
+					UPDATE {car_table}
 					SET is_sold = TRUE
 					FROM (
 						SELECT c.id
-						FROM cars c
+						FROM {car_table} c
-						LEFT JOIN _active_urls a ON c.origin_url = a.url
+						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
 						AND c.origin_id LIKE 'iaai:%%'
 					) sub
-					WHERE cars.id = sub.id
+					WHERE {car_table}.id = sub.id
-				"""))
+				""".format(car_table=CAR_TABLE_NAME)))
 				count = result.rowcount or 0
 			else:
 				# Упрощённый путь для SQLite.
 				stmt = (
 					update(Car)
-					.where(Car.origin_url.notin_(active_origin_urls))
 					.where(Car.is_sold == False)  # noqa: E712
 					.where(Car.origin_id.like("iaai:%"))
 					.values(is_sold=True)
 				)
-				result = session.execute(stmt)
+				# Загружаем active URL.
-				count = result.rowcount or 0
+				all_active = session.execute(
+					select(Car.id, Car.origin_url).where(
+						Car.is_sold == False, Car.origin_id.like("iaai:%")  # noqa: E712
+					)
+				).all()
+				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
+
+				if not mark_ids:
+					return 0
+				total_active = len(all_active)
+				if total_active > 100 and len(mark_ids) > total_active * 0.8:
+					logger.error(
+						"mark_sold safety abort: would mark %d/%d (%.0f%%) as sold — likely URL format mismatch",
+						len(mark_ids), total_active, len(mark_ids) / total_active * 100,
+					)
+					return 0
+
+				for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
+					chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
+					session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
+				count = len(mark_ids)
 
 			if count:
 				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
@@ -573,4 +631,46 @@ class PersistenceService:
 			result = session.execute(
 				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
 			)
-			return {str(row[0]) for row in result if row and row[0]}
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
+		from sqlalchemy import func as sa_func
+		with self.session_scope() as session:
+			result = session.execute(
+				select(sa_func.count()).select_from(Car).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				)
+			)
+			return int(result.scalar() or 0)
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		prefix: str = "iaai:",
+		offset: int = 0,
+		limit: int = 500,
+	) -> list[str]:
+		"""Возвращает батч origin_url активных авто для rolling refresh.
+
+		Сортировка по last_seen_at ASC — давно не обновлённые идут первыми.
+		"""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url).where(
+					Car.origin_id.like(f"{prefix}%"),
+					Car.is_sold == False,  # noqa: E712
+				).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
+			)
+			return [str(row[0]) for row in result if row and row[0]]

iaai_scraper/storage/models.py

@@ -20,10 +20,10 @@ class Base(DeclarativeBase):
 
 
 class Car(Base):
-	__tablename__ = "cars"
+	__tablename__ = "iaai_cars"
 	__table_args__ = (
-		Index("ix_cars_brand_model", "brand", "model"),
+		Index("ix_iaai_cars_brand_model", "brand", "model"),
-		Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"),
+		Index("ix_iaai_cars_origin_id_not_sold", "origin_id", "is_sold"),
 	)
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -59,17 +59,17 @@ class Car(Base):
 
 
 class Image(Base):
-	__tablename__ = "images"
+	__tablename__ = "iaai_images"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
 	preview_image: Mapped[str] = mapped_column(String(), nullable=False)
 	order_index: Mapped[int] = mapped_column(Integer, nullable=False)
-	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True)
+	car_id: Mapped[int] = mapped_column(Integer, ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False, index=True)
 	car: Mapped[Car] = relationship("Car", back_populates="images")
 
 
 class SyncRun(Base):
-	__tablename__ = "sync_runs"
+	__tablename__ = "iaai_sync_runs"
 	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
 	started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
 	finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

iaai_scraper/worker/celery_app.py

@@ -1,6 +1,9 @@
 # Инициализация Celery-приложения и периодических задач.
 
+import json
 import logging
+import os
+import time
 
 from celery import Celery
 from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
@@ -11,6 +14,56 @@ from ..core.logs import setup_logging
 
 logger = logging.getLogger("iaai_scraper.worker.celery_app")
 STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
+IAAI_SYNC_QUEUE = "iaai_sync"
+PROGRESS_KEY_PREFIX = "iaai:state:task_progress:"
+SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
+SYNC_LAST_COMPLETED_AT_KEY = "iaai:state:sync_listing_last_completed_at"
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    raw = os.getenv(name, "true" if default else "false").strip().lower()
+    return raw in {"1", "true", "yes", "on"}
+
+
+def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
+    now = int(time.time())
+    try:
+        for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
+            payload = redis_client.get(raw_key)
+            if not payload:
+                continue
+            try:
+                progress = json.loads(payload)
+            except (TypeError, ValueError):
+                continue
+            ts = int(progress.get("ts") or 0)
+            stage = str(progress.get("stage") or "")
+            if ts > 0 and now - ts <= max_age_seconds and stage not in {"segment_done", "sync_done", "failed"}:
+                return True
+    except Exception:
+        logger.warning("Failed to inspect startup progress keys", exc_info=True)
+    return False
+
+
+def _seconds_until_next_allowed_sync(redis_client: Redis) -> int:
+    """Запрещает новый автозапуск раньше чем через интервал beat после завершения полного run."""
+    min_interval = max(0, int(settings.celery.beat_sync_interval_minutes * 60))
+    if min_interval <= 0:
+        return 0
+    try:
+        full_done = str(redis_client.get(SYNC_FULL_SCAN_DONE_KEY) or "").strip().lower() in {"1", "true", "yes", "on"}
+        if not full_done:
+            return 0
+        completed_raw = redis_client.get(SYNC_LAST_COMPLETED_AT_KEY)
+        if not completed_raw:
+            return 0
+        completed_at = int(float(completed_raw))
+    except Exception:
+        logger.warning("Failed to inspect last sync completion timestamp", exc_info=True)
+        return 0
+    elapsed = int(time.time()) - completed_at
+    return max(0, min_interval - elapsed)
 
 
 @celery_setup_logging.connect
@@ -27,12 +80,6 @@ def _on_worker_process_init(**kwargs):
     level = settings.log_level if settings.log_level else "INFO"
     setup_logging(level, None)
 
-    # Сбрасываем cached engine/redis после fork — иначе child наследует
-    # коннекты родителя, что небезопасно (особенно с psycopg2).
-    from . import tasks as _tasks
-    _tasks._CACHED_PERSISTENCE = None
-    _tasks._CACHED_REDIS = None
-
 
 def _broker_url() -> str:
     return settings.celery.broker_url or settings.redis.url
@@ -54,7 +101,7 @@ _soft = settings.celery.task_soft_time_limit
 _hard = settings.celery.task_time_limit
 _max_hard = _soft + 120 if _soft else _hard
 if _hard > _max_hard:
-    logger.warning(
+    logger.info(
         "CELERY_TASK_TIME_LIMIT=%d too far from CELERY_TASK_SOFT_TIME_LIMIT=%d; "
         "clamping hard limit to %d",
         _hard, _soft, _max_hard,
@@ -72,30 +119,33 @@ celery_app.conf.update(
     task_acks_late=True,
     task_reject_on_worker_lost=True,
     task_track_started=True,
-    task_ignore_result=True,
     worker_concurrency=settings.celery.worker_concurrency,
     worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
-    worker_max_memory_per_child=settings.celery.worker_max_memory_per_child_kb,
+    worker_pool=settings.celery.worker_pool,
-    worker_pool="prefork",
     worker_prefetch_multiplier=1,
     broker_connection_retry_on_startup=True,
     broker_transport_options={
         "visibility_timeout": settings.celery.broker_visibility_timeout,
     },
-    result_expires=3600,
+    result_expires=86400,
     worker_redirect_stdouts=False,
     worker_hijack_root_logger=False,
     beat_schedule={
         "periodic-sync-listing": {
-            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+            "task": "iaai.sync_cars_feed",
             "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
             "args": (),
-            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": False},
+            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
-            "options": {"queue": "scraping"},
+            "options": {
+                "queue": IAAI_SYNC_QUEUE,
+                "expires": settings.celery.beat_sync_interval_minutes * 60.0,
+                "headers": {"iaai_beat_task": True},
+            },
         }
     },
     task_routes={
-        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
+        "iaai.sync_cars_feed": {"queue": IAAI_SYNC_QUEUE},
+        "iaai_scraper.worker.tasks.*": {"queue": IAAI_SYNC_QUEUE},
     },
 )
 
@@ -104,8 +154,12 @@ celery_app.autodiscover_tasks(["iaai_scraper.worker"])
 
 @worker_ready.connect
 def _on_worker_ready(**kwargs):
-    """Сразу при старте worker отправляем первую задачу sync_listing,
+    """При старте worker отправляем первый sync_listing, если очередь пуста."""
-    чтобы не ждать час до первого beat-цикла."""
+    if not _env_bool("IAAI_STARTUP_SYNC_ENABLED", True):
+        logger.info("Worker ready: startup sync dispatch disabled by IAAI_STARTUP_SYNC_ENABLED")
+        return
+
+    redis_client = None
     try:
         redis_client = Redis.from_url(
             settings.redis.url,
@@ -115,13 +169,50 @@ def _on_worker_ready(**kwargs):
             health_check_interval=settings.redis.health_check_interval_seconds,
             retry_on_timeout=True,
         )
-        # Дедуп TTL = интервал beat (по умолчанию 1ч), чтобы не плодить дубликаты
+
-        # при flapping-рестартах контейнера.
+        has_fresh_progress = _has_fresh_active_progress(redis_client)
-        dedupe_ttl = max(600, settings.celery.beat_sync_interval_minutes * 60)
+        next_allowed_delay = _seconds_until_next_allowed_sync(redis_client)
-        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=dedupe_ttl))
+        if next_allowed_delay > 0:
+            logger.info(
+                "Worker ready: last full sync finished recently; next auto sync allowed in %ss, skip startup dispatch",
+                next_allowed_delay,
+            )
+            return
+
+        for stale_key in (SYNC_LISTING_LOCK_KEY,):
+            try:
+                ttl = redis_client.ttl(stale_key)
+                if ttl is not None and ttl != -2 and not has_fresh_progress:
+                    redis_client.delete(stale_key)
+                    logger.info("Cleared stale lock on startup: %s (ttl was %s)", stale_key, ttl)
+                elif ttl is not None and ttl != -2:
+                    logger.info("Keeping sync lock on startup because fresh active progress exists: %s (ttl=%s)", stale_key, ttl)
+            except Exception:
+                logger.warning("Failed to inspect stale lock %s on startup", stale_key, exc_info=True)
+
+        try:
+            queue_len = int(redis_client.llen(IAAI_SYNC_QUEUE) or 0)
+        except Exception:
+            queue_len = 0
+        if queue_len > 0:
+            logger.info("Worker ready: iaai_sync queue already has %d task(s); skip startup dispatch", queue_len)
+            return
+
+        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+        if not should_dispatch and not has_fresh_progress:
+            redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
+            should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
+            if should_dispatch:
+                logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists")
     except Exception:
         logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
         return
+    finally:
+        if redis_client is not None:
+            try:
+                redis_client.close()
+            except Exception:
+                pass
 
     if not should_dispatch:
         logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
@@ -129,7 +220,8 @@ def _on_worker_ready(**kwargs):
 
     logger.info("Worker ready — dispatching initial sync_listing task")
     celery_app.send_task(
-        "iaai_scraper.worker.tasks.sync_listing_task",
+        "iaai.sync_cars_feed",
         kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
-        queue="scraping",
+        queue=IAAI_SYNC_QUEUE,
-    )
+        expires=settings.celery.beat_sync_interval_minutes * 60.0,
+    )

iaai_scraper/worker/fast_sync.py

@@ -0,0 +1,479 @@
+from __future__ import annotations
+
+import concurrent.futures
+import logging
+import random
+import time
+from dataclasses import dataclass
+from typing import Any, Callable
+
+from .browser.fast_client import FastListingVehicle, IAAIFastClient
+from .core.runtime_config import RuntimeConfig
+from .parsing.fast_mapper import INACTIVE_STATUS_VALUES, FastCarMapper
+from .storage.db import PersistenceService
+from .storage.schemas import CarRecord
+
+logger = logging.getLogger("iaai_scraper.fast_sync")
+
+
+@dataclass(slots=True)
+class FastSyncStats:
+    ids_fetched: int = 0
+    cars_upserted: int = 0
+    cars_failed: int = 0
+    cars_filtered: int = 0
+    images_upserted: int = 0
+    skipped_existing: int = 0
+    protection_events: int = 0
+
+
+def passes_condition_check(row: FastListingVehicle) -> bool:
+    if row.timed_auction_closed:
+        return False
+    status = (row.inventory_status or "").strip().upper()
+    return status not in INACTIVE_STATUS_VALUES
+
+
+class FastSyncEngine:
+    """Full iaai-fast style sync pipeline adapted to this project's storage.
+
+    Flow: hidden listing payloads -> concurrent ProductDetailsVM HTTP fetch ->
+    CarRecord preparation in memory -> single batch DB upsert. Browser is used
+    only inside IAAIFastClient to refresh cookies when IAAI challenge appears.
+    """
+
+    def __init__(
+        self,
+        *,
+        client: IAAIFastClient,
+        mapper: FastCarMapper,
+        persistence: PersistenceService,
+        batch_size: int,
+        fetch_concurrency: int,
+        report_progress: Callable[[str, Any], None] | None = None,
+    ) -> None:
+        self.client = client
+        self.mapper = mapper
+        self.persistence = persistence
+        self.batch_size = max(1, int(batch_size))
+        self.fetch_concurrency = max(1, int(fetch_concurrency))
+        self.report_progress = report_progress
+
+    @staticmethod
+    def _is_transient_detail_error(exc: Exception) -> bool:
+        text = str(exc).lower()
+        return any(
+            marker in text
+            for marker in (
+                "sslerror",
+                "ssleoferror",
+                "unexpected_eof_while_reading",
+                "eof occurred in violation of protocol",
+                "max retries exceeded",
+                "read timed out",
+                "readtimeout",
+                "connection reset",
+                "connection aborted",
+                "connection closed",
+                "temporarily unavailable",
+                "too many requests",
+                "status=429",
+                "status=500",
+                "status=502",
+                "status=503",
+                "status=504",
+            )
+        )
+
+    def sync_listing(
+        self,
+        *,
+        runtime_config: RuntimeConfig,
+        make: str | None = None,
+        model: str | None = None,
+        lane: str = "iaai_cars",
+        limit: int | None = None,
+        only_new: bool = False,
+        listing_url: str | None = None,
+        max_pages: int | None = None,
+        skip_mark_sold: bool = False,
+    ) -> dict[str, Any]:
+        del lane
+        started_at = time.perf_counter()
+        stats = FastSyncStats()
+        errors: list[dict[str, str]] = []
+        selected: dict[str, FastListingVehicle] = {}
+        rows_seen = 0
+        rows_skipped_condition = 0
+
+        filters = runtime_config.filters
+        logger.info(
+            "Fast HTTP-first listing started: make=%s listing_url=%s max_pages=%s concurrency=%s batch_size=%s",
+            make or "ALL",
+            listing_url or "default",
+            max_pages,
+            self.fetch_concurrency,
+            self.batch_size,
+        )
+        for vehicle in self.client.iter_listing_vehicles(
+            listing_start_url=listing_url,
+            make=make,
+            max_pages=max_pages,
+        ):
+            rows_seen += 1
+            if runtime_config.sync.condition_check_enabled and not passes_condition_check(vehicle):
+                rows_skipped_condition += 1
+                continue
+            if vehicle.inventory_id in selected:
+                continue
+            selected[vehicle.inventory_id] = vehicle
+            if limit is not None and limit > 0 and len(selected) >= limit:
+                break
+
+        candidates = list(selected.values())
+        all_listing_origin_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates}
+        if only_new and candidates:
+            origin_urls = [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates]
+            origin_ids = [f"iaai:{v.inventory_id}" for v in candidates]
+            existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(origin_urls, origin_ids)
+            fresh: list[FastListingVehicle] = []
+            for vehicle in candidates:
+                if f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}" in existing_urls or f"iaai:{vehicle.inventory_id}" in existing_ids:
+                    stats.skipped_existing += 1
+                    continue
+                fresh.append(vehicle)
+            candidates = fresh
+
+        self._progress(
+            "fast_listing_collected",
+            rows_seen=rows_seen,
+            rows_filtered_condition=rows_skipped_condition,
+            rows_selected=len(candidates),
+            skipped_existing=stats.skipped_existing,
+        )
+        logger.info(
+            "Fast HTTP-first listing collected: rows_seen=%d selected=%d skipped_existing=%d filtered_condition=%d only_new=%s",
+            rows_seen,
+            len(candidates),
+            stats.skipped_existing,
+            rows_skipped_condition,
+            only_new,
+        )
+
+        scan_completed = not (limit is not None and limit > 0) and not errors
+
+        if not candidates:
+            return self._result(
+                started_at=started_at,
+                stats=stats,
+                failures=errors,
+                listing={
+                    "mode": "fast_hidden_payload",
+                    "vehicles_collected": 0,
+                    "vehicle_urls": [],
+                    "early_stopped": False,
+                    "truncated_by_time_budget": False,
+                    "rows_seen": rows_seen,
+                    "rows_filtered_condition": rows_skipped_condition,
+                },
+                all_listing_origin_urls=all_listing_origin_urls,
+                full_scan_completed=scan_completed,
+            )
+
+        prepared_rows: list[CarRecord] = []
+        db_processed = 0
+        retry_candidates: list[FastListingVehicle] = []
+        transient_failure_log_count = 0
+        started_details = time.perf_counter()
+        with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor:
+            future_to_vehicle = {
+                executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
+                for vehicle in candidates
+            }
+            for index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
+                vehicle = future_to_vehicle[future]
+                try:
+                    payload = future.result()
+                    record = self.mapper.map_payload_to_record(
+                        detail_payload=payload,
+                        vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
+                        listing_vehicle=vehicle,
+                    )
+                    if model and model.casefold() not in record.model.casefold():
+                        stats.cars_filtered += 1
+                        continue
+                    if not filters.matches({
+                        "brand": record.brand,
+                        "model": record.model,
+                        "year": record.year,
+                        "body_type": record.body_type,
+                        "color": record.color,
+                        "drive": record.drive,
+                        "gearbox": record.gearbox,
+                        "price": record.price,
+                        "mileage": record.mileage,
+                    }):
+                        stats.cars_filtered += 1
+                        continue
+                    prepared_rows.append(record)
+                    stats.ids_fetched += 1
+                    if len(prepared_rows) >= self.batch_size:
+                        db_processed += self._flush_db_records(
+                            rows=prepared_rows,
+                            stats=stats,
+                            errors=errors,
+                            processed=db_processed + len(prepared_rows),
+                            total=len(candidates),
+                        )
+                        prepared_rows.clear()
+                except Exception as exc:
+                    stats.cars_failed += 1
+                    errors.append({"vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}", "error": str(exc)})
+                    if _looks_like_protection(exc):
+                        stats.protection_events += 1
+                    if self._is_transient_detail_error(exc):
+                        retry_candidates.append(vehicle)
+                        transient_failure_log_count += 1
+                        if transient_failure_log_count % 100 == 0:
+                            logger.warning(
+                                "Fast detail transient failures queued for retry: count=%d latest_inventory_id=%s",
+                                transient_failure_log_count,
+                                vehicle.inventory_id,
+                            )
+                    else:
+                        logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc)
+
+                if index % 100 == 0 or index == len(candidates):
+                    elapsed = max(0.001, time.perf_counter() - started_details)
+                    if self.client._settings.scraping_profile.verbose_progress_logs:
+                        logger.info(
+                            "Fast HTTP-first details progress: processed=%d/%d ok=%d failed=%d queued_db=%d rate=%.2f/s",
+                            index,
+                            len(candidates),
+                            stats.ids_fetched,
+                            stats.cars_failed,
+                            len(prepared_rows),
+                            index / elapsed,
+                        )
+                    self._progress(
+                        "fast_detail_progress",
+                        processed=index,
+                        total=len(candidates),
+                        ids_fetched=stats.ids_fetched,
+                        cars_failed=stats.cars_failed,
+                        queued_for_db=len(prepared_rows),
+                        throughput=round(index / elapsed, 2),
+                    )
+
+        if retry_candidates:
+            retry_started = time.perf_counter()
+            retry_workers = max(1, min(8, self.fetch_concurrency // 2, len(retry_candidates)))
+            retry_errors: list[dict[str, str]] = []
+            logger.info(
+                "Fast HTTP-first retrying transient detail failures: total=%d workers=%d",
+                len(retry_candidates),
+                retry_workers,
+            )
+            with concurrent.futures.ThreadPoolExecutor(max_workers=retry_workers) as executor:
+                future_to_vehicle = {
+                    executor.submit(self._fetch_detail_payload, vehicle.inventory_id): vehicle
+                    for vehicle in retry_candidates
+                }
+                for retry_index, future in enumerate(concurrent.futures.as_completed(future_to_vehicle), start=1):
+                    vehicle = future_to_vehicle[future]
+                    try:
+                        payload = future.result()
+                        record = self.mapper.map_payload_to_record(
+                            detail_payload=payload,
+                            vehicle_url=f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
+                            listing_vehicle=vehicle,
+                        )
+                        if model and model.casefold() not in record.model.casefold():
+                            stats.cars_filtered += 1
+                            continue
+                        if not filters.matches({
+                            "brand": record.brand,
+                            "model": record.model,
+                            "year": record.year,
+                            "body_type": record.body_type,
+                            "color": record.color,
+                            "drive": record.drive,
+                            "gearbox": record.gearbox,
+                            "price": record.price,
+                            "mileage": record.mileage,
+                        }):
+                            stats.cars_filtered += 1
+                            continue
+                        prepared_rows.append(record)
+                        stats.ids_fetched += 1
+                        stats.cars_failed = max(0, stats.cars_failed - 1)
+                        if len(prepared_rows) >= self.batch_size:
+                            db_processed += self._flush_db_records(
+                                rows=prepared_rows,
+                                stats=stats,
+                                errors=errors,
+                                processed=db_processed + len(prepared_rows),
+                                total=len(candidates),
+                            )
+                            prepared_rows.clear()
+                    except Exception as exc:
+                        retry_errors.append({
+                            "vehicle_url": f"https://www.iaai.com/VehicleDetail/{vehicle.inventory_id}",
+                            "error": str(exc),
+                        })
+                        if _looks_like_protection(exc):
+                            stats.protection_events += 1
+                    if retry_index % 100 == 0 or retry_index == len(retry_candidates):
+                        elapsed = max(0.001, time.perf_counter() - retry_started)
+                        logger.info(
+                            "Fast HTTP-first retry progress: processed=%d/%d recovered=%d remaining_failed=%d rate=%.2f/s",
+                            retry_index,
+                            len(retry_candidates),
+                            len(retry_candidates) - len(retry_errors),
+                            len(retry_errors),
+                            retry_index / elapsed,
+                        )
+            transient_urls = {f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in retry_candidates}
+            errors = [error for error in errors if error.get("vehicle_url") not in transient_urls]
+            errors.extend(retry_errors)
+
+        if prepared_rows:
+            db_processed += self._flush_db_records(
+                rows=prepared_rows,
+                stats=stats,
+                errors=errors,
+                processed=db_processed + len(prepared_rows),
+                total=len(candidates),
+            )
+            prepared_rows.clear()
+
+        self.client.persist_session_state()
+
+        scan_completed = not (limit is not None and limit > 0) and not errors
+        mark_sold_scope_partial = bool(
+            (limit is not None and limit > 0)
+            or make
+            or model
+            or listing_url
+            or only_new
+        )
+        if all_listing_origin_urls and not mark_sold_scope_partial and not skip_mark_sold and scan_completed:
+            try:
+                sold_count = self.persistence.mark_sold_not_in_listing_by_urls(all_listing_origin_urls, lane="iaai")
+            except Exception as exc:
+                sold_count = 0
+                logger.warning("Fast sold reconcile failed: %s", exc)
+        else:
+            sold_count = 0
+
+        listing = {
+            "mode": "fast_hidden_payload",
+            "vehicles_collected": len(candidates),
+            "vehicle_urls": [f"https://www.iaai.com/VehicleDetail/{v.inventory_id}" for v in candidates],
+            "early_stopped": False,
+            "truncated_by_time_budget": False,
+            "rows_seen": rows_seen,
+            "rows_filtered_condition": rows_skipped_condition,
+            "sold_marked": sold_count,
+        }
+        return self._result(
+            started_at=started_at,
+            stats=stats,
+            failures=errors,
+            listing=listing,
+            all_listing_origin_urls=all_listing_origin_urls,
+            full_scan_completed=scan_completed,
+        )
+
+    def _result(
+        self,
+        *,
+        started_at: float,
+        stats: FastSyncStats,
+        failures: list[dict[str, str]],
+        listing: dict[str, Any],
+        all_listing_origin_urls: set[str],
+        full_scan_completed: bool,
+    ) -> dict[str, Any]:
+        status = "success" if not failures else ("partial_success" if stats.cars_upserted else "failed")
+        total = int(listing.get("vehicles_collected") or 0)
+        fail_ratio = (stats.cars_failed / total) if total > 0 else 0.0
+        protection_ratio = (stats.protection_events / total) if total > 0 else 0.0
+        anti_bot_detected = total > 0 and ((stats.protection_events >= 30 and protection_ratio >= 0.10) or fail_ratio >= 0.30)
+        return {
+            "status": status,
+            "listing": listing,
+            "total": total,
+            "total_discovered": total,
+            "skipped_existing": stats.skipped_existing,
+            "cars_upserted": stats.cars_upserted,
+            "cars_failed": stats.cars_failed,
+            "cars_filtered": stats.cars_filtered,
+            "images_upserted": stats.images_upserted,
+            "protection_events": stats.protection_events,
+            "failures": failures,
+            "all_listing_origin_urls": all_listing_origin_urls,
+            "full_scan_completed": full_scan_completed and not anti_bot_detected,
+            "anti_bot_detected": anti_bot_detected,
+            "fail_ratio": round(fail_ratio, 4),
+            "protection_ratio": round(protection_ratio, 4),
+            "elapsed_seconds": round(time.perf_counter() - started_at, 3),
+        }
+
+    def _progress(self, stage: str, **meta: Any) -> None:
+        if self.report_progress is None:
+            return
+        try:
+            self.report_progress(stage, **meta)
+        except Exception:
+            pass
+
+    def _fetch_detail_payload(self, inventory_id: str) -> dict[str, Any]:
+        jitter = float(self.client._settings.scraping_profile.request_jitter_max_s)
+        if jitter > 0:
+            time.sleep(random.uniform(0.0, jitter))
+        return self.client.fetch_vehicle_detail_payload(inventory_id)
+
+    def _flush_db_records(
+        self,
+        *,
+        rows: list[CarRecord],
+        stats: FastSyncStats,
+        errors: list[dict[str, str]],
+        processed: int,
+        total: int,
+    ) -> int:
+        if not rows:
+            return 0
+        batch = list(rows)
+        try:
+            upsert = self.persistence.upsert_cars_batch(batch)
+            stats.cars_upserted += int(upsert.get("inserted", 0)) + int(upsert.get("updated", 0))
+            stats.images_upserted += int(upsert.get("images_upserted", 0))
+        except Exception as exc:
+            stats.cars_failed += len(batch)
+            errors.append({"vehicle_url": f"db_batch_{processed - len(batch)}", "error": str(exc)})
+            logger.exception("Fast DB apply failed processed=%s size=%s: %s", processed, len(batch), exc)
+        self._progress(
+            "fast_db_progress",
+            processed=processed,
+            total=total,
+            cars_upserted=stats.cars_upserted,
+            cars_failed=stats.cars_failed,
+            images_upserted=stats.images_upserted,
+        )
+        logger.info(
+            "Fast HTTP-first DB batch: processed=%d/%d batch=%d upserted=%d failed=%d images=%d",
+            processed,
+            total,
+            len(batch),
+            stats.cars_upserted,
+            stats.cars_failed,
+            stats.images_upserted,
+        )
+        return len(batch)
+
+
+def _looks_like_protection(exc: Exception) -> bool:
+    message = str(exc).lower()
+    return any(token in message for token in ("captcha", "antibot", "challenge", "blocked", "403", "429", "incapsula"))

iaai_scraper/worker/self_heal.py

@@ -0,0 +1,346 @@
+import json
+import logging
+import os
+import random
+import signal
+import time
+
+from redis import Redis
+
+
+logger = logging.getLogger("iaai_scraper.worker.self_heal")
+
+IAAI_SYNC_QUEUE = "iaai_sync"
+GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
+GLOBAL_DB_PROGRESS_TS_KEY = "iaai:state:last_db_progress_ts"
+SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress"
+SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
+SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
+SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
+SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
+TERMINAL_PROGRESS_STAGES = {
+	"segment_done",
+	"segment_failed",
+	"segment_task_completed",
+	"segment_task_failed",
+	"segment_task_soft_timeout",
+	"sync_done",
+	"failed",
+}
+
+
+def _env_bool(name: str, default: bool) -> bool:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	return value.strip().lower() in {"1", "true", "yes", "on"}
+
+
+def _env_int(name: str, default: int) -> int:
+	value = os.getenv(name)
+	if value is None:
+		return default
+	try:
+		return int(value.strip())
+	except Exception:
+		return default
+
+
+def _get_redis() -> Redis:
+	url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0")
+	return Redis.from_url(
+		url,
+		decode_responses=True,
+		socket_connect_timeout=5.0,
+		socket_timeout=10.0,
+		health_check_interval=30,
+		retry_on_timeout=True,
+	)
+
+
+def _safe_int(value: str | None, default: int = 0) -> int:
+	if value is None:
+		return default
+	try:
+		return int(str(value).strip())
+	except Exception:
+		return default
+
+
+def _read_last_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
+	# Это дороже, но выполняется только при отсутствии основного маркера.
+	max_ts = 0
+	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			ts = _safe_int(data.get("ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	max_ts = 0
+	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			if str(data.get("stage") or "") == "fast_db_progress":
+				ts = _safe_int(data.get("ts"), 0)
+			else:
+				ts = _safe_int(data.get("last_db_progress_ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _has_active_recent_progress(redis_client: Redis, now_ts: int, stall_seconds: int) -> bool:
+	"""Return True when a task is still reporting non-DB progress.
+
+	Hourly only_new runs can legitimately skip every listed vehicle as existing.
+	Those runs may not emit fast_db_progress for a long time, but they still emit
+	regular listing/segment progress. Treating old DB timestamps as fatal caused
+	the watchdog to kill healthy production workers during such scans.
+	"""
+	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			stage = str(data.get("stage") or "")
+			if stage in {"failed", "sync_done", "segment_task_failed", "segment_task_soft_timeout"}:
+				continue
+			ts = _safe_int(data.get("ts"), 0)
+			if ts > 0 and now_ts - ts <= max(60, int(stall_seconds)):
+				return True
+		except Exception:
+			continue
+	return False
+
+
+def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None:
+	pipe = redis_client.pipeline()
+	pipe.delete(SYNC_LISTING_CHECKPOINT_KEY)
+	pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
+	pipe.delete(GLOBAL_PROGRESS_TS_KEY)
+	pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY)
+	pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0")
+	pipe.execute()
+
+
+def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
+	"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
+	queue_len = _safe_int(redis_client.llen(queue_name), 0)
+	has_lock = 1 if redis_client.get(SYNC_LISTING_LOCK_KEY) else 0
+	has_task_progress = 0
+	progress_keys_seen = 0
+	stale_progress_deleted = 0
+	stale_seconds = max(180, min(_env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720), 1800))
+	now_ts = int(time.time())
+	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		progress_keys_seen += 1
+		if queue_len > 0 or has_lock == 1:
+			has_task_progress = 1
+			break
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			stage = str(data.get("stage") or "")
+			ts = _safe_int(data.get("ts"), 0)
+			is_terminal = stage in TERMINAL_PROGRESS_STAGES
+			is_stale = ts <= 0 or now_ts - ts > stale_seconds
+			if is_terminal or is_stale:
+				redis_client.delete(key)
+				stale_progress_deleted += 1
+				continue
+			has_task_progress = 1
+			break
+		except Exception:
+			# Битые progress payload не должны держать worker в вечном false-stall цикле.
+			try:
+				redis_client.delete(key)
+				stale_progress_deleted += 1
+			except Exception:
+				pass
+	flags = {
+		"queue_len": queue_len,
+		"has_lock": has_lock,
+		"has_task_progress": has_task_progress,
+		"progress_keys_seen": progress_keys_seen,
+		"stale_progress_deleted": stale_progress_deleted,
+	}
+	return (queue_len > 0 or has_lock == 1 or has_task_progress == 1), flags
+
+
+def _kill_worker_process() -> None:
+	pid_file = "/tmp/celery-worker.pid"
+	pid: int | None = None
+	try:
+		with open(pid_file, "r", encoding="utf-8") as f:
+			pid = int(f.read().strip())
+	except Exception:
+		pid = None
+
+	if not pid:
+		logger.error("Self-heal: failed to read worker pid from %s", pid_file)
+		return
+
+	logger.error("Self-heal: terminating stuck worker process pid=%s", pid)
+	try:
+		os.kill(pid, signal.SIGTERM)
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGTERM to pid=%s", pid)
+		return
+
+	time.sleep(20)
+	try:
+		# Если процесс ещё жив — принудительно убиваем.
+		os.kill(pid, 0)
+		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
+		os.kill(pid, SIGKILL_FALLBACK)
+	except ProcessLookupError:
+		pass
+	except Exception:
+		logger.exception("Self-heal: failed to send SIGKILL to pid=%s", pid)
+
+
+def main() -> None:
+	if not _env_bool("IAAI_SELF_HEAL_ENABLED", True):
+		logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED")
+		return
+
+	queue_name = os.getenv("IAAI_CELERY_QUEUE", IAAI_SYNC_QUEUE)
+	check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
+	stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720))
+	db_idle_seconds = max(60, _env_int("IAAI_DB_IDLE_RESTART_SECONDS", 3600))
+	startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
+	restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
+
+	logger.info(
+		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss db_idle=%ss startup_grace=%ss cooldown=%ss",
+		queue_name,
+		check_interval,
+		stall_seconds,
+		db_idle_seconds,
+		startup_grace,
+		restart_cooldown,
+	)
+
+	started_at = time.time()
+	redis_client: Redis | None = None
+
+	while True:
+		try:
+			if redis_client is None:
+				redis_client = _get_redis()
+				redis_client.ping()
+
+			has_inflight, inflight = _has_inflight_work(redis_client, queue_name)
+			if not has_inflight:
+				time.sleep(check_interval)
+				continue
+
+			last_progress_ts = _read_last_progress_ts(redis_client)
+			now_ts = int(time.time())
+			age = None if last_progress_ts is None else max(0, now_ts - int(last_progress_ts))
+
+			if age is None:
+				if now_ts - int(started_at) < startup_grace:
+					time.sleep(check_interval)
+					continue
+				logger.warning(
+					"Self-heal: inflight=%s but no progress timestamp found after startup grace",
+					inflight,
+				)
+				age = stall_seconds + 1
+
+			restart_reason = f"progress_age={age}s > {stall_seconds}s"
+			db_idle_restart = False
+			if age <= stall_seconds:
+				# If the task is actively reporting progress, do not require DB writes.
+				# Hourly only_new listing scans often skip already-known cars and can
+				# legitimately have no DB writes while still moving through segments.
+				if _has_active_recent_progress(redis_client, now_ts, stall_seconds):
+					time.sleep(check_interval)
+					continue
+				last_db_ts = _read_last_db_progress_ts(redis_client)
+				db_age = None if last_db_ts is None else max(0, now_ts - int(last_db_ts))
+				if db_age is None:
+					first_allowed_ts = int(started_at) + max(startup_grace, db_idle_seconds)
+					if now_ts < first_allowed_ts:
+						time.sleep(check_interval)
+						continue
+					db_age = db_idle_seconds + 1
+				if db_age <= db_idle_seconds:
+					time.sleep(check_interval)
+					continue
+				db_idle_restart = True
+				restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s"
+
+			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
+			acquired = bool(
+				redis_client.set(
+					SELF_HEAL_RESTART_LOCK_KEY,
+					str(now_ts),
+					nx=True,
+					ex=restart_cooldown,
+				)
+			)
+			if not acquired:
+				time.sleep(check_interval)
+				continue
+
+			logger.error(
+				"Self-heal: detected stall (inflight=%s, %s). Restarting worker process...",
+				inflight,
+				restart_reason,
+			)
+			if db_idle_restart:
+				logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1")
+				_reset_bootstrap_checkpoint_for_db_idle(redis_client)
+			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
+			# перезапуск был не строго одновременно.
+			time.sleep(random.uniform(0.3, 2.0))
+			_kill_worker_process()
+			# После kill pid1 контейнер будет перезапущен Docker restart-policy.
+			# На случай неуспеха не молотим цикл.
+			time.sleep(check_interval)
+
+		except Exception:
+			logger.exception("Self-heal watchdog iteration failed")
+			redis_client = None
+			time.sleep(check_interval)
+
+
+if __name__ == "__main__":
+	logging.basicConfig(
+		level=os.getenv("IAAI_LOG_LEVEL", "INFO"),
+		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
+	)
+	main()

pyproject.toml

@@ -17,6 +17,7 @@ dependencies = [
   "pydantic>=2.8.2",
   "python-dotenv>=1.0.1",
   "redis>=5.2.0",
+  "requests>=2.32.0",
   "sqlalchemy>=2.0.32",
   "uvicorn>=0.34.0",
   "urllib3>=2.0.0",

runtime_config.json

@@ -1,6 +1,6 @@
 {
   "sync": {
-    "name": null,
+    "name": "https://www.iaai.com/Search?url=B%2bU066dM8%2flZtRvnzTwIEi8Pib9%2fM2fLpCTQDIgQRm4%3d",
     "ids_initial_size": null,
     "ids_next_size": null,
     "ids_max_pages": null,

status_vps.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+import os
+import paramiko
+
+HOST = "2.26.123.84"
+USER = "root"
+PASSWORD = os.environ["VPS_PASSWORD"]
+
+commands = [
+    "cd /root/iaai-parser && docker compose ps",
+    "cd /root/iaai-parser && docker compose exec -T worker celery -A iaai_scraper.worker.celery_app inspect active reserved scheduled",
+    "docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c \"SELECT now() AS ts, (SELECT count(*) FROM iaai_cars) AS cars, (SELECT count(*) FROM iaai_sync_runs) AS runs, (SELECT status FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_status, (SELECT cars_upserted FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_upserted, (SELECT cars_failed FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_failed, (SELECT started_at FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_started;\"",
+    "docker logs --since 10m iaai-parser-worker-1 2>&1 | tail -n 160",
+]
+
+client = paramiko.SSHClient()
+client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
+client.connect(
+    HOST,
+    username=USER,
+    password=PASSWORD,
+    look_for_keys=False,
+    allow_agent=False,
+    timeout=30,
+    banner_timeout=30,
+    auth_timeout=30,
+)
+try:
+    for command in commands:
+        print(f"\n=== REMOTE_RUN {command} ===", flush=True)
+        stdin, stdout, stderr = client.exec_command(command, timeout=180)
+        exit_code = stdout.channel.recv_exit_status()
+        print(stdout.read().decode("utf-8", "replace"), end="")
+        print(stderr.read().decode("utf-8", "replace"), end="")
+        print(f"\n=== EXIT {exit_code} ===", flush=True)
+finally:
+    client.close()

tests/test_fast_client.py

@@ -0,0 +1,117 @@
+from __future__ import annotations
+
+import json
+
+from iaai_scraper.browser.fast_client import (
+    DETAIL_MARKER,
+    LISTING_MARKER,
+    build_resizer_images_from_keys,
+    is_challenge_response,
+    parse_listing_page,
+    parse_product_details_vm,
+)
+from iaai_scraper.parsing.fast_mapper import FastCarMapper
+
+
+def test_parse_listing_page_extracts_hidden_payloads() -> None:
+    gbp = {"CurrentPage": 1, "PageSize": 100}
+    vehicles = [
+        {
+            "Id": "45078011~US",
+            "Tenant": "US",
+            "InventoryStatus": "RS",
+            "Currency": "USD",
+            "PreBidIndicator": True,
+        }
+    ]
+    html = (
+        f'<input id="GBPSearchQuery" value="{json.dumps(gbp).replace(chr(34), "&quot;")}" />'
+        f'<input id="VehicleDetails" value="{json.dumps(vehicles).replace(chr(34), "&quot;")}" />'
+        '<input id="ResultCount" value="1" />'
+        '<input id="PageSize" value="100" />'
+        '<input id="CurrentPage" value="1" />'
+    )
+
+    parsed = parse_listing_page(html)
+
+    assert parsed.result_count == 1
+    assert parsed.page_size == 100
+    assert parsed.current_page == 1
+    assert parsed.vehicles[0].inventory_id == "45078011~US"
+    assert parsed.vehicles[0].inventory_status == "RS"
+
+
+def test_parse_product_details_vm_and_map_record() -> None:
+    payload = {
+        "inventoryView": {
+            "attributes": {
+                "Id": "45078011~US",
+                "Year": "2002",
+                "Make": "ACURA",
+                "Model": "RSX",
+                "Series": "BASE",
+                "Currency": "USD",
+                "ODOValue": "219187",
+                "ExteriorColor": "GRAY",
+                "DriveLineTypeDesc": "FWD",
+                "Transmission": "Automatic",
+                "BodyStyleName": "COUPE",
+                "EngineSize": "2.0L I-4",
+                "VehicleGrade": "50",
+                "PrimaryDamageDesc": "NORMAL WEAR & TEAR",
+            },
+            "imageDimensions": {
+                "keys": {
+                    "$values": [
+                        {"k": "45078011~US~I1", "w": 1600, "h": 1200, "i": 0},
+                    ]
+                }
+            },
+        },
+        "auctionInformation": {
+            "prebidInformation": {"decimalHighBidAmount": "600", "highBidAmount": "$600"},
+            "biddingInformation": {"buyNowPrice": "$0"},
+        },
+    }
+    html = f'<script id="ProductDetailsVM" type="application/json">{json.dumps(payload)}</script>'
+
+    parsed = parse_product_details_vm(html)
+    record = FastCarMapper().map_payload_to_record(
+        detail_payload=parsed,
+        vehicle_url="https://www.iaai.com/VehicleDetail/45078011~US",
+    )
+
+    assert record.origin_id == "iaai:45078011~US"
+    assert record.brand == "ACURA"
+    assert record.model == "RSX BASE"
+    assert record.year == 2002
+    assert record.price == 600
+    assert record.drive == "FWD"
+    assert record.gearbox == "AT"
+    assert record.body_type == "COUPE"
+    assert record.engine_volume == 2000
+    assert record.is_damaged is False
+    assert len(record.images) == 1
+
+
+def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None:
+    keys = [
+        {"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
+        {"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
+        {"k": "abc~2", "w": 1800, "h": 1200, "i": 1},
+    ]
+    images = build_resizer_images_from_keys(keys)
+
+    assert len(images) == 2
+    assert images[0]["order_index"] == 1
+    assert "imageKeys=abc~2" in str(images[0]["fullres_image"])
+
+
+def test_is_challenge_response_marker_rules() -> None:
+    assert is_challenge_response(status_code=403, body_text="", expected_marker=None) is True
+    assert is_challenge_response(status_code=200, body_text="<html>blocked</html>", expected_marker=LISTING_MARKER) is True
+    assert is_challenge_response(
+        status_code=200,
+        body_text=f'<html>{DETAIL_MARKER}<script src="/_Incapsula_Resource"></script></html>',
+        expected_marker=DETAIL_MARKER,
+    ) is False

tests/test_fast_sync.py

@@ -0,0 +1,141 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from iaai_scraper.browser.fast_client import FastListingVehicle
+from iaai_scraper.core.config import Settings
+from iaai_scraper.core.runtime_config import RuntimeConfig, RuntimeFiltersConfig
+from iaai_scraper.fast_sync import FastSyncEngine
+from iaai_scraper.parsing.fast_mapper import FastCarMapper
+
+
+def _listing_vehicle(inventory_id: str, *, status: str = "RS") -> FastListingVehicle:
+    return FastListingVehicle(
+        inventory_id=inventory_id,
+        tenant="US",
+        auction_id="1_1",
+        auction_date="2026-04-08T08:30:00+00:00",
+        inventory_status=status,
+        currency="USD",
+        timed_auction_closed=False,
+        timed_auction_indicator=False,
+        prebid_indicator=True,
+        buynow_indicator=False,
+    )
+
+
+def _detail_payload(inventory_id: str, *, make: str = "ACURA", model: str = "RSX", bid: int = 500) -> dict:
+    return {
+        "inventoryView": {
+            "attributes": {
+                "Id": inventory_id,
+                "Year": "2002",
+                "Make": make,
+                "Model": model,
+                "Series": "BASE",
+                "Currency": "USD",
+                "ODOValue": "219187",
+                "ExteriorColor": "GRAY",
+                "DriveLineTypeDesc": "FWD",
+                "Transmission": "Automatic",
+                "BodyStyleName": "COUPE",
+                "EngineSize": "2.0L I-4",
+                "VehicleGrade": "50",
+                "PrimaryDamageDesc": "NORMAL WEAR & TEAR",
+            },
+            "imageDimensions": {
+                "keys": {"$values": [{"k": f"{inventory_id}~I1", "w": 1600, "h": 1200, "i": 0}]}
+            },
+        },
+        "auctionInformation": {
+            "prebidInformation": {"decimalHighBidAmount": str(bid), "highBidAmount": f"${bid}"},
+            "biddingInformation": {"buyNowPrice": "$0"},
+        },
+    }
+
+
+class FakeFastClient:
+    def __init__(self, listing: list[FastListingVehicle], details: dict[str, dict]) -> None:
+        self.listing = listing
+        self.details = details
+        self.detail_calls = 0
+        self.persist_calls = 0
+
+    def iter_listing_vehicles(self, *, listing_start_url=None, make=None, max_pages=None):  # noqa: ANN001, ANN202
+        del listing_start_url, make, max_pages
+        return iter(self.listing)
+
+    def fetch_vehicle_detail_payload(self, inventory_id: str) -> dict:
+        self.detail_calls += 1
+        return self.details[inventory_id]
+
+    def persist_session_state(self) -> None:
+        self.persist_calls += 1
+
+
+@dataclass
+class FakePersistence:
+    inserted: int = 0
+    images: int = 0
+
+    def get_existing_urls_and_ids(self, origin_urls, origin_ids):  # noqa: ANN001, ANN202
+        del origin_urls, origin_ids
+        return set(), set()
+
+    def upsert_cars_batch(self, records):  # noqa: ANN001, ANN202
+        self.inserted += len(records)
+        self.images += sum(len(record.images) for record in records)
+        return {"inserted": len(records), "updated": 0, "images_upserted": sum(len(record.images) for record in records)}
+
+    def mark_sold_not_in_listing_by_urls(self, active_origin_urls, lane="iaai"):  # noqa: ANN001, ANN202
+        del active_origin_urls, lane
+        return 0
+
+
+def test_fast_sync_engine_collects_details_and_bulk_upserts() -> None:
+    listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
+    details = {
+        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
+        "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
+    }
+    client = FakeFastClient(listing, details)
+    persistence = FakePersistence()
+    engine = FastSyncEngine(
+        client=client,  # type: ignore[arg-type]
+        mapper=FastCarMapper(),
+        persistence=persistence,  # type: ignore[arg-type]
+        batch_size=10,
+        fetch_concurrency=2,
+    )
+
+    result = engine.sync_listing(runtime_config=RuntimeConfig(), only_new=False)
+
+    assert result["status"] == "success"
+    assert result["cars_upserted"] == 2
+    assert result["images_upserted"] == 2
+    assert result["listing"]["mode"] == "fast_hidden_payload"
+    assert client.detail_calls == 2
+    assert client.persist_calls == 1
+
+
+def test_fast_sync_engine_applies_runtime_filters_before_db() -> None:
+    listing = [_listing_vehicle("45078011~US"), _listing_vehicle("45268167~US")]
+    details = {
+        "45078011~US": _detail_payload("45078011~US", make="ACURA", model="RSX", bid=500),
+        "45268167~US": _detail_payload("45268167~US", make="BMW", model="X5", bid=900),
+    }
+    runtime = RuntimeConfig(filters=RuntimeFiltersConfig.from_dict({"brands": ["BMW"]}))
+    persistence = FakePersistence()
+    engine = FastSyncEngine(
+        client=FakeFastClient(listing, details),  # type: ignore[arg-type]
+        mapper=FastCarMapper(),
+        persistence=persistence,  # type: ignore[arg-type]
+        batch_size=10,
+        fetch_concurrency=2,
+    )
+
+    result = engine.sync_listing(runtime_config=runtime, only_new=False)
+
+    assert result["cars_upserted"] == 1
+    assert result["cars_filtered"] == 1
+    assert persistence.inserted == 1

tests/test_resilience.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+import unittest
+from types import SimpleNamespace
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.scraper import IAAIScraper
+from iaai_scraper.core.config import Settings
+from iaai_scraper.worker import tasks
+
+
+class TestResilience(unittest.TestCase):
+	def _make_scraper(self) -> IAAIScraper:
+		s = Settings()
+		s.log_level = "CRITICAL"
+		s.database.url = "sqlite://"
+		return IAAIScraper(s)
+
+	def test_update_task_progress_updates_global_marker(self) -> None:
+		redis_client = MagicMock()
+		pipe = MagicMock()
+		redis_client.pipeline.return_value = pipe
+
+		tasks._update_task_progress(
+			redis_client,
+			task_id="task-abc",
+			stage="batch_upserted",
+			ttl_seconds=180,
+			cars_upserted=10,
+		)
+
+		redis_client.pipeline.assert_called_once()
+		self.assertEqual(pipe.set.call_count, 2)
+		first_call = pipe.set.call_args_list[0]
+		second_call = pipe.set.call_args_list[1]
+
+		self.assertEqual(first_call.args[0], tasks._task_progress_key("task-abc"))
+		self.assertEqual(second_call.args[0], tasks.GLOBAL_PROGRESS_TS_KEY)
+		pipe.execute.assert_called_once()
+
+	def test_streaming_sync_stops_cleanly_when_page_stays_empty(self) -> None:
+		scraper = self._make_scraper()
+		scraper.settings.celery.batch_size = 50
+
+		page = MagicMock()
+		empty_page_result = SimpleNamespace(
+			page_number=1,
+			vehicle_links=[],
+			next_page_detected=True,
+		)
+
+		scraper._open_listing_for_stream = MagicMock(return_value=(
+			page,
+			{"make": None, "model": None, "year_min": None, "year_max": None},
+		))
+		scraper.listing_collector.collect_current_page = MagicMock(return_value=empty_page_result)
+		scraper._recover_empty_listing_page = MagicMock(return_value=(empty_page_result, []))
+		scraper.sync_batch = MagicMock()
+
+		result = scraper._sync_listing_streaming(
+			make=None,
+			model=None,
+			lane="iaai_cars",
+			limit=None,
+			effective_only_new=False,
+			started_at=0.0,
+			listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
+		)
+
+		self.assertEqual(result["total"], 0)
+		self.assertEqual(result["cars_upserted"], 0)
+		self.assertEqual(result["cars_failed"], 0)
+		self.assertEqual(result["listing"]["pages_collected"], 1)
+		scraper._recover_empty_listing_page.assert_called_once()
+		scraper.sync_batch.assert_not_called()
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_scraper.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
 
 import unittest
+from concurrent.futures import TimeoutError as FuturesTimeoutError
 from types import SimpleNamespace
-from unittest.mock import MagicMock
+from unittest.mock import MagicMock, patch
 
 from iaai_scraper.core.config import Settings
 from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
@@ -268,6 +269,51 @@ class TestScraperSync(unittest.TestCase):
         self.assertEqual(result["cars_upserted"], 1)
         self.assertEqual(result["total"], 1)
 
+    def test_sync_batch_timeout_does_not_duplicate_already_processed_urls(self) -> None:
+        scraper = self._make_scraper()
+        scraper.persistence.upsert_cars_batch = MagicMock(return_value={
+            "inserted": 1,
+            "updated": 0,
+            "images_upserted": 0,
+        })
+
+        urls = [
+            "https://www.iaai.com/VehicleDetail/111~US",
+            "https://www.iaai.com/VehicleDetail/222~US",
+            "https://www.iaai.com/VehicleDetail/333~US",
+        ]
+        first_record = CarRecord.model_validate(make_db_record("111"))
+
+        class _FakeExecutor:
+            def __init__(self, *args, **kwargs):
+                pass
+
+            def __enter__(self):
+                return self
+
+            def __exit__(self, exc_type, exc, tb):
+                return False
+
+            def map(self, fn, iterable, timeout=None):  # noqa: ARG002
+                yield 0, first_record
+                raise FuturesTimeoutError()
+
+        with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
+             patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
+             patch.object(scraper, "_browser_fallback_parallel", return_value={
+                 "records": [],
+                 "failures": [],
+                 "cars_failed": 0,
+                 "protection_events": 0,
+             }) as fallback_mock:
+            result = scraper.sync_batch(urls)
+
+        self.assertEqual(result["cars_upserted"], 1)
+        fallback_urls = fallback_mock.call_args.args[0]
+        self.assertEqual(len(fallback_urls), 2)
+        self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
+        self.assertNotIn(urls[0], {u for u, _ in fallback_urls})
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_self_heal.py

@@ -0,0 +1,195 @@
+from __future__ import annotations
+
+import json
+import unittest
+from unittest.mock import MagicMock, patch
+
+from iaai_scraper.worker import self_heal
+
+
+class TestSelfHeal(unittest.TestCase):
+	def test_read_last_progress_ts_uses_global_key(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.return_value = "1776800000"
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertEqual(ts, 1776800000)
+		redis_client.scan_iter.assert_not_called()
+
+	def test_read_last_progress_ts_fallbacks_to_task_progress_keys(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.side_effect = lambda key: {
+			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
+			"iaai:state:task_progress:a": '{"ts": 100}',
+			"iaai:state:task_progress:b": '{"ts": 250}',
+			"iaai:state:task_progress:c": '{"ts": 150}',
+		}.get(key)
+		redis_client.scan_iter.return_value = [
+			"iaai:state:task_progress:a",
+			"iaai:state:task_progress:b",
+			"iaai:state:task_progress:c",
+		]
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertEqual(ts, 250)
+
+	def test_read_last_progress_ts_ignores_broken_payloads(self) -> None:
+		redis_client = MagicMock()
+		redis_client.get.side_effect = lambda key: {
+			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
+			"iaai:state:task_progress:a": "{bad-json}",
+			"iaai:state:task_progress:b": '{"foo": "bar"}',
+		}.get(key)
+		redis_client.scan_iter.return_value = [
+			"iaai:state:task_progress:a",
+			"iaai:state:task_progress:b",
+		]
+
+		ts = self_heal._read_last_progress_ts(redis_client)
+
+		self.assertIsNone(ts)
+
+	def test_active_recent_progress_prevents_db_idle_restart(self) -> None:
+		redis_client = MagicMock()
+		redis_client.scan_iter.return_value = ["iaai:state:task_progress:hourly"]
+		redis_client.get.return_value = json.dumps(
+			{
+				"task_id": "hourly",
+				"stage": "fast_listing_collected",
+				"ts": 1000,
+				"last_db_progress_ts": 1,
+				"skipped_existing": 2417,
+			}
+		)
+
+		active = self_heal._has_active_recent_progress(
+			redis_client,
+			now_ts=1010,
+			stall_seconds=900,
+		)
+
+		self.assertTrue(active)
+
+	@patch("iaai_scraper.worker.self_heal.time.time", return_value=5000)
+	def test_has_inflight_work_deletes_stale_terminal_progress_without_work(self, _time_mock) -> None:
+		redis_client = MagicMock()
+		redis_client.llen.return_value = 0
+		redis_client.get.side_effect = lambda key: {
+			self_heal.SYNC_LISTING_LOCK_KEY: None,
+			"iaai:state:task_progress:old": json.dumps(
+				{
+					"task_id": "old",
+					"stage": "segment_done",
+					"ts": 1000,
+					"segment_full_scan_completed": True,
+				}
+			),
+		}.get(key)
+		redis_client.scan_iter.return_value = ["iaai:state:task_progress:old"]
+
+		has_inflight, flags = self_heal._has_inflight_work(redis_client, self_heal.IAAI_SYNC_QUEUE)
+
+		self.assertFalse(has_inflight)
+		self.assertEqual(flags["has_task_progress"], 0)
+		redis_client.delete.assert_called_once_with("iaai:state:task_progress:old")
+
+	@patch("iaai_scraper.worker.self_heal.time.time", return_value=1010)
+	def test_has_inflight_work_keeps_recent_non_terminal_progress(self, _time_mock) -> None:
+		redis_client = MagicMock()
+		redis_client.llen.return_value = 0
+		redis_client.get.side_effect = lambda key: {
+			self_heal.SYNC_LISTING_LOCK_KEY: None,
+			"iaai:state:task_progress:active": json.dumps(
+				{
+					"task_id": "active",
+					"stage": "fast_listing_collected",
+					"ts": 1000,
+				}
+			),
+		}.get(key)
+		redis_client.scan_iter.return_value = ["iaai:state:task_progress:active"]
+
+		has_inflight, flags = self_heal._has_inflight_work(redis_client, self_heal.IAAI_SYNC_QUEUE)
+
+		self.assertTrue(has_inflight)
+		self.assertEqual(flags["has_task_progress"], 1)
+		redis_client.delete.assert_not_called()
+
+	@patch("iaai_scraper.worker.self_heal.time.time", return_value=5000)
+	def test_has_inflight_work_deletes_stale_non_terminal_progress_without_work(self, _time_mock) -> None:
+		redis_client = MagicMock()
+		redis_client.llen.return_value = 0
+		redis_client.get.side_effect = lambda key: {
+			self_heal.SYNC_LISTING_LOCK_KEY: None,
+			"iaai:state:task_progress:stale": json.dumps(
+				{
+					"task_id": "stale",
+					"stage": "segment_started",
+					"ts": 1000,
+					"segment_index": 11,
+				}
+			),
+		}.get(key)
+		redis_client.scan_iter.return_value = ["iaai:state:task_progress:stale"]
+
+		has_inflight, flags = self_heal._has_inflight_work(redis_client, self_heal.IAAI_SYNC_QUEUE)
+
+		self.assertFalse(has_inflight)
+		self.assertEqual(flags["has_task_progress"], 0)
+		self.assertEqual(flags["stale_progress_deleted"], 1)
+		redis_client.delete.assert_called_once_with("iaai:state:task_progress:stale")
+
+	@patch("iaai_scraper.worker.self_heal.time.time", return_value=5000)
+	def test_has_inflight_work_keeps_stale_progress_when_lock_exists(self, _time_mock) -> None:
+		redis_client = MagicMock()
+		redis_client.llen.return_value = 0
+		redis_client.get.side_effect = lambda key: {
+			self_heal.SYNC_LISTING_LOCK_KEY: "owner",
+			"iaai:state:task_progress:stale": json.dumps(
+				{
+					"task_id": "stale",
+					"stage": "segment_started",
+					"ts": 1000,
+				}
+			),
+		}.get(key)
+		redis_client.scan_iter.return_value = ["iaai:state:task_progress:stale"]
+
+		has_inflight, flags = self_heal._has_inflight_work(redis_client, self_heal.IAAI_SYNC_QUEUE)
+
+		self.assertTrue(has_inflight)
+		self.assertEqual(flags["has_lock"], 1)
+		self.assertEqual(flags["has_task_progress"], 1)
+		redis_client.delete.assert_not_called()
+
+	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("iaai_scraper.worker.self_heal.os.kill")
+	@patch("builtins.open")
+	def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
+		open_mock.return_value.__enter__.return_value.read.return_value = "123"
+		# SIGTERM -> process alive check (pid,0) -> SIGKILL
+		kill_mock.side_effect = [None, None, None]
+
+		self_heal._kill_worker_process()
+
+		self.assertEqual(kill_mock.call_args_list[0].args[0], 123)
+		self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
+		self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
+
+	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("iaai_scraper.worker.self_heal.os.kill")
+	@patch("builtins.open")
+	def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
+		open_mock.return_value.__enter__.return_value.read.return_value = "123"
+		kill_mock.side_effect = [None, ProcessLookupError()]
+
+		self_heal._kill_worker_process()
+
+		# Только SIGTERM и проверка существования процесса.
+		self.assertEqual(len(kill_mock.call_args_list), 2)
+
+
+if __name__ == "__main__":
+	unittest.main()

tests/test_utils.py

@@ -3,7 +3,14 @@ from __future__ import annotations
 import unittest
 
 from iaai_scraper.core.utils import deep_find_key
-from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
+from iaai_scraper.core.config import (
+    IAAI_DEFAULT_MAKES,
+    _LARGE_MAKES,
+    _YEAR_SPLITS,
+    ProxyConfig,
+    build_listing_segments_for_makes,
+    parse_listing_segments,
+)
 
 
 class TestDeepFindKey(unittest.TestCase):
@@ -70,6 +77,37 @@ class TestParseListingSegments(unittest.TestCase):
         self.assertEqual(segs[0]["year_min"], 2020)
         self.assertEqual(segs[0]["year_max"], 2025)
 
+    def test_build_listing_segments_for_makes(self) -> None:
+        segs = build_listing_segments_for_makes(["toyota", "Lexus", "TOYOTA", " "])
+
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+
+        self.assertEqual(len(toyota), len(_YEAR_SPLITS))
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+
+
+class TestProxyConfig(unittest.TestCase):
+    def test_requests_proxy_url_includes_encoded_credentials(self) -> None:
+        cfg = ProxyConfig(
+            server="http://144.31.139.6:3128",
+            username="carsproxy",
+            password="pass@word:with/slash",
+        )
+
+        self.assertEqual(
+            cfg.to_requests_proxy_url(),
+            "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128",
+        )
+        self.assertEqual(
+            cfg.to_requests_proxies(),
+            {
+                "http": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128",
+                "https": "http://carsproxy:pass%40word%3Awith%2Fslash@144.31.139.6:3128",
+            },
+        )
+
 
 if __name__ == "__main__":
     unittest.main()

tests/test_worker_tasks.py

@@ -1,12 +1,36 @@
 from __future__ import annotations
 
+import json
+import os
+from dataclasses import replace
+import tempfile
 import unittest
 from unittest.mock import MagicMock, patch
 
 from iaai_scraper.worker import tasks
+from iaai_scraper.core.config import Settings, settings as base_settings
 
 
 class TestWorkerTaskLockHelpers(unittest.TestCase):
+    def test_build_listing_segments_from_runtime_config_brands(self) -> None:
+        with tempfile.NamedTemporaryFile("w", encoding="utf-8", suffix=".json", delete=False) as fh:
+            json.dump({"filters": {"brands": ["Toyota", "Lexus", "Toyota"]}}, fh)
+            runtime_config_file = fh.name
+
+        settings = Settings(runtime_config_file=runtime_config_file)
+        settings.listing.listing_segments_json = "runtime"
+        settings.scraping_profile.http_first = False
+        settings.listing.fast_segment_year_splits = True
+
+        segs = tasks._build_listing_segments(settings)
+
+        toyota = [s for s in segs if s["make"] == "TOYOTA"]
+        lexus = [s for s in segs if s["make"] == "LEXUS"]
+        self.assertEqual(len(toyota), 3)
+        self.assertEqual(len(lexus), 1)
+        self.assertIsNone(lexus[0]["year_min"])
+        os.unlink(runtime_config_file)
+
     def test_lock_acquire_refresh_release(self) -> None:
         redis_client = MagicMock()
 
@@ -67,7 +91,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             get_redis.return_value = redis_client
             stop_event = MagicMock()
             heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, MagicMock(), heartbeat_thread)
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
 
             tasks.sync_listing_task.push_request(id="task-123")
             try:
@@ -132,7 +156,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             get_redis.return_value = redis_client
             stop_event = MagicMock()
             heartbeat_thread = MagicMock()
-            start_heartbeat.return_value = (stop_event, MagicMock(), heartbeat_thread)
+            start_heartbeat.return_value = (stop_event, heartbeat_thread)
 
             tasks.sync_listing_task.push_request(id="task-456")
             try:
@@ -197,7 +221,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
                 "1" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
             )
             get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock(), MagicMock())
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
 
             sync_segmented_mock = MagicMock(return_value={
                 "run_id": 11, "status": "success", "full_scan_completed": True,
@@ -240,7 +264,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
                 "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
             )
             get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock(), MagicMock())
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
 
             sync_segmented_mock = MagicMock(return_value={
                 "run_id": 14, "status": "success", "full_scan_completed": True,
@@ -281,7 +305,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
                 "99" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
             )
             get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock(), MagicMock())
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
 
             sync_segmented_mock = MagicMock(return_value={
                 "run_id": 15, "status": "success", "full_scan_completed": True,
@@ -327,7 +351,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             redis_client = MagicMock()
             redis_client.get.return_value = None
             get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock(), MagicMock())
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
 
             tasks.sync_listing_task.push_request(id="task-791")
             try:
@@ -403,7 +427,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             redis_client = MagicMock()
             redis_client.get.return_value = None
             get_redis.return_value = redis_client
-            start_heartbeat.return_value = (MagicMock(), MagicMock(), MagicMock())
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
 
             with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
                 tasks.sync_listing_task.push_request(id="task-900")
@@ -417,6 +441,177 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         clear_pending.assert_called()
         task_app.send_task.assert_not_called()
 
+    def test_sync_listing_task_soft_timeout_deduplicates_continuation(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_run_browser_job", side_effect=tasks.SoftTimeLimitExceeded()), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=False) as set_pending, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-soft-timeout")
+                try:
+                    result = tasks.sync_listing_task.run(make="Toyota")
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "timed_out")
+        set_pending.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_stops_immediate_bootstrap_continuation_after_limit(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat", return_value=(MagicMock(), MagicMock())), \
+             patch.object(tasks, "_release_lock_if_owner"), \
+             patch.object(tasks, "_try_set_followup_pending", return_value=True), \
+             patch.object(tasks, "_bump_bootstrap_continuation_streak", return_value=(999, False)), \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job", return_value={
+                 "run_id": 101,
+                 "status": "partial_success",
+                 "full_scan_completed": False,
+                 "cars_upserted": 2,
+                 "cars_failed": 0,
+                 "images_upserted": 1,
+                 "skipped_existing": 0,
+                 "elapsed_seconds": 1.0,
+                 "failures": [],
+                 "listing": {"vehicles_collected": 2},
+             }):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app:
+                tasks.sync_listing_task.push_request(id="task-breaker-stop")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "partial_success")
+        # Сначала bootstrap помечается незавершённым, затем breaker переключает на hourly.
+        self.assertTrue(any(call.args == (redis_client, False) for call in set_full_scan_done.call_args_list))
+        self.assertTrue(any(call.args == (redis_client, True) for call in set_full_scan_done.call_args_list))
+        clear_checkpoint.assert_called_once()
+        task_app.send_task.assert_not_called()
+
+    def test_sync_listing_task_always_full_scan_forces_bootstrap_even_after_done(self) -> None:
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch.object(tasks, "_run_browser_job") as run_job, \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            run_job.return_value = {
+                "run_id": 17,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            }
+
+            tasks.sync_listing_task.push_request(id="task-always-full")
+            try:
+                result = tasks.sync_listing_task.run()
+            finally:
+                tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        set_full_scan_done.assert_called_with(redis_client, False)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_always_full_scan_uses_segmented_resume_path(self) -> None:
+        segments = [{"make": "TOYOTA"}, {"make": "FORD"}, {"make": "HONDA"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
+             patch.object(tasks.sync_listing_task, "update_state"), \
+             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.get.side_effect = lambda key: (
+                "0" if key == tasks.SYNC_LISTING_CHECKPOINT_KEY else None
+            )
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            sync_segmented_mock = MagicMock(return_value={
+                "run_id": 18,
+                "status": "success",
+                "full_scan_completed": True,
+                "cars_upserted": 1,
+                "cars_failed": 0,
+                "images_upserted": 0,
+                "skipped_existing": 0,
+                "elapsed_seconds": 1.0,
+                "failures": [],
+            })
+            scraper_ctx = MagicMock()
+            scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
+            scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
+            scraper_ctx.__exit__.return_value = None
+
+            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+                tasks.sync_listing_task.push_request(id="task-always-full-resume")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_segment"], 1)
+        self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
+        release_lock.assert_called_once()
+
 
 if __name__ == "__main__":
     unittest.main()

vps_check.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd /root/iaai-parser
+
+echo '--- services ---'
+docker compose ps
+
+echo '--- env speed ---'
+grep -nE 'IAAI_PROFILE|IAAI_SCRAPING_PROFILE|IAAI_HTTP_FIRST|IAAI_LISTING_SEGMENTS|IAAI_FAST_SEGMENT_YEAR_SPLITS|CELERY_WORKER_CONCURRENCY|IAAI_FETCH_CONCURRENCY|CELERY_BATCH_SIZE' .env
+
+echo '--- db count ---'
+docker compose exec -T postgres psql -U iaai -d iaai_scraper -t -c "select count(*) as cars_count, max(last_seen_at) as last_seen from iaai_cars;"
+
+echo '--- redis progress ---'
+docker compose exec -T redis redis-cli MGET iaai:state:sync_listing_checkpoint iaai:state:sync_segments_done iaai:state:sync_segments_total iaai:state:sync_full_scan_done
+
+echo '--- recent worker progress ---'
+docker compose logs --tail=80 worker | grep -E 'Segment [0-9]+/[0-9]+|Fast HTTP-first listing started|Fast HTTP-first listing collected|Fast HTTP-first DB batch|ERROR|WARNING' | tail -40