update scraper package

.env.example

@@ -1,51 +1,51 @@
-IAAI_HEADLESS=true
+DUBIZZLE_HEADLESS=true
-IAAI_LOG_LEVEL=INFO
+DUBIZZLE_LOG_LEVEL=INFO
 
-IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
+DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY=true
-IAAI_MAX_CAPTURED_REQUESTS=40
+DUBIZZLE_MAX_CAPTURED_REQUESTS=40
-IAAI_MAX_CAPTURED_JSON_RESPONSES=20
+DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES=20
 
-IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
+DUBIZZLE_CARS_LISTING_URL=https://uae.dubizzle.com/motors/used-cars/
-IAAI_LISTING_SEGMENTS=auto
+DUBIZZLE_LISTING_SEGMENTS=auto
-IAAI_MAX_PAGES_PER_RUN=999999
+DUBIZZLE_MAX_PAGES_PER_RUN=999999
-IAAI_MAX_VEHICLES_PER_RUN=999999
+DUBIZZLE_MAX_VEHICLES_PER_RUN=999999
-IAAI_PAGE_LINK_LIMIT=999999
+DUBIZZLE_PAGE_LINK_LIMIT=999999
-IAAI_INCLUDE_PAGINATION=true
+DUBIZZLE_INCLUDE_PAGINATION=true
-IAAI_COLLECT_CURRENT_PAGE_ONLY=false
+DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY=false
 
-IAAI_HUMAN_PACE_ENABLED=true
+DUBIZZLE_HUMAN_PACE_ENABLED=true
-IAAI_PARALLEL_TABS=10
+DUBIZZLE_PARALLEL_TABS=10
 CELERY_BATCH_SIZE=100
-IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
+DUBIZZLE_AFTER_LISTING_OPEN_MIN_S=0.2
-IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
+DUBIZZLE_AFTER_LISTING_OPEN_MAX_S=0.5
-IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
+DUBIZZLE_AFTER_FILTER_ACTION_MIN_S=0.2
-IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
+DUBIZZLE_AFTER_FILTER_ACTION_MAX_S=0.5
-IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
+DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S=0.02
-IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
+DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S=0.08
-IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
+DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S=0.02
-IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
+DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S=0.08
-IAAI_BETWEEN_VEHICLES_MIN_S=0.02
+DUBIZZLE_BETWEEN_VEHICLES_MIN_S=0.02
-IAAI_BETWEEN_VEHICLES_MAX_S=0.08
+DUBIZZLE_BETWEEN_VEHICLES_MAX_S=0.08
-IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
+DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S=0.2
-IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
+DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S=0.5
 
-IAAI_SYNC_ONLY_NEW=false
+DUBIZZLE_SYNC_ONLY_NEW=false
-IAAI_TOKENS_FILE=/data/tokens.json
+DUBIZZLE_TOKENS_FILE=/data/tokens.json
-IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
+DUBIZZLE_RUNTIME_CONFIG_FILE=/app/runtime_config.json
 
-IAAI_MAX_RETRIES=5
+DUBIZZLE_MAX_RETRIES=5
-IAAI_RETRY_DELAY_SECONDS=4
+DUBIZZLE_RETRY_DELAY_SECONDS=4
-IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
+DUBIZZLE_RETRY_BACKOFF_MULTIPLIER=2.0
-IAAI_RETRY_JITTER_SECONDS=0.5
+DUBIZZLE_RETRY_JITTER_SECONDS=0.5
-IAAI_TIMEOUT_MS=90000
+DUBIZZLE_TIMEOUT_MS=90000
-IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
+DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS=30000
 
-IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+DUBIZZLE_DATABASE_URL=postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
-IAAI_DATABASE_ECHO=false
+DUBIZZLE_DATABASE_ECHO=false
-IAAI_DATABASE_POOL_SIZE=5
+DUBIZZLE_DATABASE_POOL_SIZE=5
-IAAI_DATABASE_MAX_OVERFLOW=5
+DUBIZZLE_DATABASE_MAX_OVERFLOW=5
 
-IAAI_REDIS_URL=redis://redis:6379/0
+DUBIZZLE_REDIS_URL=redis://redis:6379/0
 
 CELERY_BROKER_URL=redis://redis:6379/0
 CELERY_RESULT_BACKEND=redis://redis:6379/0
@@ -55,6 +55,13 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=1
 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
 CELERY_BEAT_SYNC_LIMIT=0
-IAAI_ALWAYS_FULL_SCAN=true
+DUBIZZLE_ALWAYS_FULL_SCAN=true
+CELERY_PARALLEL_SEGMENTS=true
+DUBIZZLE_DISCOVERY_MODE=algolia
+DUBIZZLE_ALGOLIA_APPLICATION_ID=WD0PTZ13ZS
+DUBIZZLE_ALGOLIA_API_KEY=cef139620248f1bc328a00fddc7107a6
+DUBIZZLE_ALGOLIA_INDEX_NAME=motors.com
+DUBIZZLE_ALGOLIA_BASE_URL=https://WD0PTZ13ZS-dsn.algolia.net
+DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY=false
 SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
 SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600

Dockerfile

@@ -1,8 +1,43 @@
-FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
+FROM python:3.12-slim-bookworm
 
 ENV PYTHONDONTWRITEBYTECODE=1 \
     PYTHONUNBUFFERED=1
 
+ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        curl \
+        ca-certificates \
+        bash \
+        build-essential \
+        libasound2 \
+        libatk-bridge2.0-0 \
+        libatk1.0-0 \
+        libc6 \
+        libcairo2 \
+        libcups2 \
+        libdbus-1-3 \
+        libdrm2 \
+        libgbm1 \
+        libglib2.0-0 \
+        libgtk-3-0 \
+        libnspr4 \
+        libnss3 \
+        libpango-1.0-0 \
+        libx11-6 \
+        libx11-xcb1 \
+        libxcb1 \
+        libxcomposite1 \
+        libxdamage1 \
+        libxext6 \
+        libxfixes3 \
+        libxkbcommon0 \
+        libxrandr2 \
+        wget \
+        xdg-utils \
+    && rm -rf /var/lib/apt/lists/*
+
 RUN groupadd --system app && useradd --system --gid app --create-home app
 
 WORKDIR /app
@@ -19,7 +54,8 @@ RUN python -m playwright install chromium firefox
 
 COPY . .
 RUN pip install --no-cache-dir -e .
-RUN python -m compileall -q iaai_scraper
+RUN python -m compileall -q dubizzle_scraper
+RUN sed -i 's/\r$//' /app/entrypoint.sh
 RUN chmod +x entrypoint.sh
 RUN chown -R app:app /app
 
@@ -29,4 +65,4 @@ USER app
 
 # По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
-CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
+CMD ["uvicorn", "dubizzle_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

alembic.ini

@@ -3,7 +3,7 @@
 [alembic]
 script_location = alembic
 prepend_sys_path = .
-sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
+sqlalchemy.url = postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper
 
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool
 # Добавляем корень проекта в sys.path
 sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
 
-from iaai_scraper.core.config import Settings
+from dubizzle_scraper.core.config import Settings
-from iaai_scraper.storage.models import Base
+from dubizzle_scraper.storage.models import Base
 
 config = context.config
 

alembic/versions/001_initial.py

@@ -11,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Таблица cars — аукционные машины с IAAI
+    # Таблица cars — аукционные машины с DUBIZZLE
     op.create_table(
         "cars",
         sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),

alembic/versions/003_add_composite_indexes.py

@@ -10,12 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None
 
 
 def upgrade() -> None:
-    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Partial index для активных машин DUBIZZLE (is_sold = FALSE)
     # Ускоряет поиск при mark_sold операциях
     op.execute(
-        "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
+        "CREATE INDEX IF NOT EXISTS ix_cars_active_dubizzle "
         "ON cars (origin_url) "
-        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
+        "WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%'"
     )
 
     # Composite index для проверки дубликатов изображений
@@ -35,4 +35,4 @@ def upgrade() -> None:
 def downgrade() -> None:
     op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
     op.drop_index("ix_images_car_id_fullres", table_name="images")
-    op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")
+    op.execute("DROP INDEX IF EXISTS ix_cars_active_dubizzle")

docker-compose.yml

@@ -1,35 +1,40 @@
 x-app-env: &app-env
   # Всегда используем Postgres.
-  IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
+  DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
-  IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
+  DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0}
   CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
   CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
-  IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
+  DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
   # 0 = без лимита.
   CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
+  CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true}
   CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
   CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
-  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
+  DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3}
-  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
+  DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9}
-  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
+  DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8}
-  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
+  DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true}
-  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
+  DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia}
-  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
+  DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto}
-  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
+  DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/}
-  IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
+  DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999}
-  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
+  DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000}
-  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
+  DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true}
-  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  # Работаем через Algolia; fallback в sitemap по умолчанию отключён.
-  IAAI_BROWSER_ENGINE: chromium
+  DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false}
+  DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true}
+  DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json}
+  DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
+  DUBIZZLE_BROWSER_ENGINE: chromium
   # Self-heal для worker.
-  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
+  DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true}
-  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
+  DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
-  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
+  DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900}
-  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
+  DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
-  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
+  DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file
@@ -42,29 +47,32 @@ x-app-service: &app-service
   environment: *app-env
   volumes:
     - ./runtime_config.json:/app/runtime_config.json:ro
+    - ./artifacts:/app/artifacts
 
 x-worker-service: &worker-service
   <<: *app-service
   volumes:
     - ./runtime_config.json:/app/runtime_config.json:ro
+    - ./artifacts:/app/artifacts
     - tokens_data:/data
 
 services:
   # PostgreSQL.
   postgres:
     image: postgres:16-alpine
-    container_name: iaai-postgres
+    container_name: dubizzle-postgres
     restart: unless-stopped
     environment:
-      POSTGRES_USER: iaai
+      POSTGRES_USER: dubizzle
-      POSTGRES_PASSWORD: iaai
+      POSTGRES_PASSWORD: dubizzle
-      POSTGRES_DB: iaai_scraper
+      POSTGRES_DB: dubizzle_scraper
     ports:
-      - "127.0.0.1:5432:5432"
+      # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
+      - "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432"
     volumes:
       - pgdata:/var/lib/postgresql/data
     healthcheck:
-      test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
+      test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"]
       interval: 5s
       timeout: 3s
       retries: 5
@@ -77,10 +85,11 @@ services:
   # Redis.
   redis:
     image: redis:7-alpine
-    container_name: iaai-redis
+    container_name: dubizzle-redis
     restart: unless-stopped
     ports:
-      - "127.0.0.1:6379:6379"
+      # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
+      - "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379"
     healthcheck:
       test: ["CMD", "redis-cli", "ping"]
       interval: 5s
@@ -101,7 +110,7 @@ services:
   # Миграции.
   migrate:
     <<: *app-service
-    container_name: iaai-migrate
+    container_name: dubizzle-migrate
     restart: "no"
     depends_on:
       postgres:
@@ -111,17 +120,18 @@ services:
   # API.
   api:
     <<: *app-service
-    container_name: iaai-api
+    container_name: dubizzle-api
     restart: unless-stopped
     ports:
-      - "127.0.0.1:8000:8000"
+      # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
+      - "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000"
     depends_on:
       migrate:
         condition: service_completed_successfully
       redis:
         condition: service_healthy
     command: >
-      uvicorn iaai_scraper.api.app:app
+      uvicorn dubizzle_scraper.api.app:app
       --host 0.0.0.0 --port 8000 --workers 1
     healthcheck:
       test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -148,13 +158,13 @@ services:
         condition: service_healthy
     stop_grace_period: 60s
     command: >
-      celery -A iaai_scraper.worker.celery_app worker
+      celery -A dubizzle_scraper.worker.celery_app worker
       --loglevel=info --concurrency=1 --pool=solo
       --pidfile=/tmp/celery-worker.pid
       -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
     healthcheck:
       # Проверка worker.
-      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
+      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"]
       interval: 60s
       timeout: 10s
       retries: 3
@@ -168,7 +178,7 @@ services:
   # Beat.
   beat:
     <<: *worker-service
-    container_name: iaai-beat
+    container_name: dubizzle-beat
     restart: unless-stopped
     init: true
     depends_on:
@@ -177,7 +187,7 @@ services:
       redis:
         condition: service_healthy
     command: >
-      celery -A iaai_scraper.worker.celery_app beat
+      celery -A dubizzle_scraper.worker.celery_app beat
       --loglevel=info
       --pidfile=/tmp/celery-beat.pid
       --schedule=/tmp/celerybeat-schedule
@@ -193,6 +203,25 @@ services:
         max-size: "10m"
         max-file: "5"
 
+  # Разовый прогон sync-listing через Docker CLI.
+  sync:
+    <<: *worker-service
+    container_name: dubizzle-sync
+    restart: "no"
+    profiles: ["manual"]
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    command: >
+      bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json"
+    logging:
+      driver: json-file
+      options:
+        max-size: "10m"
+        max-file: "5"
+
 volumes:
   pgdata:
   redisdata:

dubizzle_scraper/api/app.py

@@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
     _settings = settings or Settings()
 
     app = FastAPI(
-        title="IAAI Scraper API",
+        title="Dubizzle Scraper API",
-        description="REST API для управления задачами скрапинга IAAI и просмотра данных",
+        description="REST API для управления задачами скрапинга Dubizzle и просмотра данных",
         version="1.0.0",
         lifespan=lifespan,
     )

dubizzle_scraper/api/routes/health.py

@@ -9,7 +9,7 @@ from ..deps import get_persistence
 from ...storage.db import PersistenceService
 
 router = APIRouter()
-logger = logging.getLogger("iaai_scraper.api.health")
+logger = logging.getLogger("dubizzle_scraper.api.health")
 
 
 @router.get("/health")
@@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
 
     return {
         "status": "ok" if db_ok else "degraded",
-        "service": "iaai-scraper-api",
+        "service": "dubizzle-scraper-api",
         "database": "connected" if db_ok else "unavailable",
     }

dubizzle_scraper/api/routes/tasks.py

@@ -15,13 +15,13 @@ router = APIRouter()
 
 class SyncVehicleRequest(BaseModel):
     vehicle_url: str
-    lane: str = "iaai"
+    lane: str = "dubizzle"
 
 
 class SyncListingRequest(BaseModel):
     make: str | None = None
     model: str | None = None
-    lane: str = "iaai_cars"
+    lane: str = "dubizzle_cars"
     limit: int | None = None
     only_new: bool | None = None
 

dubizzle_scraper/browser/factory.py

@@ -11,7 +11,7 @@ except ImportError:
 
 from ..core.config import Settings
 
-logger = logging.getLogger("iaai_scraper.browser")
+logger = logging.getLogger("dubizzle_scraper.browser")
 
 
 def _build_init_script() -> str:
@@ -72,11 +72,11 @@ class BrowserFactory:
         # Выбор движка.
         engine = self.settings.browser_engine.strip().lower()
         if engine == "auto":
-            # Для IAAI стабильнее Chromium.
+            # Для DUBIZZLE стабильнее Chromium.
             return "chromium"
         if engine in ("firefox", "chromium"):
             return engine
-        logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
+        logger.warning("Unknown DUBIZZLE_BROWSER_ENGINE=%r, falling back to auto", engine)
         return "chromium"
 
     def create_browser(self, playwright: Playwright) -> Browser:

dubizzle_scraper/browser/listing.py

@@ -11,9 +11,15 @@ from .pace import HumanPacer
 from ..core.config import Settings
 from ..core.utils import first_non_empty
 
-logger = logging.getLogger("iaai_scraper.listing")
+logger = logging.getLogger("dubizzle_scraper.listing")
-VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_HREF_RE = re.compile(
-VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
+	r'(?:/VehicleDetail/(?P<veh_id>\d+)(?:~[A-Z]{2})?)|(?:/motors/used-cars/[^"\s]+?(?:/ad-(?P<ad_id>\d+)/?|---(?P<slug_id>[a-f0-9]{32})/?))|(?:/s/(?P<short_id>[A-Za-z0-9]+))',
+	re.IGNORECASE,
+)
+VEHICLE_LINK_SELECTOR = (
+	"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail'], "
+	"a[href*='/motors/used-cars/'], a[href*='/s/']"
+)
 COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
 	"button:has-text('Accept All')",
 	"button:has-text('Accept all')",
@@ -108,7 +114,7 @@ class ListingCollector:
 			try:
 				page.wait_for_function(
 					f"""() => {{
-						const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
+						const a = document.querySelector(\"{VEHICLE_LINK_SELECTOR}\");
 						return a && a.getAttribute('href') !== '{old_first_href}';
 					}}""",
 					timeout=12000,
@@ -331,7 +337,10 @@ class ListingCollector:
 			match = VEHICLE_HREF_RE.search(href)
 			if not match:
 				continue
-			lot_number = match.group(1)
+			lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or None
+			if lot_number is None:
+				short = match.group("short_id")
+				lot_number = short if short else None
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
@@ -341,7 +350,7 @@ class ListingCollector:
 			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
 
-		# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
+		# Fallback: на DUBIZZLE ссылки иногда не рендерятся как <a>,
 		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
 		if not links:
 			try:
@@ -381,7 +390,7 @@ class ListingCollector:
 		seen: set[str] = set()
 
 		for match in VEHICLE_HREF_RE.finditer(normalized):
-			lot_number = match.group(1)
+			lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or match.group("short_id") or ""
 			absolute = urljoin(self.settings.home_url, match.group(0))
 			if absolute in seen:
 				continue
@@ -424,7 +433,7 @@ class ListingCollector:
 				self.pacer.after_page_change()
 				return True
 
-		# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
+		# Fallback для DUBIZZLE: пагинация часто рендерится как набор номеров страниц
 		# + стрелка с иконкой, без явного текста Next.
 		try:
 			clicked = bool(page.evaluate(
@@ -573,7 +582,7 @@ class ListingCollector:
 			):
 				page_known = sum(
 					1 for item in page_result.vehicle_links
-					if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
+					if item.lot_number and f"dubizzle:{item.lot_number}" in known_origin_ids
 				)
 				page_new = len(page_result.vehicle_links) - page_known
 				ratio = page_known / len(page_result.vehicle_links)
@@ -651,6 +660,10 @@ class ListingCollector:
 			count = locator.count()
 			if count == 0:
 				continue
+			# Тестовые/fake локаторы могут не поддерживать nth/get_attribute.
+			# В таком случае считаем наличие селектора достаточным признаком next.
+			if not hasattr(locator, "nth"):
+				return True
 			# Проверяем, что хотя бы один элемент не disabled.
 			# Disabled "Next" на последней странице не означает наличия следующей.
 			for i in range(min(count, 3)):

dubizzle_scraper/browser/network.py

@@ -8,7 +8,7 @@ from playwright.sync_api import Page, Request, Response
 
 from ..core.config import Settings
 
-logger = logging.getLogger("iaai_scraper.network")
+logger = logging.getLogger("dubizzle_scraper.network")
 
 
 @dataclass
@@ -44,7 +44,7 @@ class NetworkCapture:
         if not self.settings.capture.capture_same_origin_only or not self._origin:
             return True
         netloc = urlparse(url).netloc.lower()
-        return netloc == self._origin or netloc.endswith(".iaai.com")
+        return netloc == self._origin or netloc.endswith(".dubizzle.com")
 
     def _on_request(self, request: Request) -> None:
         # Берём только xhr/fetch

dubizzle_scraper/cli.py

@@ -1,13 +1,13 @@
 import argparse
 from pathlib import Path
 
-from .core.config import Settings
+from .core.config import Settings, parse_listing_segments
 from .core.utils import save_to_json
-from .scraper import IAAIScraper
+from .scraper import DUBIZZLEScraper
 
 
 def build_parser() -> argparse.ArgumentParser:
-    parser = argparse.ArgumentParser(description="IAAI scraper CLI")
+    parser = argparse.ArgumentParser(description="Dubizzle scraper CLI")
     parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
     parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
     subparsers = parser.add_subparsers(dest="command", required=True)
@@ -19,24 +19,24 @@ def build_parser() -> argparse.ArgumentParser:
     listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
     listing_parser.add_argument("--make", default=None)
     listing_parser.add_argument("--model", default=None)
-    listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
+    listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_listing_links.json"))
 
     scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
     scrape_parser.add_argument("vehicle_url")
-    scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
+    scrape_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_vehicle_detail.json"))
 
     sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
     sync_vehicle_parser.add_argument("vehicle_url")
-    sync_vehicle_parser.add_argument("--lane", default="iaai")
+    sync_vehicle_parser.add_argument("--lane", default="dubizzle")
-    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
+    sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_vehicle.json"))
 
     sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
     sync_listing_parser.add_argument("--make", default=None)
     sync_listing_parser.add_argument("--model", default=None)
-    sync_listing_parser.add_argument("--lane", default="iaai_cars")
+    sync_listing_parser.add_argument("--lane", default="dubizzle_cars")
     sync_listing_parser.add_argument("--limit", type=int, default=None)
     sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
-    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
+    sync_listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_listing.json"))
 
     return parser
 
@@ -53,7 +53,7 @@ def main() -> None:
         if args.debug:
             runtime_settings.log_level = "DEBUG"
 
-    with IAAIScraper(runtime_settings) as scraper:
+    with DUBIZZLEScraper(runtime_settings) as scraper:
         if args.command == "init-db":
             data = scraper.init_db()
             print(f"DB initialized: {data}")
@@ -66,6 +66,21 @@ def main() -> None:
             data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
         else:
             only_new = None if args.only_new is None else args.only_new == "true"
+            segments = parse_listing_segments(scraper.settings.listing.listing_segments_json)
+            use_segmented = (
+                bool(segments)
+                and args.make is None
+                and args.model is None
+                and args.limit is None
+                and scraper.settings.discovery.mode in {"listing", "algolia"}
+            )
+            if use_segmented:
+                data = scraper.sync_listing_segmented(
+                    segments=segments,
+                    lane=args.lane,
+                    only_new=only_new,
+                )
+            else:
                 data = scraper.sync_listing(
                     make=args.make,
                     model=args.model,

dubizzle_scraper/core/config.py

@@ -0,0 +1,315 @@
+import json
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+TRUE_VALUES = {"1", "true", "yes", "on"}
+
+
+def _resolve_env_value(name: str) -> str | None:
+    """Возвращает значение env с поддержкой legacy-префиксов DUBIZZLE_/IAAI_."""
+    value = os.getenv(name)
+    if value is not None:
+        return value
+    if name.startswith("DUBIZZLE_"):
+        return os.getenv("IAAI_" + name[len("DUBIZZLE_"):])
+    return None
+
+
+#  Хелперы для чтения env-переменных с приведением типов 
+
+def _env_str(name: str, default: str) -> str:
+    value = _resolve_env_value(name)
+    return value if value is not None else default
+
+
+def _env_optional_str(name: str) -> str | None:
+    value = _resolve_env_value(name)
+    if value is None:
+        return None
+    value = value.strip()
+    return value or None
+
+
+def _env_path_str(name: str) -> str | None:
+    value = _env_optional_str(name)
+    if value is None:
+        return None
+    return str(Path(value).expanduser())
+
+
+def _env_bool(name: str, default: bool) -> bool:
+    fallback = "true" if default else "false"
+    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
+
+
+def _env_int(name: str, default: int) -> int:
+    return int(_env_str(name, str(default)).strip())
+
+
+def _env_float(name: str, default: float) -> float:
+    return float(_env_str(name, str(default)).strip())
+
+
+#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
+
+@dataclass(slots=True)
+class FingerprintConfig:
+    user_agent: str = (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/135.0.0.0 Safari/537.36"
+    )
+    viewport_presets: tuple[dict[str, int], ...] = (
+        {"width": 1920, "height": 1080},
+        {"width": 1600, "height": 900},
+        {"width": 1536, "height": 864},
+        {"width": 1440, "height": 900},
+        {"width": 1366, "height": 768},
+    )
+    timezone_candidates: tuple[str, ...] = (
+        "America/New_York",
+        "America/Chicago",
+        "America/Los_Angeles",
+    )
+    locale: str = "en-US"
+    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
+
+
+#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
+
+@dataclass(slots=True)
+class CaptureConfig:
+    capture_same_origin_only: bool = _env_bool("DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY", True)
+    max_requests: int = _env_int("DUBIZZLE_MAX_CAPTURED_REQUESTS", 40)
+    max_json_responses: int = _env_int("DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES", 30)
+
+
+#  Конфиг пауз между действиями (имитация человека) 
+
+@dataclass(slots=True)
+class HumanPaceConfig:
+    enabled: bool = _env_bool("DUBIZZLE_HUMAN_PACE_ENABLED", True)
+    after_listing_open_min_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MIN_S", 0.5)
+    after_listing_open_max_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MAX_S", 1.2)
+    after_filter_action_min_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MIN_S", 0.5)
+    after_filter_action_max_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MAX_S", 1.2)
+    before_vehicle_open_min_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
+    before_vehicle_open_max_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
+    after_vehicle_open_min_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
+    after_vehicle_open_max_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
+    between_vehicles_min_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MIN_S", 0.05)
+    between_vehicles_max_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MAX_S", 0.15)
+    after_page_change_min_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S", 0.8)
+    after_page_change_max_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S", 1.8)
+
+
+#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
+
+@dataclass(slots=True)
+class ListingConfig:
+    cars_url: str = _env_str("DUBIZZLE_CARS_LISTING_URL", "https://dubai.dubizzle.com/motors/used-cars/")
+    max_pages_per_run: int = _env_int("DUBIZZLE_MAX_PAGES_PER_RUN", 9999)
+    max_vehicles_per_run: int = _env_int("DUBIZZLE_MAX_VEHICLES_PER_RUN", 50000)
+    page_link_limit: int = _env_int("DUBIZZLE_PAGE_LINK_LIMIT", 500)
+    include_pagination: bool = _env_bool("DUBIZZLE_INCLUDE_PAGINATION", True)
+    collect_current_page_only: bool = _env_bool("DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY", False)
+    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
+    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
+    early_stop_threshold: float = _env_float("DUBIZZLE_EARLY_STOP_THRESHOLD", 0.8)
+    # Сегментация листинга по брендам для обхода лимита пагинации DUBIZZLE (~22 600 машин).
+    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
+    # Пустая строка = без сегментации (backward compatible).
+    listing_segments_json: str = _env_str("DUBIZZLE_LISTING_SEGMENTS", "")
+
+
+# Пагинационный потолок одного Algolia-запроса: ~100 страниц × 100 = 10 000 результатов.
+DUBIZZLE_PAGINATION_CEILING = 10_000
+
+# Авто-сегментация по году. Эти диапазоны подобраны так, чтобы каждый сегмент
+# оставался ниже лимита Algolia и собирался быстрее, чем старая make/year схема.
+_AUTO_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
+    (1900, 2012),
+    (2013, 2015),
+    (2016, 2017),
+    (2018, 2019),
+    (2020, 2021),
+    (2022, 2023),
+    (2024, 2025),
+    (2026, 2027),
+)
+
+
+def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
+    """Парсит DUBIZZLE_LISTING_SEGMENTS в список сегментов.
+
+    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
+    Специальное значение ``"auto"`` генерирует быстрые year-only сегменты,
+    которые гарантированно проходят через Algolia без упора в лимит ~10k.
+    """
+    raw = raw.strip()
+    if not raw:
+        return []
+    if raw.lower() == "auto":
+        return [
+            {"make": None, "year_min": yr_min, "year_max": yr_max}
+            for yr_min, yr_max in _AUTO_YEAR_SPLITS
+        ]
+    try:
+        data = json.loads(raw)
+    except (json.JSONDecodeError, ValueError):
+        return []
+    if not isinstance(data, list):
+        return []
+    segments = []
+    for item in data:
+        if isinstance(item, str):
+            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
+        elif isinstance(item, dict):
+            segments.append({
+                "make": str(item.get("make") or "").upper() or None,
+                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
+                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
+            })
+    return segments
+
+
+# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
+
+@dataclass(slots=True)
+class DatabaseConfig:
+    url: str = _env_str("DUBIZZLE_DATABASE_URL", "postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper")
+    echo: bool = _env_bool("DUBIZZLE_DATABASE_ECHO", False)
+    pool_size: int = _env_int("DUBIZZLE_DATABASE_POOL_SIZE", 5)
+    max_overflow: int = _env_int("DUBIZZLE_DATABASE_MAX_OVERFLOW", 10)
+    pool_recycle_seconds: int = _env_int("DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
+    auto_create_tables: bool = _env_bool("DUBIZZLE_DATABASE_AUTO_CREATE_TABLES", False)
+
+
+# --- Конфиг Redis (URL для Celery broker) ---
+
+@dataclass(slots=True)
+class RedisConfig:
+    url: str = _env_str("DUBIZZLE_REDIS_URL", "redis://localhost:6379/0")
+    socket_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
+    socket_connect_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
+    health_check_interval_seconds: int = _env_int("DUBIZZLE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
+
+
+# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
+
+@dataclass(slots=True)
+class DiscoveryConfig:
+    mode: str = _env_str("DUBIZZLE_DISCOVERY_MODE", "algolia")
+    hourly_mode: str = _env_str("DUBIZZLE_HOURLY_MODE", "rolling_refresh")
+    hourly_refresh_batch_size: int = _env_int("DUBIZZLE_HOURLY_REFRESH_BATCH_SIZE", 500)
+    always_full_scan: bool = _env_bool("DUBIZZLE_ALWAYS_FULL_SCAN", False)
+    sitemap_fallback_on_empty: bool = _env_bool("DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY", False)
+
+
+@dataclass(slots=True)
+class AlgoliaConfig:
+    application_id: str = _env_str("DUBIZZLE_ALGOLIA_APPLICATION_ID", "WD0PTZ13ZS")
+    api_key: str = _env_str(
+        "DUBIZZLE_ALGOLIA_API_KEY",
+        "cef139620248f1bc328a00fddc7107a6",
+    )
+    index_name: str = _env_str("DUBIZZLE_ALGOLIA_INDEX_NAME", "motors.com")
+    category_slug: str = _env_str("DUBIZZLE_ALGOLIA_CATEGORY_SLUG", "motors/used-cars")
+    base_url: str = _env_str("DUBIZZLE_ALGOLIA_BASE_URL", "https://WD0PTZ13ZS-dsn.algolia.net")
+    hits_per_page: int = _env_int("DUBIZZLE_ALGOLIA_HITS_PER_PAGE", 100)
+
+
+# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
+
+@dataclass(slots=True)
+class CeleryConfig:
+    broker_url: str = _env_str("CELERY_BROKER_URL", "")
+    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
+    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
+    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
+    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
+    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
+    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
+    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
+    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
+    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
+    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
+    parallel_tabs: int = _env_int("DUBIZZLE_PARALLEL_TABS", 8)
+    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
+    block_resources: bool = _env_bool("DUBIZZLE_BLOCK_RESOURCES", True)
+
+
+# --- Конфиг прокси (server, username, password) ---
+
+@dataclass(slots=True)
+class ProxyConfig:
+    server: str | None = _env_optional_str("DUBIZZLE_PROXY_SERVER")
+    username: str | None = _env_optional_str("DUBIZZLE_PROXY_USERNAME")
+    password: str | None = _env_optional_str("DUBIZZLE_PROXY_PASSWORD")
+
+    @property
+    def enabled(self) -> bool:
+        return bool(self.server)
+
+    def to_playwright_dict(self) -> dict[str, str] | None:
+        if not self.server:
+            return None
+        result: dict[str, str] = {"server": self.server}
+        if self.username:
+            result["username"] = self.username
+        if self.password:
+            result["password"] = self.password
+        return result
+
+
+#  Главный объект настроек: собирает все блоки конфигурации 
+
+@dataclass(slots=True)
+class Settings:
+    home_url: str = "https://www.dubizzle.com/"
+    default_timeout_ms: int = _env_int("DUBIZZLE_TIMEOUT_MS", 45000)
+    network_settle_ms: int = _env_int("DUBIZZLE_NETWORK_SETTLE_MS", 400)
+    fast_path_timeout_ms: int = _env_int("DUBIZZLE_FAST_PATH_TIMEOUT_MS", 5000)
+    fast_path_max_attempts: int = _env_int("DUBIZZLE_FAST_PATH_MAX_ATTEMPTS", 1)
+    fallback_navigation_timeout_ms: int = _env_int("DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS", 15000)
+    max_retries: int = _env_int("DUBIZZLE_MAX_RETRIES", 3)
+    retry_delay_seconds: float = _env_float("DUBIZZLE_RETRY_DELAY_SECONDS", 2.5)
+    retry_backoff_multiplier: float = _env_float("DUBIZZLE_RETRY_BACKOFF_MULTIPLIER", 2.0)
+    retry_jitter_seconds: float = _env_float("DUBIZZLE_RETRY_JITTER_SECONDS", 0.25)
+    headless: bool = _env_bool("DUBIZZLE_HEADLESS", True)
+    browser_engine: str = _env_str("DUBIZZLE_BROWSER_ENGINE", "auto")
+    log_level: str = _env_str("DUBIZZLE_LOG_LEVEL", "INFO")
+    log_file: str | None = _env_optional_str("DUBIZZLE_LOG_FILE")
+    enable_trace_id_logs: bool = _env_bool("DUBIZZLE_ENABLE_TRACE_ID_LOGS", True)
+    sync_only_new: bool = _env_bool("DUBIZZLE_SYNC_ONLY_NEW", False)
+    raw_output_json: str | None = _env_optional_str("DUBIZZLE_RAW_OUTPUT_JSON")
+    tokens_file: str | None = _env_path_str("DUBIZZLE_TOKENS_FILE")
+    runtime_config_file: str | None = _env_path_str("DUBIZZLE_RUNTIME_CONFIG_FILE")
+    scheduler_interval_minutes: int = _env_int("DUBIZZLE_SCHEDULER_INTERVAL_MINUTES", 60)
+    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
+    capture: CaptureConfig = field(default_factory=CaptureConfig)
+    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
+    listing: ListingConfig = field(default_factory=ListingConfig)
+    database: DatabaseConfig = field(default_factory=DatabaseConfig)
+    redis: RedisConfig = field(default_factory=RedisConfig)
+    celery: CeleryConfig = field(default_factory=CeleryConfig)
+    proxy: ProxyConfig = field(default_factory=ProxyConfig)
+    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
+    algolia: AlgoliaConfig = field(default_factory=AlgoliaConfig)
+
+    @property
+    def parallel_tabs(self) -> int:
+        return self.celery.parallel_tabs
+
+    @property
+    def block_resources(self) -> bool:
+        return self.celery.block_resources
+
+# Глобальный синглтон — используется по умолчанию во всех модулях.
+settings = Settings()

dubizzle_scraper/core/retry.py

@@ -9,7 +9,7 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
 
 from .exceptions import AntiBotDetectedError
 
-logger = logging.getLogger("iaai_scraper.retry")
+logger = logging.getLogger("dubizzle_scraper.retry")
 
 # Типы исключений, при которых retry имеет смысл.
 RETRYABLE_EXCEPTIONS = (

dubizzle_scraper/core/runtime_config.py

@@ -5,7 +5,7 @@ from pathlib import Path
 from typing import Any
 
 
-logger = logging.getLogger("iaai_scraper.runtime_config")
+logger = logging.getLogger("dubizzle_scraper.runtime_config")
 
 
 def _normalize_text(value: str) -> str:

dubizzle_scraper/discovery/__init__.py

@@ -5,6 +5,11 @@ from .sitemap import (
     discover_vehicle_urls_from_sitemap,
     discover_vehicle_urls_from_sitemap_with_stats,
 )
+from .algolia import (
+    AlgoliaDiscoveryError,
+    AlgoliaDiscoveryResult,
+    discover_vehicle_hits_from_algolia,
+)
 
 __all__ = [
     "SitemapDiscoveryError",
@@ -12,4 +17,7 @@ __all__ = [
     "SitemapDiscoveryStats",
     "discover_vehicle_urls_from_sitemap",
     "discover_vehicle_urls_from_sitemap_with_stats",
+    "AlgoliaDiscoveryError",
+    "AlgoliaDiscoveryResult",
+    "discover_vehicle_hits_from_algolia",
 ]

dubizzle_scraper/discovery/algolia.py

@@ -0,0 +1,479 @@
+from __future__ import annotations
+
+import json
+import logging
+import time
+from dataclasses import dataclass, field
+from typing import Any
+from urllib.parse import parse_qs, urlencode, urlparse
+from urllib.request import Request, urlopen
+
+
+logger = logging.getLogger("dubizzle_scraper.discovery.algolia")
+
+ALGOLIA_HARD_PAGE_LIMIT = 100
+ALGOLIA_MAX_HITS_PER_QUERY = ALGOLIA_HARD_PAGE_LIMIT * 100
+ALGOLIA_ID_RANGE_START = 0
+ALGOLIA_ID_RANGE_END = 20_000_000
+ALGOLIA_ID_RANGE_MIN_WINDOW = 100
+
+
+class AlgoliaDiscoveryError(RuntimeError):
+    pass
+
+
+@dataclass(slots=True)
+class AlgoliaDiscoveryPageStat:
+    page_number: int
+    hits_count: int
+
+
+@dataclass(slots=True)
+class AlgoliaDiscoveryResult:
+    vehicle_urls: list[str] = field(default_factory=list)
+    hit_records: dict[str, dict[str, Any]] = field(default_factory=dict)
+    origin_ids_by_url: dict[str, str] = field(default_factory=dict)
+    pages: list[dict[str, int]] = field(default_factory=list)
+    early_stopped: bool = False
+    truncated_by_time_budget: bool = False
+    total_hits: int = 0
+
+
+@dataclass(slots=True)
+class _AlgoliaShard:
+    category_slug: str
+    id_min: int | None = None
+    id_max: int | None = None
+    year_min: int | None = None
+    year_max: int | None = None
+
+    def filter_expr(self) -> str:
+        parts = [f"(category_v2.slug_paths:{self.category_slug})"]
+        if self.year_min is not None:
+            parts.append(f"year>={int(self.year_min)}")
+        if self.year_max is not None:
+            parts.append(f"year<={int(self.year_max)}")
+        if self.id_min is not None:
+            parts.append(f"id>={int(self.id_min)}")
+        if self.id_max is not None:
+            parts.append(f"id<={int(self.id_max)}")
+        return " AND ".join(parts)
+
+    def label(self) -> str:
+        label = self.category_slug
+        if self.year_min is not None or self.year_max is not None:
+            label += f"[year:{self.year_min}-{self.year_max}]"
+        if self.id_min is None and self.id_max is None:
+            return label
+        return f"{label}[id:{self.id_min}-{self.id_max}]"
+
+
+@dataclass(slots=True)
+class _AlgoliaHttpClient:
+    endpoint: str
+    app_id: str
+    api_key: str
+    user_agent: str
+    index_name: str
+    hits_per_page: int
+
+    def request(self, *, page: int, filters: str) -> dict[str, Any]:
+        params = urlencode(
+            {
+                "query": "",
+                "page": page,
+                "hitsPerPage": self.hits_per_page,
+                "filters": filters,
+            }
+        )
+        payload = {"requests": [{"indexName": self.index_name, "params": params}]}
+        request = Request(
+            self.endpoint,
+            data=json.dumps(payload).encode("utf-8"),
+            headers={
+                "content-type": "application/json",
+                "x-algolia-application-id": self.app_id,
+                "x-algolia-api-key": self.api_key,
+                "accept": "application/json",
+                "user-agent": self.user_agent,
+            },
+            method="POST",
+        )
+        with urlopen(request, timeout=30) as response:
+            body = response.read().decode("utf-8", errors="ignore")
+        parsed = json.loads(body)
+        results = parsed.get("results") or []
+        return results[0] if results and isinstance(results[0], dict) else {}
+
+
+def _build_origin_id_from_hit(hit: dict[str, Any]) -> str | None:
+    for key in ("id", "objectID", "uuid"):
+        value = hit.get(key)
+        if value is None:
+            continue
+        text = str(value).strip()
+        if text:
+            return f"dubizzle:{text}"
+    permalink = str(hit.get("permalink") or "").strip()
+    if permalink:
+        tail = permalink.rstrip("/").split("/")[-1]
+        if tail:
+            return f"dubizzle:{tail}"
+    return None
+
+
+def _build_vehicle_url_from_hit(hit: dict[str, Any]) -> str | None:
+    permalink = str(hit.get("permalink") or "").strip()
+    if permalink:
+        if permalink.startswith("http://") or permalink.startswith("https://"):
+            return permalink
+        if permalink.startswith("/"):
+            return f"https://www.dubizzle.com{permalink}"
+        return f"https://www.dubizzle.com/{permalink.lstrip('/')}"
+
+    short = str(hit.get("short_url") or "").strip()
+    if short:
+        if short.startswith("http://") or short.startswith("https://"):
+            return short
+        return f"https://dubizzle.com/s/{short.strip('/')}"
+
+    hit_id = hit.get("id") or hit.get("objectID")
+    if hit_id is not None:
+        return f"https://www.dubizzle.com/motors/used-cars/ad-{hit_id}/"
+    return None
+
+
+def _extract_make_from_listing_url(listing_url: str | None) -> str | None:
+    if not listing_url:
+        return None
+    try:
+        parsed = urlparse(listing_url)
+        params = parse_qs(parsed.query)
+        candidate = (params.get("Make") or params.get("make") or [None])[0]
+        if candidate:
+            return str(candidate).strip()
+        parts = [p for p in parsed.path.split("/") if p]
+        if len(parts) >= 3 and parts[0].lower() == "motors" and parts[1].lower() == "used-cars":
+            slug = parts[2].strip().lower()
+            if slug and slug != "s":
+                return slug.replace("-", " ")
+    except Exception:
+        return None
+    return None
+
+
+def _make_slug(make: str | None) -> str | None:
+    if not make:
+        return None
+    slug = make.strip().lower().replace(" ", "-")
+    return slug or None
+
+
+def _hit_matches_filters(
+    hit: dict[str, Any],
+    *,
+    make: str | None,
+    model: str | None,
+    year_min: int | None,
+    year_max: int | None,
+) -> bool:
+    if make:
+        hit_make = str(hit.get("make") or "").strip().lower()
+        if hit_make != make.strip().lower():
+            return False
+    if model:
+        hit_model = str(hit.get("model") or "").strip().lower()
+        if hit_model != model.strip().lower():
+            return False
+
+    hit_year_raw = hit.get("year")
+    hit_year: int | None = None
+    if hit_year_raw is not None:
+        try:
+            hit_year = int(hit_year_raw)
+        except Exception:
+            hit_year = None
+
+    if year_min is not None and (hit_year is None or hit_year < year_min):
+        return False
+    if year_max is not None and (hit_year is None or hit_year > year_max):
+        return False
+
+    return True
+
+
+def _probe_total_hits(client: _AlgoliaHttpClient, shard: _AlgoliaShard) -> int:
+    first = client.request(page=0, filters=shard.filter_expr())
+    return int(first.get("nbHits") or 0)
+
+
+def _split_id_range(shard: _AlgoliaShard) -> tuple[_AlgoliaShard, _AlgoliaShard] | None:
+    lo = shard.id_min if shard.id_min is not None else ALGOLIA_ID_RANGE_START
+    hi = shard.id_max if shard.id_max is not None else ALGOLIA_ID_RANGE_END
+    if hi - lo <= ALGOLIA_ID_RANGE_MIN_WINDOW:
+        return None
+    mid = (lo + hi) // 2
+    return (
+        _AlgoliaShard(
+            category_slug=shard.category_slug,
+            id_min=lo,
+            id_max=mid,
+            year_min=shard.year_min,
+            year_max=shard.year_max,
+        ),
+        _AlgoliaShard(
+            category_slug=shard.category_slug,
+            id_min=mid + 1,
+            id_max=hi,
+            year_min=shard.year_min,
+            year_max=shard.year_max,
+        ),
+    )
+
+
+def _expand_shards(client: _AlgoliaHttpClient, initial_shard: _AlgoliaShard, max_duration_seconds: float | None, started_at: float) -> tuple[list[_AlgoliaShard], bool, int]:
+    queue: list[_AlgoliaShard] = [initial_shard]
+    ready: list[_AlgoliaShard] = []
+    truncated = False
+    total_hits = 0
+
+    while queue:
+        if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
+            truncated = True
+            break
+        shard = queue.pop(0)
+        shard_total = _probe_total_hits(client, shard)
+        if shard is initial_shard:
+            total_hits = shard_total
+        if shard_total == 0:
+            continue
+        if shard_total <= ALGOLIA_MAX_HITS_PER_QUERY:
+            ready.append(shard)
+            continue
+        split = _split_id_range(shard)
+        if split is None:
+            logger.warning(
+                "Shard %s still exceeds limit=%d but id-window is too small to split further (hits=%d)",
+                shard.label(),
+                ALGOLIA_MAX_HITS_PER_QUERY,
+                shard_total,
+            )
+            ready.append(shard)
+            continue
+        logger.warning(
+            "Splitting Algolia shard %s with hits=%d into %s and %s",
+            shard.label(),
+            shard_total,
+            split[0].label(),
+            split[1].label(),
+        )
+        queue.insert(0, split[1])
+        queue.insert(0, split[0])
+
+    return ready, truncated, total_hits
+
+
+def _fetch_shard_hits(
+    *,
+    client: _AlgoliaHttpClient,
+    shard: _AlgoliaShard,
+    make: str | None,
+    model: str | None,
+    year_min: int | None,
+    year_max: int | None,
+    known_origin_ids: set[str] | None,
+    threshold: float,
+    max_duration_seconds: float | None,
+    started_at: float,
+) -> tuple[list[str], dict[str, dict[str, Any]], dict[str, str], list[dict[str, int]], bool, bool]:
+    urls: list[str] = []
+    hit_records: dict[str, dict[str, Any]] = {}
+    origin_ids_by_url: dict[str, str] = {}
+    pages: list[dict[str, int]] = []
+    early_stopped = False
+    truncated_by_time_budget = False
+    page = 0
+    nb_pages = None
+
+    while True:
+        if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
+            truncated_by_time_budget = True
+            break
+
+        try:
+            first = client.request(page=page, filters=shard.filter_expr())
+        except Exception as exc:
+            raise AlgoliaDiscoveryError(
+                f"Algolia request failed for shard={shard.label()} page={page}: {exc}"
+            ) from exc
+
+        hits = first.get("hits") or []
+        if not isinstance(hits, list):
+            hits = []
+
+        if page == 0:
+            nb_pages = min(int(first.get("nbPages") or 0), ALGOLIA_HARD_PAGE_LIMIT)
+
+        pages.append({"page_number": page + 1, "links_found": len(hits), "shard": shard.label()})
+        if not hits:
+            break
+
+        page_known = 0
+        page_new = 0
+        for raw_hit in hits:
+            if not isinstance(raw_hit, dict):
+                continue
+            if not _hit_matches_filters(
+                raw_hit,
+                make=make,
+                model=model,
+                year_min=year_min,
+                year_max=year_max,
+            ):
+                continue
+
+            url = _build_vehicle_url_from_hit(raw_hit)
+            if not url:
+                continue
+            origin_id = _build_origin_id_from_hit(raw_hit)
+            if origin_id and known_origin_ids is not None and origin_id in known_origin_ids:
+                page_known += 1
+            else:
+                page_new += 1
+
+            if url in hit_records:
+                continue
+            urls.append(url)
+            hit_records[url] = raw_hit
+            if origin_id:
+                origin_ids_by_url[url] = origin_id
+
+        if known_origin_ids is not None and threshold > 0 and (page_known + page_new) > 0:
+            ratio = page_known / (page_known + page_new)
+            if ratio >= threshold and page_new == 0:
+                early_stopped = True
+                break
+
+        page += 1
+        if nb_pages is not None and page >= nb_pages:
+            break
+
+    return urls, hit_records, origin_ids_by_url, pages, early_stopped, truncated_by_time_budget
+
+
+def discover_vehicle_hits_from_algolia(
+    *,
+    settings,
+    make: str | None = None,
+    model: str | None = None,
+    limit: int | None = None,
+    year_min: int | None = None,
+    year_max: int | None = None,
+    listing_url: str | None = None,
+    known_origin_ids: set[str] | None = None,
+    max_duration_seconds: float | None = None,
+) -> AlgoliaDiscoveryResult:
+    app_id = settings.algolia.application_id.strip()
+    api_key = settings.algolia.api_key.strip()
+    index_name = settings.algolia.index_name.strip()
+    if not app_id or not api_key or not index_name:
+        raise AlgoliaDiscoveryError("Algolia credentials/index are not configured")
+
+    effective_make = make or _extract_make_from_listing_url(listing_url)
+    hits_per_page = max(1, min(100, int(settings.algolia.hits_per_page)))
+    base_url = settings.algolia.base_url.strip().rstrip("/")
+    if not base_url:
+        base_url = f"https://{app_id}-dsn.algolia.net"
+
+    category_slug = settings.algolia.category_slug.strip() or "motors/used-cars"
+    make_slug = _make_slug(effective_make)
+    if make_slug:
+        category_slug = f"{category_slug}/{make_slug}"
+
+    client = _AlgoliaHttpClient(
+        endpoint=f"{base_url}/1/indexes/*/queries",
+        app_id=app_id,
+        api_key=api_key,
+        user_agent=settings.fingerprint.user_agent,
+        index_name=index_name,
+        hits_per_page=hits_per_page,
+    )
+    threshold = float(settings.listing.early_stop_threshold)
+    started_at = time.perf_counter()
+
+    initial_shard = _AlgoliaShard(
+        category_slug=category_slug,
+        id_min=ALGOLIA_ID_RANGE_START,
+        id_max=ALGOLIA_ID_RANGE_END,
+        year_min=year_min,
+        year_max=year_max,
+    )
+    shards, shard_build_truncated, total_hits = _expand_shards(
+        client,
+        initial_shard,
+        max_duration_seconds,
+        started_at,
+    )
+
+    collected_urls: list[str] = []
+    hits_by_url: dict[str, dict[str, Any]] = {}
+    origin_ids_by_url: dict[str, str] = {}
+    pages: list[dict[str, int]] = []
+    early_stopped = False
+    truncated_by_time_budget = shard_build_truncated
+
+    logger.warning(
+        "Algolia shard plan ready: total_hits=%d shards=%d category=%s",
+        total_hits,
+        len(shards),
+        category_slug,
+    )
+
+    for shard in shards:
+        shard_urls, shard_hits, shard_origin_ids, shard_pages, shard_early_stop, shard_truncated = _fetch_shard_hits(
+            client=client,
+            shard=shard,
+            make=effective_make,
+            model=model,
+            year_min=year_min,
+            year_max=year_max,
+            known_origin_ids=known_origin_ids,
+            threshold=threshold,
+            max_duration_seconds=max_duration_seconds,
+            started_at=started_at,
+        )
+        pages.extend(shard_pages)
+        early_stopped = early_stopped or shard_early_stop
+        truncated_by_time_budget = truncated_by_time_budget or shard_truncated
+        for url in shard_urls:
+            if url in hits_by_url:
+                continue
+            collected_urls.append(url)
+            hits_by_url[url] = shard_hits[url]
+            if url in shard_origin_ids:
+                origin_ids_by_url[url] = shard_origin_ids[url]
+        if limit is not None and limit > 0 and len(collected_urls) >= limit:
+            break
+        if truncated_by_time_budget:
+            break
+
+    logger.info(
+        "Algolia discovery done: urls=%d total_hits=%d pages=%d shards=%d",
+        len(collected_urls),
+        total_hits,
+        len(pages),
+        len(shards),
+    )
+
+    if limit is not None and limit > 0 and len(collected_urls) > limit:
+        collected_urls = collected_urls[:limit]
+
+    return AlgoliaDiscoveryResult(
+        vehicle_urls=collected_urls,
+        hit_records={url: hits_by_url[url] for url in collected_urls if url in hits_by_url},
+        origin_ids_by_url={url: origin_ids_by_url[url] for url in collected_urls if url in origin_ids_by_url},
+        pages=pages,
+        early_stopped=early_stopped,
+        truncated_by_time_budget=truncated_by_time_budget,
+        total_hits=total_hits,
+    )

dubizzle_scraper/discovery/sitemap.py

@@ -10,9 +10,9 @@ from urllib.parse import urlsplit, urlunsplit
 from urllib.request import Request, urlopen, ProxyHandler, build_opener
 import xml.etree.ElementTree as ET
 
-logger = logging.getLogger("iaai_scraper.discovery.sitemap")
+logger = logging.getLogger("dubizzle_scraper.discovery.sitemap")
 
-DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
+DEFAULT_SITEMAP_INDEX_URL = "https://www.dubizzle.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
 _SITEMAP_TIMEOUT_SECONDS = 30
 _LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
 

dubizzle_scraper/parsing/mapper.py

@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
 
 
 class CarMapper:
-	# Маппер IAAI в CarRecord.
+	# Маппер DUBIZZLE в CarRecord.
 
 	BODY_MAP = {
 		"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -44,6 +44,7 @@ class CarMapper:
 	COUNTRY_MAP = {
 		"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
 		"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
+		"ae": "AE", "uae": "AE", "united arab emirates": "AE", "dubai": "AE", "abu dhabi": "AE",
 	}
 	NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
 
@@ -59,9 +60,14 @@ class CarMapper:
 
 		origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
 		parser_id = self._generate_parser_id(origin_id)
-		brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
+		brand = self._resolve_make(core, vehicle_summary) or "UNKNOWN"
-		model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
+		model = self._resolve_model(core, vehicle_summary) or "UNKNOWN"
-		year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
+		year = self._to_year(first_non_empty([
+			core.get("year"),
+			vehicle_summary.get("year"),
+			self._extract_detail_v2_value(vehicle_summary, "year"),
+			self._extract_detail_value(vehicle_summary, "Year"),
+		]))
 		price = self._first_parsed_int(
 			[
 				pricing.get("buy_now"),
@@ -69,29 +75,90 @@ class CarMapper:
 				vehicle_summary.get("buy_now"),
 				vehicle_summary.get("current_bid"),
 				pricing.get("actual_cash_value"),
+				vehicle_summary.get("price"),
+				self._extract_detail_v2_value(vehicle_summary, "price"),
+				self._extract_detail_value(vehicle_summary, "Price"),
 			],
 			self._to_money_int,
 		)
 		mileage = self._parse_odometer(
-			first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
+			first_non_empty([
+				core.get("odometer"),
+				core.get("kilometers"),
+				vehicle_summary.get("odometer"),
+				vehicle_summary.get("kilometers"),
+				self._extract_detail_v2_value(vehicle_summary, "kilometers"),
+				self._extract_detail_value(vehicle_summary, "Kilometers"),
+			])
 		)
-		color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
+		color = self._normalize_color(first_non_empty([
-		drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
+			core.get("color"),
+			vehicle_summary.get("color"),
+			vehicle_summary.get("exterior_color"),
+			self._extract_detail_v2_value(vehicle_summary, "exterior_color"),
+			self._extract_detail_value(vehicle_summary, "Exterior Color"),
+			"other",
+		]))
+		drive = self._normalize_drive(first_non_empty([
+			core.get("drive"),
+			vehicle_summary.get("drive"),
+			vehicle_summary.get("drive_type"),
+			self._extract_detail_v2_value(vehicle_summary, "drive_system"),
+			self._extract_detail_value(vehicle_summary, "Drive Type"),
+			self._extract_detail_value(vehicle_summary, "Drive System"),
+		]))
 		# Пробуем взять привод из двигателя.
 		if not drive or drive == "NA":
-			engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
+			engine_text = self._as_str(first_non_empty([
+				core.get("engine"),
+				vehicle_summary.get("engine"),
+				self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
+				self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
+			]))
 			if engine_text:
 				inferred_drive = self._normalize_drive(engine_text)
 				if inferred_drive and inferred_drive != "NA":
 					drive = inferred_drive
-		gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
+		gearbox = self._normalize_gearbox(first_non_empty([
-		steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
+			core.get("gearbox"),
-		body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
+			vehicle_summary.get("gearbox"),
-		engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
+			vehicle_summary.get("transmission_type"),
+			vehicle_summary.get("transmission"),
+			self._extract_detail_v2_value(vehicle_summary, "transmission_type"),
+			self._extract_detail_value(vehicle_summary, "Transmission Type"),
+		]))
+		steering = self._normalize_steering(first_non_empty([
+			core.get("steering_wheel"),
+			vehicle_summary.get("steering_wheel"),
+			self._extract_detail_v2_value(vehicle_summary, "steering_side"),
+			self._extract_detail_value(vehicle_summary, "Steering Side"),
+		])) or "LEFT"
+		body_type = self._normalize_body_type(first_non_empty([
+			core.get("body_type"),
+			vehicle_summary.get("body_type"),
+			self._extract_detail_v2_value(vehicle_summary, "body_type"),
+			self._extract_detail_value(vehicle_summary, "Body Type"),
+		]))
+		engine_volume = self._to_engine_cc(first_non_empty([
+			core.get("engine"),
+			core.get("engine_volume"),
+			vehicle_summary.get("engine"),
+			vehicle_summary.get("engine_volume"),
+			self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
+			self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
+		]))
 		title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
 		seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
-		location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
+		location = self._as_str(first_non_empty([
-		country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
+			core.get("location"),
+			vehicle_summary.get("location"),
+			vehicle_summary.get("location_name"),
+			self._extract_nested_text(vehicle_summary.get("neighbourhood"), "en"),
+			self._extract_location_country(vehicle_summary),
+			auction.get("branch"),
+			"",
+		]))
+		country = self._normalize_country(first_non_empty([core.get("country"), location, "AE"]))
 		is_damaged = self._bool_damage(damage, vehicle_summary)
 		is_sold = self._bool_sold(auction)
 		one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
@@ -115,13 +182,13 @@ class CarMapper:
 		)
 		slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
 		images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
-		origin = "IAAI"
+		origin = "DUBIZZLE"
 
 		return CarRecord(
 			parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
 			mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
 			steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
-			selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
+			selling_type=self._normalize_selling_type(first_non_empty([core.get("selling_type"), "STOCK"])), one_owner=one_owner, new_car=new_car,
 			is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
 			evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
 			slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
@@ -131,6 +198,146 @@ class CarMapper:
 	def _as_str(value: Any) -> str:
 		return "" if value is None else str(value).strip()
 
+	def _resolve_make(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
+		category_make, _ = self._extract_category_make_model(vehicle_summary)
+		slug_make, _ = self._extract_slug_make_model(vehicle_summary)
+		return self._first_text(
+			[
+				core.get("make"),
+				vehicle_summary.get("make"),
+				self._extract_detail_v2_value(vehicle_summary, "make"),
+				self._extract_detail_value(vehicle_summary, "Make"),
+				category_make,
+				slug_make,
+			]
+		)
+
+	def _resolve_model(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
+		_, category_model = self._extract_category_make_model(vehicle_summary)
+		_, slug_model = self._extract_slug_make_model(vehicle_summary)
+		return self._first_text(
+			[
+				core.get("model"),
+				vehicle_summary.get("model"),
+				self._extract_detail_v2_value(vehicle_summary, "model"),
+				self._extract_detail_value(vehicle_summary, "Model"),
+				category_model,
+				slug_model,
+			]
+		)
+
+	def _first_text(self, values: list[Any]) -> str | None:
+		for value in values:
+			text = self._coerce_text(value)
+			if text:
+				return text
+		return None
+
+	def _coerce_text(self, value: Any) -> str | None:
+		if value is None:
+			return None
+		if isinstance(value, list):
+			for item in value:
+				text = self._coerce_text(item)
+				if text:
+					return text
+			return None
+		if isinstance(value, dict):
+			for key in ("en", "value", "name", "text", "label"):
+				if key in value:
+					text = self._coerce_text(value.get(key))
+					if text:
+						return text
+			for nested in value.values():
+				text = self._coerce_text(nested)
+				if text:
+					return text
+			return None
+		text = self._as_str(value)
+		return text or None
+
+	def _extract_nested_text(self, value: Any, preferred_key: str = "en") -> str | None:
+		if not isinstance(value, dict):
+			return self._coerce_text(value)
+		return self._coerce_text(value.get(preferred_key)) or self._coerce_text(value)
+
+	def _extract_detail_v2_value(self, vehicle_summary: dict[str, Any], slug: str) -> str | None:
+		details_v2 = vehicle_summary.get("details_v2")
+		if not isinstance(details_v2, dict):
+			return None
+		target = slug.strip().lower()
+		for section in details_v2.values():
+			if not isinstance(section, list):
+				continue
+			for item in section:
+				if not isinstance(item, dict):
+					continue
+				if self._as_str(item.get("slug")).lower() != target:
+					continue
+				text = self._coerce_text(item.get("value"))
+				if text:
+					return text
+		return None
+
+	def _extract_detail_value(self, vehicle_summary: dict[str, Any], detail_key: str) -> str | None:
+		details = vehicle_summary.get("details")
+		if not isinstance(details, dict):
+			return None
+		target = detail_key.strip().lower()
+		for key, value in details.items():
+			if self._as_str(key).lower() != target:
+				continue
+			text = self._coerce_text(value)
+			if text:
+				return text
+		return None
+
+	def _extract_category_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
+		category_v2 = vehicle_summary.get("category_v2")
+		if isinstance(category_v2, dict):
+			names_en = category_v2.get("names_en")
+			if isinstance(names_en, list) and len(names_en) >= 4:
+				return self._coerce_text(names_en[2]), self._coerce_text(names_en[3])
+
+		category = vehicle_summary.get("category")
+		if isinstance(category, dict):
+			names_en = category.get("en")
+			if isinstance(names_en, list) and len(names_en) >= 3:
+				return self._coerce_text(names_en[1]), self._coerce_text(names_en[2])
+
+		return None, None
+
+	def _extract_slug_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
+		category_v2 = vehicle_summary.get("category_v2")
+		if not isinstance(category_v2, dict):
+			return None, None
+		slug_paths = category_v2.get("slug_paths")
+		if not isinstance(slug_paths, list) or len(slug_paths) < 3:
+			return None, None
+		make = self._humanize_slug_path_part(slug_paths[2])
+		model = self._humanize_slug_path_part(slug_paths[3]) if len(slug_paths) >= 4 else None
+		return make, model
+
+	def _humanize_slug_path_part(self, value: Any) -> str | None:
+		text = self._as_str(value)
+		if not text:
+			return None
+		slug = text.split("/")[-1].strip().strip("-")
+		if not slug:
+			return None
+		return slug.replace("-", " ").title()
+
+	def _extract_location_country(self, vehicle_summary: dict[str, Any]) -> str | None:
+		site = vehicle_summary.get("site")
+		if isinstance(site, dict):
+			return self._coerce_text(site.get("en"))
+		location_list = vehicle_summary.get("location_list")
+		if isinstance(location_list, dict):
+			en_values = location_list.get("en")
+			if isinstance(en_values, list) and en_values:
+				return self._coerce_text(en_values[0])
+		return None
+
 	@staticmethod
 	def _to_int(value: Any) -> int | None:
 		if value is None:
@@ -238,7 +445,7 @@ class CarMapper:
 			return None
 		if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
 			return int(float(liters_match.group(1)) * 1000)
-		if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
+		if cubic_match := re.search(r"(\d{3,5})(?:\+)?\s*(?:cc|cm3|cubic)", text):
 			return int(cubic_match.group(1))
 		return int(text) if re.match(r"^\d+$", text) else None
 
@@ -278,12 +485,16 @@ class CarMapper:
 		return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
 
 	def _normalize_country(self, value: Any) -> str:
-		fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
+		fallback = "AE" if "AE" in COUNTRY_ENUM_VALUES else "NA"
-		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
+		return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="AE", fallback=fallback) or fallback
 
 	def _normalize_selling_type(self, value: Any) -> str:
-		text = self._as_str(value) or "AUCTION"
+		text = (self._as_str(value) or "STOCK").upper()
-		return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
+		if text in SELLING_TYPE_ENUM_VALUES:
+			return text
+		if text in {"DEALER", "PRIVATE", "CLASSIFIED"}:
+			return "STOCK"
+		return "STOCK"
 
 	def _map_value(
 		self,
@@ -357,13 +568,13 @@ class CarMapper:
 
 	@staticmethod
 	def _make_fullres_url(url: str) -> str:
-		if "vis.iaai.com" in url:
+		if "vis.dubizzle.com" in url:
 			return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
 		return url
 
 	@staticmethod
 	def _make_preview_url(url: str) -> str:
-		if "vis.iaai.com" in url:
+		if "vis.dubizzle.com" in url:
 			preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
 			if preview != url:
 				return preview
@@ -387,16 +598,22 @@ class CarMapper:
 		return "car-" + "".join(chars)
 
 	def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
-		# Формат: iaai:{lot_number}.
+		# Формат: dubizzle:{lot_number}.
-		for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
+		for value in [
+			core.get("lot_number"),
+			vehicle_summary.get("lot_number"),
+			vehicle_summary.get("id"),
+			vehicle_summary.get("objectID"),
+			vehicle_summary.get("uuid"),
+		]:
 			text = self._as_str(value)
 			if text:
-				return f"iaai:{text}"
+				return f"dubizzle:{text}"
 		tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
 		if "~" in tail:
 			tail = tail.split("~")[0]
 		raw = tail or self._slugify(vehicle_url)
-		return f"iaai:{raw}"
+		return f"dubizzle:{raw}"
 
 	@staticmethod
 	def _slugify(value: str) -> str:

dubizzle_scraper/parsing/parser.py

@@ -6,7 +6,7 @@ from typing import Any
 
 from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
 
-logger = logging.getLogger("iaai_scraper.parsers")
+logger = logging.getLogger("dubizzle_scraper.parsers")
 
 
 class VehicleParser:
@@ -350,7 +350,7 @@ class VehicleParser:
 			if isinstance(item, str) and item.startswith("http"):
 				cleaned = html_module.unescape(item)
 				lowered = cleaned.lower()
-				if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+				if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
 					continue
 				if cleaned not in seen_flat:
 					seen_flat.add(cleaned)
@@ -360,7 +360,7 @@ class VehicleParser:
 					if isinstance(child, str) and child.startswith("http"):
 						cleaned = html_module.unescape(child)
 						lowered = cleaned.lower()
-						if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
+						if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
 							continue
 						if cleaned not in seen_flat:
 							seen_flat.add(cleaned)
@@ -374,13 +374,13 @@ class VehicleParser:
 				if vehicle_key and vehicle_key in url:
 					seen_flat.add(url)
 					flat.append(url)
-				elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
+				elif any(token in lowered for token in ["vis.dubizzle.com", "anvis", "vehicleimage"]):
 					if vehicle_key and vehicle_key not in url:
 						continue
 					seen_flat.add(url)
 					flat.append(url)
 		if vehicle_key:
-			for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
+			for url in re.findall(r'https?://vis\.dubizzle\.com[^\s"\'<>]+', html or ""):
 				cleaned = html_module.unescape(url)
 				if cleaned not in seen_flat and vehicle_key in cleaned:
 					seen_flat.add(cleaned)
@@ -390,7 +390,7 @@ class VehicleParser:
 			lowered = url.lower()
 			if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
 				continue
-			if "vis.iaai.com" in lowered and "/resizer" not in lowered:
+			if "vis.dubizzle.com" in lowered and "/resizer" not in lowered:
 				continue
 			filtered.append(url)
 		return filtered

dubizzle_scraper/storage/db.py

@@ -11,7 +11,7 @@ from ..core.config import Settings
 from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord
 
-logger = logging.getLogger("iaai_scraper.db")
+logger = logging.getLogger("dubizzle_scraper.db")
 
 
 CAR_DB_FIELDS = {
@@ -492,7 +492,7 @@ class PersistenceService:
 				logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
 		return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
 
-	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
+	def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "dubizzle") -> int:
 		"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
 		if not active_origin_ids:
 			return 0
@@ -501,7 +501,7 @@ class PersistenceService:
 				update(Car)
 				.where(Car.origin_id.notin_(active_origin_ids))
 				.where(Car.is_sold == False)  # noqa: E712
-				.where(Car.origin_id.like("iaai:%"))
+				.where(Car.origin_id.like("dubizzle:%"))
 				.values(is_sold=True)
 			)
 			result = session.execute(stmt)
@@ -510,7 +510,7 @@ class PersistenceService:
 				logger.info("Marked %d cars as sold (no longer in listing)", count)
 			return count
 
-	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
+	def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "dubizzle") -> int:
 		"""Помечает авто как проданные, если их URL нет в активном листинге.
 
 		Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
@@ -532,7 +532,7 @@ class PersistenceService:
 			if is_postgres:
 				# Считаем кандидатов на sold.
 				total_active = session.execute(
-					text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
+					text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%%'")
 				).scalar() or 0
 
 				if total_active == 0:
@@ -560,7 +560,7 @@ class PersistenceService:
 					LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 					WHERE a.url IS NULL
 					AND c.is_sold = FALSE
-					AND c.origin_id LIKE 'iaai:%%'
+					AND c.origin_id LIKE 'dubizzle:%%'
 				""")).scalar() or 0
 
 				# Защита от аномалии.
@@ -581,7 +581,7 @@ class PersistenceService:
 						LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
-						AND c.origin_id LIKE 'iaai:%%'
+						AND c.origin_id LIKE 'dubizzle:%%'
 					) sub
 					WHERE cars.id = sub.id
 				"""))
@@ -591,13 +591,13 @@ class PersistenceService:
 				stmt = (
 					update(Car)
 					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like("iaai:%"))
+					.where(Car.origin_id.like("dubizzle:%"))
 					.values(is_sold=True)
 				)
 				# Загружаем active URL.
 				all_active = session.execute(
 					select(Car.id, Car.origin_url).where(
-						Car.is_sold == False, Car.origin_id.like("iaai:%")  # noqa: E712
+						Car.is_sold == False, Car.origin_id.like("dubizzle:%")  # noqa: E712
 					)
 				).all()
 				mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
@@ -621,7 +621,7 @@ class PersistenceService:
 				logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
 			return count
 
-	def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
+	def get_all_origin_ids_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
 		"""Возвращает все известные origin_id для заданного префикса.
 
 		Использует yield_per для потоковой загрузки при большом количестве записей.
@@ -632,7 +632,7 @@ class PersistenceService:
 			)
 			return {str(row[0]) for row in result if row and row[0]}
 
-	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
 		"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
 		with self.session_scope() as session:
 			result = session.execute(
@@ -643,7 +643,7 @@ class PersistenceService:
 			)
 			return {str(row[0]) for row in result if row and row[0]}
 
-	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+	def count_active_cars_for_lane(self, prefix: str = "dubizzle:") -> int:
 		"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
 		from sqlalchemy import func as sa_func
 		with self.session_scope() as session:
@@ -657,7 +657,7 @@ class PersistenceService:
 
 	def get_active_origin_urls_batch_for_refresh(
 		self,
-		prefix: str = "iaai:",
+		prefix: str = "dubizzle:",
 		offset: int = 0,
 		limit: int = 500,
 	) -> list[str]:

dubizzle_scraper/storage/enums.py

@@ -16,9 +16,9 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
-COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "AE", "NA")
 ORIGIN_ENUM_VALUES = (
-	"IAAI",
+	"DUBIZZLE",
 	"NA",
 )
 SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

dubizzle_scraper/worker/celery_app.py

@@ -9,8 +9,7 @@ from redis import Redis
 from ..core.config import settings
 from ..core.logs import setup_logging
 
-logger = logging.getLogger("iaai_scraper.worker.celery_app")
+logger = logging.getLogger("dubizzle_scraper.worker.celery_app")
-STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
 
 
 @celery_setup_logging.connect
@@ -37,7 +36,7 @@ def _result_backend() -> str:
 
 
 celery_app = Celery(
-    "iaai_scraper",
+    "dubizzle_scraper",
     broker=_broker_url(),
     backend=_result_backend(),
 )
@@ -79,10 +78,13 @@ celery_app.conf.update(
     worker_hijack_root_logger=False,
     beat_schedule={
         "periodic-sync-listing": {
-            "task": "iaai_scraper.worker.tasks.sync_listing_task",
+            "task": "dubizzle_scraper.worker.tasks.sync_listing_task",
             "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
             "args": (),
-            "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
+            "kwargs": {
+                "limit": settings.celery.beat_sync_limit,
+                "only_new": False if settings.discovery.always_full_scan else True,
+            },
             "options": {
                 "queue": "scraping",
                 "expires": settings.celery.beat_sync_interval_minutes * 60.0,
@@ -90,11 +92,11 @@ celery_app.conf.update(
         }
     },
     task_routes={
-        "iaai_scraper.worker.tasks.*": {"queue": "scraping"}
+        "dubizzle_scraper.worker.tasks.*": {"queue": "scraping"}
     },
 )
 
-celery_app.autodiscover_tasks(["iaai_scraper.worker"])
+celery_app.autodiscover_tasks(["dubizzle_scraper.worker"])
 
 
 @worker_ready.connect
@@ -111,7 +113,7 @@ def _on_worker_ready(**kwargs):
             retry_on_timeout=True,
         )
 
-        for stale_key in ("iaai:locks:sync_listing",):
+        for stale_key in ("dubizzle:locks:sync_listing",):
             try:
                 ttl = redis_client.ttl(stale_key)
                 if ttl is not None and ttl != -2:
@@ -127,10 +129,8 @@ def _on_worker_ready(**kwargs):
         if queue_len > 0:
             logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len)
             return
-
-        should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
     except Exception:
-        logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
+        logger.warning("Worker ready startup sync check failed; skipping immediate dispatch", exc_info=True)
         return
     finally:
         if redis_client is not None:
@@ -139,13 +139,9 @@ def _on_worker_ready(**kwargs):
             except Exception:
                 pass
 
-    if not should_dispatch:
-        logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
-        return
-
     logger.info("Worker ready — dispatching initial sync_listing task")
     celery_app.send_task(
-        "iaai_scraper.worker.tasks.sync_listing_task",
+        "dubizzle_scraper.worker.tasks.sync_listing_task",
         kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
         queue="scraping",
         expires=settings.celery.beat_sync_interval_minutes * 60.0,

dubizzle_scraper/worker/self_heal.py

@@ -8,21 +8,28 @@ import time
 from redis import Redis
 
 
-logger = logging.getLogger("iaai_scraper.worker.self_heal")
+logger = logging.getLogger("dubizzle_scraper.worker.self_heal")
 
-GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
+GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts"
-SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress"
+SELF_HEAL_RESTART_LOCK_KEY = "dubizzle:state:self_heal_restart_in_progress"
+
+
+def _env_str(name: str, default: str) -> str:
+	value = os.getenv(name)
+	if value is None and name.startswith("DUBIZZLE_"):
+		value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):])
+	return value if value is not None else default
 
 
 def _env_bool(name: str, default: bool) -> bool:
-	value = os.getenv(name)
+	value = _env_str(name, "true" if default else "false")
 	if value is None:
 		return default
 	return value.strip().lower() in {"1", "true", "yes", "on"}
 
 
 def _env_int(name: str, default: int) -> int:
-	value = os.getenv(name)
+	value = _env_str(name, str(default))
 	if value is None:
 		return default
 	try:
@@ -32,7 +39,7 @@ def _env_int(name: str, default: int) -> int:
 
 
 def _get_redis() -> Redis:
-	url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0")
+	url = _env_str("DUBIZZLE_REDIS_URL", "redis://redis:6379/0")
 	return Redis.from_url(
 		url,
 		decode_responses=True,
@@ -62,7 +69,7 @@ def _read_last_progress_ts(redis_client: Redis) -> int | None:
 	# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
 	# Это дороже, но выполняется только при отсутствии основного маркера.
 	max_ts = 0
-	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+	for key in redis_client.scan_iter(match="dubizzle:state:task_progress:*"):
 		try:
 			payload = redis_client.get(key)
 			if not payload:
@@ -101,7 +108,8 @@ def _kill_worker_process() -> None:
 		# Если процесс ещё жив — принудительно убиваем.
 		os.kill(pid, 0)
 		logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
-		os.kill(pid, signal.SIGKILL)
+		kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM)
+		os.kill(pid, kill_signal)
 	except ProcessLookupError:
 		pass
 	except Exception:
@@ -109,15 +117,15 @@ def _kill_worker_process() -> None:
 
 
 def main() -> None:
-	if not _env_bool("IAAI_SELF_HEAL_ENABLED", True):
+	if not _env_bool("DUBIZZLE_SELF_HEAL_ENABLED", True):
-		logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED")
+		logger.info("Self-heal watchdog disabled via DUBIZZLE_SELF_HEAL_ENABLED/DUBIZZLE_SELF_HEAL_ENABLED")
 		return
 
-	queue_name = os.getenv("IAAI_CELERY_QUEUE", "scraping")
+	queue_name = _env_str("DUBIZZLE_CELERY_QUEUE", "scraping")
-	check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
+	check_interval = max(5, _env_int("DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
-	stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720))
+	stall_seconds = max(180, _env_int("DUBIZZLE_SELF_HEAL_STALL_SECONDS", 720))
-	startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
+	startup_grace = max(30, _env_int("DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
-	restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
+	restart_cooldown = max(60, _env_int("DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
 
 	logger.warning(
 		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
@@ -195,7 +203,7 @@ def main() -> None:
 
 if __name__ == "__main__":
 	logging.basicConfig(
-		level=os.getenv("IAAI_LOG_LEVEL", "INFO"),
+		level=_env_str("DUBIZZLE_LOG_LEVEL", "INFO"),
 		format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
 	)
 	main()

dubizzle_scraper/worker/tasks.py

@@ -1,4 +1,4 @@
-# Задачи Celery для синхронизации автомобилей и листинга IAAI.
+# Задачи Celery для синхронизации автомобилей и листинга Dubizzle.
 
 import json
 import logging
@@ -13,25 +13,39 @@ from celery import shared_task
 from redis import Redis
 
 from ..core.config import Settings, parse_listing_segments
-from ..scraper import IAAIScraper
+from ..scraper import DUBIZZLEScraper
 from ..storage.db import PersistenceService
 from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats
 
-logger = logging.getLogger("iaai_scraper.worker.tasks")
+logger = logging.getLogger("dubizzle_scraper.worker.tasks")
 
-# Минимальная пауза между батчами (секунды) — не давит IAAI.
-_INTER_BATCH_DELAY = max(float(os.getenv("IAAI_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0)
-# Если доля failed в батче превышает порог — прерываем (IAAI блокирует).
-_FAIL_RATE_THRESHOLD = min(max(float(os.getenv("IAAI_FAIL_RATE_THRESHOLD", "0.9")), 0.0), 1.0)
 
-SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+def _env_str(name: str, default: str) -> str:
-SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
+    value = os.getenv(name)
-SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
+    if value is None and name.startswith("DUBIZZLE_"):
+        value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):])
+    return value if value is not None else default
+
+
+def _env_float(name: str, default: float) -> float:
+    try:
+        return float(_env_str(name, str(default)).strip())
+    except Exception:
+        return default
+
+# Минимальная пауза между батчами (секунды).
+_INTER_BATCH_DELAY = max(_env_float("DUBIZZLE_INTER_BATCH_DELAY_SECONDS", 0.3), 0.0)
+# Если доля failed в батче превышает порог — прерываем.
+_FAIL_RATE_THRESHOLD = min(max(_env_float("DUBIZZLE_FAIL_RATE_THRESHOLD", 0.9), 0.0), 1.0)
+
+SYNC_LISTING_LOCK_KEY = "dubizzle:locks:sync_listing"
+SYNC_FULL_SCAN_DONE_KEY = "dubizzle:state:sync_full_scan_done"
+SYNC_LISTING_CHECKPOINT_KEY = "dubizzle:state:sync_listing_checkpoint"
 SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60
-SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak"
+SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_failure_streak"
 SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3
 SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60
-SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "iaai:state:sync_listing_bootstrap_continuation_streak"
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_continuation_streak"
 SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT = max(
     1,
     int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT", "6")),
@@ -40,19 +54,45 @@ SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS = max(
     60,
     int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS", str(6 * 60 * 60))),
 )
-HOURLY_FAILURE_STREAK_KEY = "iaai:state:hourly_failure_streak"
+HOURLY_FAILURE_STREAK_KEY = "dubizzle:state:hourly_failure_streak"
 HOURLY_FAILURE_STREAK_LIMIT = 3
 HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60  # сброс через 6 часов
-SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
+SYNC_LISTING_FOLLOWUP_PENDING_KEY = "dubizzle:state:sync_listing_followup_pending"
-SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task"
+SYNC_LISTING_TASK_NAME = "dubizzle_scraper.worker.tasks.sync_listing_task"
-SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}"
+SYNC_SEGMENT_LOCK_KEY_FMT = "dubizzle:locks:sync_segment:{idx}"
-SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress"
+SYNC_SEGMENTS_PROGRESS_KEY = "dubizzle:state:sync_segments_progress"
-SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total"
+SYNC_SEGMENTS_TOTAL_KEY = "dubizzle:state:sync_segments_total"
+SYNC_SEGMENTED_SCAN_ACTIVE_KEY = "dubizzle:state:sync_segmented_scan_active"
 SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60
-TASK_PROGRESS_KEY_FMT = "iaai:state:task_progress:{task_id}"
+TASK_PROGRESS_KEY_FMT = "dubizzle:state:task_progress:{task_id}"
-GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
+GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts"
-SITEMAP_HOURLY_LAST_COUNT_KEY = "iaai:state:sitemap_hourly_last_count"
+SITEMAP_HOURLY_LAST_COUNT_KEY = "dubizzle:state:sitemap_hourly_last_count"
-SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "iaai:state:sitemap_hourly_refresh_offset"
+SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "dubizzle:state:sitemap_hourly_refresh_offset"
+
+
+def _runtime_key(key: str) -> str:
+    return key.replace("dubizzle:", "dubizzle:")
+
+
+def _origin_prefix() -> str:
+    return _env_str("DUBIZZLE_ORIGIN_PREFIX", "dubizzle:")
+
+
+SYNC_LISTING_LOCK_KEY = _runtime_key(SYNC_LISTING_LOCK_KEY)
+SYNC_FULL_SCAN_DONE_KEY = _runtime_key(SYNC_FULL_SCAN_DONE_KEY)
+SYNC_LISTING_CHECKPOINT_KEY = _runtime_key(SYNC_LISTING_CHECKPOINT_KEY)
+SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)
+SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY)
+HOURLY_FAILURE_STREAK_KEY = _runtime_key(HOURLY_FAILURE_STREAK_KEY)
+SYNC_LISTING_FOLLOWUP_PENDING_KEY = _runtime_key(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
+SYNC_SEGMENT_LOCK_KEY_FMT = _runtime_key(SYNC_SEGMENT_LOCK_KEY_FMT)
+SYNC_SEGMENTS_PROGRESS_KEY = _runtime_key(SYNC_SEGMENTS_PROGRESS_KEY)
+SYNC_SEGMENTS_TOTAL_KEY = _runtime_key(SYNC_SEGMENTS_TOTAL_KEY)
+SYNC_SEGMENTED_SCAN_ACTIVE_KEY = _runtime_key(SYNC_SEGMENTED_SCAN_ACTIVE_KEY)
+TASK_PROGRESS_KEY_FMT = _runtime_key(TASK_PROGRESS_KEY_FMT)
+GLOBAL_PROGRESS_TS_KEY = _runtime_key(GLOBAL_PROGRESS_TS_KEY)
+SITEMAP_HOURLY_LAST_COUNT_KEY = _runtime_key(SITEMAP_HOURLY_LAST_COUNT_KEY)
+SITEMAP_HOURLY_REFRESH_OFFSET_KEY = _runtime_key(SITEMAP_HOURLY_REFRESH_OFFSET_KEY)
 STALL_WATCHDOG_NAVIGATION_STAGES = {
     "listing_next_page_started",
     "listing_resume_progress",
@@ -150,7 +190,16 @@ def _clear_task_progress(redis_client: Redis, task_id: str) -> None:
         logger.warning("Failed to clear task progress for %s", task_id, exc_info=True)
 
 
-def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int:
+def _stall_timeout_for_progress(
+    stage: str | None,
+    default_timeout: int,
+    *,
+    heartbeat_only: bool = False,
+) -> int:
+    if heartbeat_only:
+        # Heartbeat-пульсы не считаем полноценным прогрессом:
+        # при долгом зависании даём только ограниченный grace-период.
+        return min(int(default_timeout), 120)
     if stage in STALL_WATCHDOG_NAVIGATION_STAGES:
         return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS)
     return int(default_timeout)
@@ -165,12 +214,12 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool |
     discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings)
     discovered_urls = discovery_result.vehicle_urls
     active_urls = set(discovered_urls)
-    existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:")
+    existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix())
 
     new_urls = [url for url in discovered_urls if url not in existing_urls]
     sold_count = 0
     if active_urls:
-        sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai")
+        sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":"))
 
     cars_upserted = 0
     cars_failed = 0
@@ -179,7 +228,7 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool |
     failures: list[dict[str, str]] = []
 
     if new_urls:
-        with IAAIScraper(settings) as scraper:
+        with DUBIZZLEScraper(settings) as scraper:
             if progress_callback:
                 scraper.set_progress_callback(progress_callback)
             batch_size = settings.celery.batch_size
@@ -194,7 +243,7 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool |
                 protection_events += batch_protection
                 images_upserted += int(batch_result.get("images_upserted", 0))
                 failures.extend(batch_result.get("failures", []))
-                # Fail-fast: если слишком много ошибок — IAAI блокирует, не тратим ресурсы.
+                # Fail-fast: если слишком много ошибок — DUBIZZLE блокирует, не тратим ресурсы.
                 total_in_batch = batch_ok + batch_fail
                 if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD:
                     logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch)
@@ -233,7 +282,7 @@ def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new:
     active_urls = set(discovered_urls)
     sold_count = 0
     if active_urls:
-        sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai")
+        sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":"))
 
     cars_upserted = 0
     cars_failed = 0
@@ -241,7 +290,7 @@ def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new:
     images_upserted = 0
     failures: list[dict[str, str]] = []
 
-    with IAAIScraper(settings) as scraper:
+    with DUBIZZLEScraper(settings) as scraper:
         if progress_callback:
             scraper.set_progress_callback(progress_callback)
         batch_size = settings.celery.batch_size
@@ -301,12 +350,12 @@ def _hourly_sitemap_rolling_refresh_sync(
     active_urls = set(discovered_urls)
     sold_count = 0
     if active_urls:
-        sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai")
+        sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":"))
 
-    existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:")
+    existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix())
     new_urls = [url for url in discovered_urls if url not in existing_urls]
 
-    total_active = persistence.count_active_cars_for_lane("iaai:")
+    total_active = persistence.count_active_cars_for_lane(_origin_prefix())
     batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size))
     try:
         offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0)
@@ -314,14 +363,14 @@ def _hourly_sitemap_rolling_refresh_sync(
         offset = 0
 
     refresh_urls = persistence.get_active_origin_urls_batch_for_refresh(
-        prefix="iaai:",
+        prefix=_origin_prefix(),
         offset=offset,
         limit=batch_size,
     )
     if not refresh_urls and total_active > 0:
         offset = 0
         refresh_urls = persistence.get_active_origin_urls_batch_for_refresh(
-            prefix="iaai:",
+            prefix=_origin_prefix(),
             offset=0,
             limit=batch_size,
         )
@@ -352,7 +401,7 @@ def _hourly_sitemap_rolling_refresh_sync(
     failures: list[dict[str, str]] = []
 
     if target_urls:
-        with IAAIScraper(settings) as scraper:
+        with DUBIZZLEScraper(settings) as scraper:
             if progress_callback:
                 scraper.set_progress_callback(progress_callback)
             worker_batch_size = settings.celery.batch_size
@@ -438,12 +487,19 @@ def _start_stall_watchdog(
                     no_data_count = 0
                     data = json.loads(raw)
                     stage = data.get("stage")
+                    heartbeat_only = bool(data.get("heartbeat_only"))
                     last_ts = int(data.get("ts") or 0)
                     if not last_ts:
                         continue
-                    effective_stall_timeout = _stall_timeout_for_progress(stage, stall_timeout_seconds)
+                    effective_stall_timeout = _stall_timeout_for_progress(
+                        stage,
+                        stall_timeout_seconds,
+                        heartbeat_only=heartbeat_only,
+                    )
                     age = int(time.time()) - last_ts
                     if age < effective_stall_timeout:
+                        # Heartbeat-пульс не должен бесконечно продлевать дедлайн.
+                        if not heartbeat_only:
                             watchdog_born = time.monotonic()  # reset absolute deadline on real progress
                         continue
                     logger.error(
@@ -620,6 +676,15 @@ def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> N
 
 
 def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool:
+    return _has_running_task_named(celery_app, SYNC_LISTING_TASK_NAME, exclude_task_id=exclude_task_id)
+
+
+def _has_running_task_named(
+    celery_app,
+    task_name: str,
+    *,
+    exclude_task_id: str | None = None,
+) -> bool:
     try:
         inspector = celery_app.control.inspect(timeout=1.0)
         snapshots = [
@@ -634,8 +699,8 @@ def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None =
     for snapshot in snapshots:
         for entries in snapshot.values():
             for entry in entries or []:
-                task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
+                entry_task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
-                if task_name != SYNC_LISTING_TASK_NAME:
+                if entry_task_name != task_name:
                     continue
                 # Исключаем текущую задачу — она не считается "другой запущенной"
                 entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "")
@@ -645,6 +710,59 @@ def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None =
     return False
 
 
+def _has_running_sync_segment_tasks(celery_app) -> bool:
+    return _has_running_task_named(celery_app, "dubizzle_scraper.worker.tasks.sync_segment_task")
+
+
+def _set_segmented_scan_active(redis_client: Redis, *, total: int) -> None:
+    try:
+        payload = {
+            "ts": int(time.time()),
+            "total": int(total),
+        }
+        redis_client.set(
+            SYNC_SEGMENTED_SCAN_ACTIVE_KEY,
+            json.dumps(payload, ensure_ascii=False),
+            ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS,
+        )
+    except Exception:
+        logger.warning("Failed to persist segmented scan active state", exc_info=True)
+
+
+def _clear_segmented_scan_active(redis_client: Redis) -> None:
+    try:
+        redis_client.delete(SYNC_SEGMENTED_SCAN_ACTIVE_KEY)
+    except Exception:
+        logger.warning("Failed to clear segmented scan active state", exc_info=True)
+
+
+def _is_segmented_scan_in_progress(redis_client: Redis, celery_app) -> bool:
+    try:
+        marker = redis_client.get(SYNC_SEGMENTED_SCAN_ACTIVE_KEY)
+    except Exception:
+        logger.warning("Failed to read segmented scan active state", exc_info=True)
+        return False
+
+    if not marker:
+        return False
+
+    if _has_running_sync_segment_tasks(celery_app):
+        return True
+
+    try:
+        queue_len = int(redis_client.llen("scraping") or 0)
+    except Exception:
+        logger.warning("Failed to inspect scraping queue length", exc_info=True)
+        queue_len = 0
+
+    if queue_len > 0:
+        return True
+
+    logger.warning("Clearing stale segmented scan active state: no running segment tasks detected")
+    _clear_segmented_scan_active(redis_client)
+    return False
+
+
 def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool:
     try:
         owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY)
@@ -902,7 +1020,7 @@ def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[in
 
 
 @shared_task(
-    name="iaai_scraper.worker.tasks.sync_segment_task",
+    name="dubizzle_scraper.worker.tasks.sync_segment_task",
     bind=True,
     max_retries=2,
     default_retry_delay=60,
@@ -912,7 +1030,7 @@ def sync_segment_task(
     self,
     segment_index: int,
     segment: dict,
-    lane: str = "iaai_cars",
+    lane: str = "dubizzle_cars",
     only_new: bool | None = None,
     is_bootstrap: bool = False,
 ):
@@ -965,7 +1083,7 @@ def sync_segment_task(
 
     try:
         def _job():
-            with IAAIScraper() as scraper:
+            with DUBIZZLEScraper() as scraper:
                 scraper.set_progress_callback(
                     lambda stage, meta: _update_task_progress(
                         redis_client,
@@ -1011,9 +1129,15 @@ def sync_segment_task(
                 segment_index, seg_label, completed, total,
             )
             if total > 0 and completed >= total:
-                _set_full_scan_done(redis_client, True)
+                always_full_scan = bool(Settings().discovery.always_full_scan)
+                _set_full_scan_done(redis_client, False if always_full_scan else True)
                 _clear_sync_checkpoint(redis_client)
+                _clear_segmented_scan_active(redis_client)
                 _clear_bootstrap_failure_streak(redis_client)
+                _clear_bootstrap_continuation_streak(redis_client)
+                if always_full_scan:
+                    logger.warning("All %d segments completed; next hourly run will restart from segment 0", total)
+                else:
                     logger.warning("All %d segments completed; bootstrap full scan done", total)
 
         return {
@@ -1075,20 +1199,20 @@ def sync_segment_task(
 
 
 @shared_task(
-    name="iaai_scraper.worker.tasks.sync_vehicle_task",
+    name="dubizzle_scraper.worker.tasks.sync_vehicle_task",
     bind=True,
     max_retries=2,
     default_retry_delay=30,
     acks_late=True,
 )
-def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
+def sync_vehicle_task(self, vehicle_url: str, lane: str = "dubizzle"):
     # Скрапинг и upsert одного автомобиля.
     persistence = _get_persistence()
     persistence.create_tables()
 
     try:
         def _job():
-            with IAAIScraper() as scraper:
+            with DUBIZZLEScraper() as scraper:
                 return scraper.sync_vehicle(vehicle_url, lane=lane)
 
         result = _run_browser_job(_job)
@@ -1106,7 +1230,7 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
 
 
 @shared_task(
-    name="iaai_scraper.worker.tasks.sync_listing_task",
+    name="dubizzle_scraper.worker.tasks.sync_listing_task",
     bind=True,
     max_retries=3,
     default_retry_delay=120,
@@ -1116,7 +1240,7 @@ def sync_listing_task(
     self,
     make: str | None = None,
     model: str | None = None,
-    lane: str = "iaai_cars",
+    lane: str = "dubizzle_cars",
     limit: int | None = None,
     only_new: bool | None = None,
 ):
@@ -1159,11 +1283,12 @@ def sync_listing_task(
         continuation_streak, should_continue = _bump_bootstrap_continuation_streak(redis_client)
         if not should_continue:
             _clear_followup_pending(redis_client)
-            # Переключаемся на часовой beat-режим и останавливаем
+            # Останавливаем немедленные bootstrap continuation, чтобы не зациклиться.
-            # немедленные bootstrap continuation, чтобы не зациклиться.
+            # Фиксируем done + чистим checkpoint даже при always_full_scan:
-            if not always_full_scan:
+            # это безопасно и предотвращает повторный старт со stale-сегмента.
             _set_full_scan_done(redis_client, True)
             _clear_sync_checkpoint(redis_client)
+            if not always_full_scan:
                 logger.error(
                     "Bootstrap continuation stopped after %d immediate runs; switching to hourly schedule",
                     continuation_streak,
@@ -1178,7 +1303,7 @@ def sync_listing_task(
         try:
             followup_expires = max(int(lock_ttl), int(delay_seconds) + 300)
             self.app.send_task(
-                "iaai_scraper.worker.tasks.sync_listing_task",
+                "dubizzle_scraper.worker.tasks.sync_listing_task",
                 kwargs={
                     "make": make,
                     "model": model,
@@ -1247,17 +1372,15 @@ def sync_listing_task(
         )
 
         full_scan_done_before_run = _is_full_scan_done(redis_client)
+        hourly_mode = settings.discovery.hourly_mode.strip().lower()
+        discovery_mode = settings.discovery.mode.strip().lower()
         always_full_scan = bool(settings.discovery.always_full_scan)
         force_bootstrap_full_scan = always_full_scan or (not full_scan_done_before_run)
         effective_limit = None if force_bootstrap_full_scan else limit
         effective_only_new = False if force_bootstrap_full_scan else only_new
-        hourly_mode = settings.discovery.hourly_mode.strip().lower()
-        discovery_mode = settings.discovery.mode.strip().lower()
-        if always_full_scan:
-            # Для режима "полный прогон каждый запуск" приоритет — устойчивый resume,
-            # поэтому принудительно уходим в listing/segmented path вместо sitemap-mainline.
-            discovery_mode = "listing"
         prefer_sitemap_mainline = (
+            discovery_mode == "sitemap"
+            and
             not force_bootstrap_full_scan
             and make is None
             and model is None
@@ -1268,22 +1391,23 @@ def sync_listing_task(
         use_hourly_sitemap_sync = (not always_full_scan) and full_scan_done_before_run and prefer_sitemap_mainline
 
         # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
-        # Используется только во время bootstrap для пропуска уже обработанных сегментов.
+        # Используется во время bootstrap/full-scan resume.
-        # Никаких page-level resume — внутри сегмента всегда стартуем с page 1.
         last_completed_segment: int | None = None
-        if force_bootstrap_full_scan:
+        resume_from_checkpoint = force_bootstrap_full_scan and (
+            always_full_scan or (not full_scan_done_before_run)
+        )
+        if resume_from_checkpoint:
             last_completed_segment = _load_last_completed_segment(redis_client)
         else:
-            # После завершения bootstrap чекпоинт не нужен никогда.
             _clear_sync_checkpoint(redis_client)
 
         if force_bootstrap_full_scan:
             logger.info(
                 "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run",
             )
-            if always_full_scan:
+            if discovery_mode == "algolia":
                 logger.info(
-                    "Always full scan mode enabled (IAAI_ALWAYS_FULL_SCAN=true): using listing resume path",
+                    "Algolia full scan enabled: using segmented Algolia traversal when configured",
                 )
 
         logger.info(
@@ -1413,28 +1537,31 @@ def sync_listing_task(
 
         self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
 
-        # Определяем сегменты из конфига.
         segments = parse_listing_segments(settings.listing.listing_segments_json)
         use_segmented = (
             bool(segments)
             and make is None
             and model is None
             and not prefer_sitemap_mainline
-            and discovery_mode != "sitemap"
+            and discovery_mode in {"listing", "algolia"}
         )
 
-        if prefer_sitemap_mainline:
+        if prefer_sitemap_mainline and segments:
-            if discovery_mode != "sitemap":
-                logger.warning(
-                    "Unfiltered full scan forcing sitemap discovery despite IAAI_DISCOVERY_MODE=%s",
-                    discovery_mode or "unset",
-                )
-            if segments:
             logger.info("Ignoring configured listing segments for unfiltered sitemap full scan")
 
-        # --- Параллельный диспатч сегментов: dispatch & exit ---
         if use_segmented and settings.celery.parallel_segments:
-            # Сегменты уже завершённые (для bootstrap resume) пропускаем по Redis SET.
+            segmented_scan_in_progress = False
+            if force_bootstrap_full_scan:
+                segmented_scan_in_progress = _is_segmented_scan_in_progress(redis_client, self.app)
+                if segmented_scan_in_progress:
+                    logger.warning("Parallel segments: scan already in progress, skipping duplicate dispatch")
+                    return {
+                        "status": "skipped",
+                        "reason": "segmented_scan_in_progress",
+                        "task_id": task_id,
+                        "mode": "parallel_segments",
+                    }
+
             already_completed: set[int] = set()
             if force_bootstrap_full_scan:
                 try:
@@ -1443,17 +1570,20 @@ def sync_listing_task(
                 except Exception:
                     already_completed = set()
 
+                if always_full_scan:
+                    already_completed = set()
+
             pending = [
                 (idx, seg) for idx, seg in enumerate(segments)
                 if idx not in already_completed
             ]
 
             if not pending:
-                # Всё уже сделано — фиксируем bootstrap done.
                 if force_bootstrap_full_scan:
                     _set_full_scan_done(redis_client, True)
                     _clear_sync_checkpoint(redis_client)
                     _clear_bootstrap_failure_streak(redis_client)
+                    _clear_segmented_scan_active(redis_client)
                 logger.warning("Parallel segments: nothing to dispatch (all completed)")
                 return {
                     "status": "success",
@@ -1464,15 +1594,18 @@ def sync_listing_task(
                     "segments_already_completed": len(already_completed),
                 }
 
-            # При первом запуске bootstrap фиксируем total, чтобы знать когда остановиться.
+            if force_bootstrap_full_scan and (always_full_scan or not already_completed):
-            if force_bootstrap_full_scan and not already_completed:
                 _reset_segments_progress(redis_client, len(segments))
+                already_completed = set()
+
+            if force_bootstrap_full_scan:
+                _set_segmented_scan_active(redis_client, total=len(segments))
 
             dispatched = 0
             for idx, seg in pending:
                 try:
                     self.app.send_task(
-                        "iaai_scraper.worker.tasks.sync_segment_task",
+                        "dubizzle_scraper.worker.tasks.sync_segment_task",
                         kwargs={
                             "segment_index": idx,
                             "segment": seg,
@@ -1498,13 +1631,11 @@ def sync_listing_task(
                 "segments_dispatched": dispatched,
                 "segments_already_completed": len(already_completed),
             }
-        # --- конец параллельной ветки ---
 
         resume_from_segment = 0
         if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None:
             resume_from_segment = max(0, last_completed_segment + 1)
             if resume_from_segment >= len(segments):
-                # Все сегменты уже пройдены — чекпоинт устарел, начинаем заново.
                 logger.info(
                     "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0",
                     last_completed_segment, len(segments),
@@ -1527,7 +1658,7 @@ def sync_listing_task(
             )
 
         def _job():
-            with IAAIScraper() as scraper:
+            with DUBIZZLEScraper() as scraper:
                 scraper.set_progress_callback(_progress_cb_main)
                 if use_segmented:
                     return scraper.sync_listing_segmented(
@@ -1538,7 +1669,7 @@ def sync_listing_task(
                         start_page=1,
                         progress_callback=(
                             (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx))
-                            if force_bootstrap_full_scan else None
+                            if resume_from_checkpoint else None
                         ),
                     )
                 return scraper.sync_listing(
@@ -1654,7 +1785,7 @@ def sync_listing_task(
                 if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl):
                     try:
                         self.app.send_task(
-                            "iaai_scraper.worker.tasks.sync_listing_task",
+                                "dubizzle_scraper.worker.tasks.sync_listing_task",
                             kwargs={
                                 "make": make,
                                 "model": model,

entrypoint.sh

@@ -4,7 +4,7 @@ set -euo pipefail
 is_worker_command() {
   local joined="$*"
   case "$joined" in
-    *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
+    *"celery -A dubizzle_scraper.worker.celery_app worker"*|*" celery -A dubizzle_scraper.worker.celery_app worker"*)
       return 0
       ;;
   esac
@@ -36,19 +36,19 @@ start_proxy_bridge_if_needed() {
 
   echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
 
-  if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
+  if [ -z "${DUBIZZLE_PROXY_SERVER:-}" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+    export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899"
-  elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
+  elif [ "${DUBIZZLE_PROXY_SERVER}" = "http://localhost:8899" ]; then
-    export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
+    export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899"
   fi
 
-  echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
+  echo "[entrypoint] Using browser proxy: ${DUBIZZLE_PROXY_SERVER}"
-  python -m iaai_scraper.proxy_bridge &
+  python -m dubizzle_scraper.proxy_bridge &
   BRIDGE_PID=$!
   sleep 1
 
   if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
-    echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
+    echo "[entrypoint] ERROR: dubizzle_scraper.proxy_bridge failed to start"
     exit 1
   fi
 
@@ -65,13 +65,13 @@ start_self_heal_watchdog_if_worker() {
   # Удаляем его перед запуском worker-процесса.
   rm -f /tmp/celery-worker.pid
 
-  if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then
+  if [ "${DUBIZZLE_SELF_HEAL_ENABLED:-true}" = "false" ]; then
     echo "[entrypoint] Self-heal watchdog disabled"
     return 0
   fi
 
   echo "[entrypoint] Starting self-heal watchdog for worker..."
-  python -m iaai_scraper.worker.self_heal &
+  python -m dubizzle_scraper.worker.self_heal &
   SELF_HEAL_PID=$!
   sleep 1
 

iaai_scraper/core/config.py

@@ -1,308 +0,0 @@
-import json
-import os
-from dataclasses import dataclass, field
-from pathlib import Path
-
-from dotenv import load_dotenv
-
-load_dotenv()
-
-
-TRUE_VALUES = {"1", "true", "yes", "on"}
-
-
-#  Хелперы для чтения env-переменных с приведением типов 
-
-def _env_str(name: str, default: str) -> str:
-    value = os.getenv(name)
-    return value if value is not None else default
-
-
-def _env_optional_str(name: str) -> str | None:
-    value = os.getenv(name)
-    if value is None:
-        return None
-    value = value.strip()
-    return value or None
-
-
-def _env_path_str(name: str) -> str | None:
-    value = _env_optional_str(name)
-    if value is None:
-        return None
-    return str(Path(value).expanduser())
-
-
-def _env_bool(name: str, default: bool) -> bool:
-    fallback = "true" if default else "false"
-    return _env_str(name, fallback).strip().lower() in TRUE_VALUES
-
-
-def _env_int(name: str, default: int) -> int:
-    return int(_env_str(name, str(default)).strip())
-
-
-def _env_float(name: str, default: float) -> float:
-    return float(_env_str(name, str(default)).strip())
-
-
-#  Конфиг браузерного отпечатка (User-Agent, viewport, timezone) 
-
-@dataclass(slots=True)
-class FingerprintConfig:
-    user_agent: str = (
-        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
-        "AppleWebKit/537.36 (KHTML, like Gecko) "
-        "Chrome/135.0.0.0 Safari/537.36"
-    )
-    viewport_presets: tuple[dict[str, int], ...] = (
-        {"width": 1920, "height": 1080},
-        {"width": 1600, "height": 900},
-        {"width": 1536, "height": 864},
-        {"width": 1440, "height": 900},
-        {"width": 1366, "height": 768},
-    )
-    timezone_candidates: tuple[str, ...] = (
-        "America/New_York",
-        "America/Chicago",
-        "America/Los_Angeles",
-    )
-    locale: str = "en-US"
-    sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
-
-
-#  Конфиг перехвата сетевых запросов (лимиты на кол-во) 
-
-@dataclass(slots=True)
-class CaptureConfig:
-    capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
-    max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
-    max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
-
-
-#  Конфиг пауз между действиями (имитация человека) 
-
-@dataclass(slots=True)
-class HumanPaceConfig:
-    enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
-    after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
-    after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
-    after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
-    after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
-    before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
-    before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
-    after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
-    after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
-    between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
-    between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
-    after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
-    after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
-
-
-#  Конфиг сбора листинга (URL, лимиты страниц и машин) 
-
-@dataclass(slots=True)
-class ListingConfig:
-    cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
-    max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
-    max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
-    page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
-    include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
-    collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
-    # Порог раннего останова: если доля уже известных машин на странице >= этого значения,
-    # прекращаем листать — все новые машины уже найдены. 0 = отключено.
-    early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
-    # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
-    # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
-    # Пустая строка = без сегментации (backward compatible).
-    listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
-
-
-# Список брендов IAAI для автоматической сегментации.
-# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
-IAAI_DEFAULT_MAKES: tuple[str, ...] = (
-    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
-    "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
-    "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
-    "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
-    "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
-    "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
-    "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
-    "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
-    "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
-    "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
-    "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
-)
-
-# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
-IAAI_PAGINATION_CEILING = 22_600
-
-# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
-_LARGE_MAKES: frozenset[str] = frozenset({
-    "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
-    "KIA", "DODGE", "JEEP",
-})
-_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
-    (1900, 2012),
-    (2013, 2019),
-    (2020, 2027),
-)
-
-
-def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
-    """Парсит IAAI_LISTING_SEGMENTS в список сегментов.
-
-    Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
-    Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
-    Крупные бренды автоматически разбиваются по диапазонам годов.
-    """
-    raw = raw.strip()
-    if not raw:
-        return []
-    if raw.lower() == "auto":
-        segments: list[dict[str, str | int | None]] = []
-        for m in IAAI_DEFAULT_MAKES:
-            if m in _LARGE_MAKES:
-                for yr_min, yr_max in _YEAR_SPLITS:
-                    segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
-            else:
-                segments.append({"make": m, "year_min": None, "year_max": None})
-        return segments
-    try:
-        data = json.loads(raw)
-    except (json.JSONDecodeError, ValueError):
-        return []
-    if not isinstance(data, list):
-        return []
-    segments = []
-    for item in data:
-        if isinstance(item, str):
-            segments.append({"make": item.upper(), "year_min": None, "year_max": None})
-        elif isinstance(item, dict):
-            segments.append({
-                "make": str(item.get("make") or "").upper() or None,
-                "year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
-                "year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
-            })
-    return segments
-
-
-# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) 
-
-@dataclass(slots=True)
-class DatabaseConfig:
-    url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
-    echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
-    pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
-    max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
-    pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
-    auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
-
-
-# --- Конфиг Redis (URL для Celery broker) ---
-
-@dataclass(slots=True)
-class RedisConfig:
-    url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
-    socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
-    socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
-    health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
-
-
-# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
-
-@dataclass(slots=True)
-class DiscoveryConfig:
-    mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
-    hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
-    hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
-    always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", True)
-
-
-# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
-
-@dataclass(slots=True)
-class CeleryConfig:
-    broker_url: str = _env_str("CELERY_BROKER_URL", "")
-    result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
-    task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
-    task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
-    task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
-    worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
-    worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
-    broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
-    beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
-    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
-    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
-    parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
-    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
-    block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
-
-
-# --- Конфиг прокси (server, username, password) ---
-
-@dataclass(slots=True)
-class ProxyConfig:
-    server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
-    username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
-    password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
-
-    @property
-    def enabled(self) -> bool:
-        return bool(self.server)
-
-    def to_playwright_dict(self) -> dict[str, str] | None:
-        if not self.server:
-            return None
-        result: dict[str, str] = {"server": self.server}
-        if self.username:
-            result["username"] = self.username
-        if self.password:
-            result["password"] = self.password
-        return result
-
-
-#  Главный объект настроек: собирает все блоки конфигурации 
-
-@dataclass(slots=True)
-class Settings:
-    home_url: str = "https://www.iaai.com/"
-    default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
-    network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
-    fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
-    fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
-    fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
-    max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
-    retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
-    retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
-    retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
-    headless: bool = _env_bool("IAAI_HEADLESS", True)
-    browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
-    log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
-    log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
-    enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
-    sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
-    raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
-    tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
-    runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
-    scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
-    fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
-    capture: CaptureConfig = field(default_factory=CaptureConfig)
-    pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
-    listing: ListingConfig = field(default_factory=ListingConfig)
-    database: DatabaseConfig = field(default_factory=DatabaseConfig)
-    redis: RedisConfig = field(default_factory=RedisConfig)
-    celery: CeleryConfig = field(default_factory=CeleryConfig)
-    proxy: ProxyConfig = field(default_factory=ProxyConfig)
-    discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
-
-    @property
-    def parallel_tabs(self) -> int:
-        return self.celery.parallel_tabs
-
-    @property
-    def block_resources(self) -> bool:
-        return self.celery.block_resources
-
-# Глобальный синглтон — используется по умолчанию во всех модулях.
-settings = Settings()

pyproject.toml

@@ -3,9 +3,9 @@ requires = ["setuptools>=68", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
-name = "iaai-scraper"
+name = "dubizzle-scraper"
 version = "0.1.0"
-description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
+description = "Dubizzle scraper service with FastAPI, Celery, Playwright and PostgreSQL"
 readme = "README.md"
 requires-python = ">=3.11"
 dependencies = [
@@ -29,13 +29,13 @@ dev = [
 ]
 
 [project.scripts]
-iaai = "iaai_scraper.cli:main"
+dubizzle = "dubizzle_scraper.cli:main"
 
 [tool.setuptools]
 include-package-data = true
 
 [tool.setuptools.packages.find]
-include = ["iaai_scraper*"]
+include = ["dubizzle_scraper*"]
 
 [tool.pytest.ini_options]
 addopts = "-q --disable-warnings"

runtime_config.json

@@ -5,7 +5,7 @@
     "ids_next_size": null,
     "ids_max_pages": null,
     "condition_check_enabled": false,
-    "lane": "iaai_cars",
+    "lane": "dubizzle_cars",
     "only_new": false,
     "limit": null
   },

tests/test_db.py

@@ -6,10 +6,10 @@ from pathlib import Path
 
 from sqlalchemy import select
 
-from iaai_scraper.core.config import Settings
+from dubizzle_scraper.core.config import Settings
-from iaai_scraper.storage.db import PersistenceService
+from dubizzle_scraper.storage.db import PersistenceService
-from iaai_scraper.storage.models import Car, Image, SyncRun
+from dubizzle_scraper.storage.models import Car, Image, SyncRun
-from iaai_scraper.storage.schemas import CarRecord, ImageRecord
+from dubizzle_scraper.storage.schemas import CarRecord, ImageRecord
 
 
 class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
     @staticmethod
     def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
         return CarRecord(
-            parser_id=f"iaai:{origin_id}",
+            parser_id=f"dubizzle:{origin_id}",
             brand="Toyota",
             model="Camry",
             year=2014,
             price=price,
-            origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
+            origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
             origin_id=origin_id,
             slug=f"toyota-camry-{origin_id}",
             images=[
                 ImageRecord(
-                    fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+                    fullres_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
-                    preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
+                    preview_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=400&height=300",
                     order_index=0,
                 )
             ],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
         second = self._record("888")
         second.images = [
             ImageRecord(
-                fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
+                fullres_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=845&height=633",
-                preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+                preview_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
                 order_index=0,
             )
         ]
@@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
 
     def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
         first = self._record("OLD-ID")
-        first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        first.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
         self.persistence.upsert_car(first)
 
         second = self._record("NEW-ID")
-        second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        second.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
         result = self.persistence.upsert_car(second)
 
         self.assertEqual(result["action"], "updated")

tests/test_listing.py

@@ -2,9 +2,9 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.browser.pace import HumanPacer
+from dubizzle_scraper.browser.pace import HumanPacer
-from iaai_scraper.core.config import Settings
+from dubizzle_scraper.core.config import Settings
-from iaai_scraper.browser.listing import ListingCollector
+from dubizzle_scraper.browser.listing import ListingCollector
 
 
 class _FakePage:
@@ -58,8 +58,8 @@ class TestListingUnit(unittest.TestCase):
         self.assertEqual(
             links,
             [
-                ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
+                ("https://www.dubizzle.com/VehicleDetail/45184893~US", "45184893"),
-                ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
+                ("https://www.dubizzle.com/VehicleDetail/45171480~US", "45171480"),
             ],
         )
 

tests/test_mappers.py

@@ -2,22 +2,109 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.parsing.mapper import CarMapper
+from dubizzle_scraper.parsing.mapper import CarMapper
 
 
 class TestCarMapper(unittest.TestCase):
     def setUp(self) -> None:
         self.mapper = CarMapper()
 
+    def test_extracts_make_model_from_algolia_details(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://dubizzle.com/s/DOBI7J3",
+            vehicle_summary={
+                "id": 16810399,
+                "details_v2": {
+                    "tertiary": [
+                        {"slug": "make", "value": {"en": "Toyota", "ar": "تويوتا"}},
+                        {"slug": "model", "value": {"en": "Land Cruiser", "ar": "لاند كروزر"}},
+                    ]
+                },
+                "category_v2": {
+                    "names_en": ["Motors", "Used Cars", "Toyota", "Land Cruiser"],
+                    "slug_paths": [
+                        "motors",
+                        "motors/used-cars",
+                        "motors/used-cars/toyota",
+                        "motors/used-cars/toyota/land-cruiser",
+                    ],
+                },
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.brand, "Toyota")
+        self.assertEqual(record.model, "Land Cruiser")
+
+    def test_extracts_make_model_from_legacy_details(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://dubizzle.com/s/DOBIlegacy",
+            vehicle_summary={
+                "objectID": "item:legacy:1",
+                "details": {
+                    "Make": {"en": {"label": "Make", "value": "Honda"}},
+                    "Model": {"en": {"label": "Model", "value": "Civic"}},
+                },
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.brand, "Honda")
+        self.assertEqual(record.model, "Civic")
+
+    def test_extracts_extended_algolia_fields(self) -> None:
+        record = self.mapper.map_to_car_record(
+            vehicle_url="https://dubizzle.com/s/DOBGaGs",
+            vehicle_summary={
+                "id": 16345114,
+                "year": 2012,
+                "price": 59000.0,
+                "kilometers": 91800,
+                "seller_type": "DL",
+                "site": {"en": "Dubai"},
+                "category_v2": {
+                    "names_en": ["Motors", "Used Cars", "Porsche", "Cayenne"],
+                    "slug_paths": [
+                        "motors",
+                        "motors/used-cars",
+                        "motors/used-cars/porsche",
+                        "motors/used-cars/porsche/cayenne",
+                    ],
+                },
+                "details": {
+                    "Body Type": {"en": {"value": "SUV"}},
+                    "Transmission Type": {"en": {"value": "Automatic Transmission"}},
+                    "Exterior Color": {"en": {"value": "Brown"}},
+                    "Steering Side": {"en": {"value": "Left Hand"}},
+                    "Engine Capacity (cc)": {"en": {"value": "4800 cc"}},
+                    "Make": {"en": {"value": "Porsche"}},
+                    "Model": {"en": {"value": "Cayenne"}},
+                },
+                "photo_mains": ["https://dbz-images.dubizzle.com/images/example.jpeg?impolicy=dpv"],
+            },
+            payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
+        )
+
+        self.assertEqual(record.brand, "Porsche")
+        self.assertEqual(record.model, "Cayenne")
+        self.assertEqual(record.price, 59000)
+        self.assertEqual(record.mileage, 91800)
+        self.assertEqual(record.color, "brown")
+        self.assertEqual(record.body_type, "SUV")
+        self.assertEqual(record.gearbox, "AT")
+        self.assertEqual(record.steering_wheel, "LEFT")
+        self.assertEqual(record.engine_volume, 4800)
+        self.assertEqual(record.country, "AE")
+
     def test_deduplicates_images_by_image_key(self) -> None:
         urls = [
-            "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
+            "https://vis.dubizzle.com/resizer?imageKeys=1&width=200&height=150",
-            "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
+            "https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
-            "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
+            "https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
         ]
 
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
@@ -28,7 +115,7 @@ class TestCarMapper(unittest.TestCase):
 
     def test_no_damage_marker_is_not_damaged(self) -> None:
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
             vehicle_summary={"make": "Ford", "model": "Focus"},
             payload_insights={
                 "vehicle_core": {},
@@ -43,7 +130,7 @@ class TestCarMapper(unittest.TestCase):
 
     def test_unknown_empty_values_and_normalization(self) -> None:
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/999~US",
             vehicle_summary={"make": "  ", "model": None, "drive": "???", "gearbox": "unknown"},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
@@ -54,7 +141,7 @@ class TestCarMapper(unittest.TestCase):
 
         # Нормализация регистра и пробелов.
         record2 = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/888~US",
             vehicle_summary={"make": "Honda", "model": "Civic", "drive": "  Front Wheel Drive  ", "gearbox": " AUTOMATIC "},
             payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
         )
@@ -63,7 +150,7 @@ class TestCarMapper(unittest.TestCase):
 
     def test_price_and_currency_parsing(self) -> None:
         record = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/777~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
             payload_insights={
                 "vehicle_core": {},
@@ -76,7 +163,7 @@ class TestCarMapper(unittest.TestCase):
         self.assertEqual(record.price, 5200)
 
         record2 = self.mapper.map_to_car_record(
-            vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
+            vehicle_url="https://www.dubizzle.com/VehicleDetail/778~US",
             vehicle_summary={"make": "Toyota", "model": "Corolla"},
             payload_insights={
                 "vehicle_core": {},

tests/test_parser.py

@@ -2,7 +2,7 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.parsing.parser import VehicleParser
+from dubizzle_scraper.parsing.parser import VehicleParser
 
 
 class TestVehicleParserUnit(unittest.TestCase):
@@ -29,11 +29,11 @@ class TestVehicleParserUnit(unittest.TestCase):
         self.assertEqual(parsed["model"], "CAMRY")
 
     def test_extract_image_urls_filters_and_deduplicates(self) -> None:
-        vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
+        vehicle_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
         payloads = [{"imageUrls": [
-            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-            "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
+            "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
-            "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
+            "https://vis.dubizzle.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
         ]}]
         urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
         self.assertEqual(len(urls), 1)

tests/test_resilience.py

@@ -4,17 +4,17 @@ import unittest
 from types import SimpleNamespace
 from unittest.mock import MagicMock, patch
 
-from iaai_scraper.scraper import IAAIScraper
+from dubizzle_scraper.scraper import DUBIZZLEScraper
-from iaai_scraper.core.config import Settings
+from dubizzle_scraper.core.config import Settings
-from iaai_scraper.worker import tasks
+from dubizzle_scraper.worker import tasks
 
 
 class TestResilience(unittest.TestCase):
-	def _make_scraper(self) -> IAAIScraper:
+	def _make_scraper(self) -> DUBIZZLEScraper:
 		s = Settings()
 		s.log_level = "CRITICAL"
 		s.database.url = "sqlite://"
-		return IAAIScraper(s)
+		return DUBIZZLEScraper(s)
 
 	def test_update_task_progress_updates_global_marker(self) -> None:
 		redis_client = MagicMock()
@@ -60,11 +60,11 @@ class TestResilience(unittest.TestCase):
 		result = scraper._sync_listing_streaming(
 			make=None,
 			model=None,
-			lane="iaai_cars",
+			lane="dubizzle_cars",
 			limit=None,
 			effective_only_new=False,
 			started_at=0.0,
-			listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
+			listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST",
 		)
 
 		self.assertEqual(result["total"], 0)

tests/test_scraper.py

@@ -5,29 +5,29 @@ from concurrent.futures import TimeoutError as FuturesTimeoutError
 from types import SimpleNamespace
 from unittest.mock import MagicMock, patch
 
-from iaai_scraper.core.config import Settings
+from dubizzle_scraper.core.config import Settings
-from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
+from dubizzle_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
-from iaai_scraper.scraper import IAAIScraper
+from dubizzle_scraper.scraper import DUBIZZLEScraper
-from iaai_scraper.storage.schemas import CarRecord
+from dubizzle_scraper.storage.schemas import CarRecord
 
 
 def make_db_record(origin_id: str) -> dict[str, object]:
     return CarRecord(
-        parser_id=f"iaai:{origin_id}",
+        parser_id=f"dubizzle:{origin_id}",
         brand="Toyota",
         model="Camry",
-        origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
+        origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
         origin_id=origin_id,
         slug=f"toyota-camry-{origin_id}",
     ).model_dump(mode="json")
 
 
 class TestScraperSync(unittest.TestCase):
-    def _make_scraper(self) -> IAAIScraper:
+    def _make_scraper(self) -> DUBIZZLEScraper:
         s = Settings()
         s.log_level = "CRITICAL"
         s.database.url = "sqlite://"
-        return IAAIScraper(s)
+        return DUBIZZLEScraper(s)
 
     def test_sync_vehicle_uses_db_record(self) -> None:
         scraper = self._make_scraper()
@@ -37,7 +37,7 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
         scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
 
-        result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
+        result = scraper.sync_vehicle("https://www.dubizzle.com/VehicleDetail/111~US")
 
         self.assertEqual(result["status"], "success")
         self.assertIn("trace_id", result)
@@ -50,13 +50,13 @@ class TestScraperSync(unittest.TestCase):
         scraper.persistence.start_sync_run = MagicMock(return_value=2)
         scraper.persistence.finish_sync_run = MagicMock()
         scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
-            {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
+            {"https://www.dubizzle.com/VehicleDetail/111~US"}, {"dubizzle:222"},
         ))
         scraper.collect_listing = MagicMock(return_value={
             "vehicle_urls": [
-                "https://www.iaai.com/VehicleDetail/111~US",
+                "https://www.dubizzle.com/VehicleDetail/111~US",
-                "https://www.iaai.com/VehicleDetail/222~US",
+                "https://www.dubizzle.com/VehicleDetail/222~US",
-                "https://www.iaai.com/VehicleDetail/333~US",
+                "https://www.dubizzle.com/VehicleDetail/333~US",
             ]
         })
         scraper.sync_batch = MagicMock(return_value={
@@ -79,7 +79,7 @@ class TestScraperSync(unittest.TestCase):
         })
         result2 = scraper2.sync_listing(
             only_new=True,
-            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
             year_min=2020, year_max=2027,
         )
         self.assertEqual(result2["cars_upserted"], 10)
@@ -155,35 +155,35 @@ class TestScraperSync(unittest.TestCase):
         self.assertEqual(result["status"], "partial_success")
 
     def test_build_segment_listing_url(self) -> None:
-        base = "https://www.iaai.com/Vehiclelisting/Cars"
+        base = "https://www.dubizzle.com/Vehiclelisting/Cars"
         self.assertEqual(
-            IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
+            DUBIZZLEScraper._build_segment_listing_url(base, "TOYOTA"),
-            "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
+            "https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
         )
         self.assertEqual(
-            IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
+            DUBIZZLEScraper._build_segment_listing_url(base, "LAND ROVER"),
-            "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
+            "https://www.dubizzle.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
         )
-        self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
+        self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, None), base)
-        self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
+        self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, ""), base)
 
     def test_guard_and_protection_detection(self) -> None:
         with self.assertRaises(AntiBotDetectedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
+            DUBIZZLEScraper._raise_if_blocked_or_incomplete(
                 {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
                  "access_notes": {}, "vehicle_summary": {}},
-                "https://www.iaai.com/VehicleDetail/999~US",
+                "https://www.dubizzle.com/VehicleDetail/999~US",
             )
         with self.assertRaises(SiteStructureChangedError):
-            IAAIScraper._raise_if_blocked_or_incomplete(
+            DUBIZZLEScraper._raise_if_blocked_or_incomplete(
                 {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
                  "access_notes": {"possible_captcha": False, "possible_antibot": False},
                  "vehicle_summary": {}},
-                "https://www.iaai.com/VehicleDetail/999~US",
+                "https://www.dubizzle.com/VehicleDetail/999~US",
             )
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
+        self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
-        self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
+        self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
-        self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
+        self.assertFalse(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
 
     def test_close_resets_browser_state(self) -> None:
         scraper = self._make_scraper()
@@ -201,7 +201,7 @@ class TestScraperSync(unittest.TestCase):
         scraper = self._make_scraper()
         page = MagicMock()
         page_result = SimpleNamespace(
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
+            vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/123~US")],
         )
         scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
         scraper.listing_collector.open_cars_listing = MagicMock()
@@ -228,7 +228,7 @@ class TestScraperSync(unittest.TestCase):
         page = MagicMock()
         page_result = SimpleNamespace(
             page_number=1,
-            vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
+            vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/999~US", lot_number="999")],
             next_page_detected=False,
         )
 
@@ -245,7 +245,7 @@ class TestScraperSync(unittest.TestCase):
             "year_max": None,
         })
         scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
-        scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
+        scraper._extract_page_urls = MagicMock(return_value=["https://www.dubizzle.com/VehicleDetail/999~US"])
         scraper.sync_batch = MagicMock(return_value={
             "cars_upserted": 1,
             "cars_failed": 0,
@@ -256,11 +256,11 @@ class TestScraperSync(unittest.TestCase):
         result = scraper._sync_listing_streaming(
             make=None,
             model=None,
-            lane="iaai_cars",
+            lane="dubizzle_cars",
             limit=None,
             effective_only_new=False,
             started_at=0.0,
-            listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
+            listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=EAGLE",
         )
 
         scraper.listing_collector.open_cars_listing.assert_called_once()
@@ -278,9 +278,9 @@ class TestScraperSync(unittest.TestCase):
         })
 
         urls = [
-            "https://www.iaai.com/VehicleDetail/111~US",
+            "https://www.dubizzle.com/VehicleDetail/111~US",
-            "https://www.iaai.com/VehicleDetail/222~US",
+            "https://www.dubizzle.com/VehicleDetail/222~US",
-            "https://www.iaai.com/VehicleDetail/333~US",
+            "https://www.dubizzle.com/VehicleDetail/333~US",
         ]
         first_record = CarRecord.model_validate(make_db_record("111"))
 
@@ -298,8 +298,8 @@ class TestScraperSync(unittest.TestCase):
                 yield 0, first_record
                 raise FuturesTimeoutError()
 
-        with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
+        with patch("dubizzle_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
-             patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
+             patch("dubizzle_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
              patch.object(scraper, "_browser_fallback_parallel", return_value={
                  "records": [],
                  "failures": [],

tests/test_self_heal.py

@@ -3,7 +3,7 @@ from __future__ import annotations
 import unittest
 from unittest.mock import MagicMock, patch
 
-from iaai_scraper.worker import self_heal
+from dubizzle_scraper.worker import self_heal
 
 
 class TestSelfHeal(unittest.TestCase):
@@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase):
 		redis_client = MagicMock()
 		redis_client.get.side_effect = lambda key: {
 			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
-			"iaai:state:task_progress:a": '{"ts": 100}',
+			"dubizzle:state:task_progress:a": '{"ts": 100}',
-			"iaai:state:task_progress:b": '{"ts": 250}',
+			"dubizzle:state:task_progress:b": '{"ts": 250}',
-			"iaai:state:task_progress:c": '{"ts": 150}',
+			"dubizzle:state:task_progress:c": '{"ts": 150}',
 		}.get(key)
 		redis_client.scan_iter.return_value = [
-			"iaai:state:task_progress:a",
+			"dubizzle:state:task_progress:a",
-			"iaai:state:task_progress:b",
+			"dubizzle:state:task_progress:b",
-			"iaai:state:task_progress:c",
+			"dubizzle:state:task_progress:c",
 		]
 
 		ts = self_heal._read_last_progress_ts(redis_client)
@@ -38,20 +38,20 @@ class TestSelfHeal(unittest.TestCase):
 		redis_client = MagicMock()
 		redis_client.get.side_effect = lambda key: {
 			self_heal.GLOBAL_PROGRESS_TS_KEY: None,
-			"iaai:state:task_progress:a": "{bad-json}",
+			"dubizzle:state:task_progress:a": "{bad-json}",
-			"iaai:state:task_progress:b": '{"foo": "bar"}',
+			"dubizzle:state:task_progress:b": '{"foo": "bar"}',
 		}.get(key)
 		redis_client.scan_iter.return_value = [
-			"iaai:state:task_progress:a",
+			"dubizzle:state:task_progress:a",
-			"iaai:state:task_progress:b",
+			"dubizzle:state:task_progress:b",
 		]
 
 		ts = self_heal._read_last_progress_ts(redis_client)
 
 		self.assertIsNone(ts)
 
-	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
-	@patch("iaai_scraper.worker.self_heal.os.kill")
+	@patch("dubizzle_scraper.worker.self_heal.os.kill")
 	@patch("builtins.open")
 	def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
 		open_mock.return_value.__enter__.return_value.read.return_value = "123"
@@ -64,8 +64,8 @@ class TestSelfHeal(unittest.TestCase):
 		self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
 		self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
 
-	@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
+	@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
-	@patch("iaai_scraper.worker.self_heal.os.kill")
+	@patch("dubizzle_scraper.worker.self_heal.os.kill")
 	@patch("builtins.open")
 	def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
 		open_mock.return_value.__enter__.return_value.read.return_value = "123"

tests/test_utils.py

@@ -2,8 +2,8 @@ from __future__ import annotations
 
 import unittest
 
-from iaai_scraper.core.utils import deep_find_key
+from dubizzle_scraper.core.utils import deep_find_key
-from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
+from dubizzle_scraper.core.config import parse_listing_segments, _AUTO_YEAR_SPLITS
 
 
 class TestDeepFindKey(unittest.TestCase):
@@ -28,32 +28,19 @@ class TestParseListingSegments(unittest.TestCase):
         self.assertEqual(parse_listing_segments("invalid"), [])
 
     def test_auto_segments_complete_coverage(self) -> None:
-        """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
+        """auto: все годовые диапазоны покрыты, без дыр и без make-фильтра."""
         segs = parse_listing_segments("auto")
 
-        # Точное число сегментов.
+        self.assertEqual(len(segs), len(_AUTO_YEAR_SPLITS))
-        expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
+        self.assertEqual(
-        self.assertEqual(len(segs), expected)
+            [(s["year_min"], s["year_max"]) for s in segs],
-
+            list(_AUTO_YEAR_SPLITS),
-        # Все бренды из списка присутствуют.
+        )
-        makes_in_segments = {s["make"] for s in segs}
+        self.assertTrue(all(s["make"] is None for s in segs))
-        for make in IAAI_DEFAULT_MAKES:
-            self.assertIn(make, makes_in_segments)
-
-        # Крупные бренды разбиты на 3 сегмента с годами.
-        toyota = [s for s in segs if s["make"] == "TOYOTA"]
-        self.assertEqual(len(toyota), 3)
-        for s in toyota:
-            self.assertIsNotNone(s["year_min"])
-
-        # Мелкие бренды без годов.
-        lexus = [s for s in segs if s["make"] == "LEXUS"]
-        self.assertEqual(len(lexus), 1)
-        self.assertIsNone(lexus[0]["year_min"])
 
         # Годовые диапазоны покрывают 1950-2027.
         years = set()
-        for yr_min, yr_max in _YEAR_SPLITS:
+        for yr_min, yr_max in _AUTO_YEAR_SPLITS:
             years.update(range(yr_min, yr_max + 1))
         for year in range(1950, 2027):
             self.assertIn(year, years)

tests/test_worker_tasks.py

@@ -5,8 +5,8 @@ from dataclasses import replace
 import unittest
 from unittest.mock import MagicMock, patch
 
-from iaai_scraper.worker import tasks
+from dubizzle_scraper.worker import tasks
-from iaai_scraper.core.config import settings as base_settings
+from dubizzle_scraper.core.config import settings as base_settings
 
 
 class TestWorkerTaskLockHelpers(unittest.TestCase):
@@ -187,13 +187,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         ]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(
+                         base_settings.discovery,
+                         always_full_scan=False,
+                         mode="listing",
+                     ),
+                     celery=replace(
+                         base_settings.celery,
+                         parallel_segments=False,
+                     ),
+                 ),
+             ), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             redis_client.get.side_effect = lambda key: (
@@ -211,7 +227,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-resume-seg")
                 try:
                     result = tasks.sync_listing_task.run()
@@ -228,6 +244,22 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         segments = [{"make": "ACURA"}, {"make": "AUDI"}]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                    discovery=replace(
+                        base_settings.discovery,
+                        always_full_scan=False,
+                        mode="listing",
+                    ),
+                    celery=replace(
+                        base_settings.celery,
+                        parallel_segments=False,
+                    ),
+                 ),
+             ), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=True), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -235,7 +267,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             # Оставшийся чекпоинт не должен использоваться.
@@ -254,10 +286,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-792")
                 try:
-                    result = tasks.sync_listing_task.run()
+                    result = tasks.sync_listing_task.run(limit=1)
                 finally:
                     tasks.sync_listing_task.pop_request()
 
@@ -271,13 +303,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         segments = [{"make": "ACURA"}, {"make": "AUDI"}]
         with patch.object(tasks, "_get_persistence") as get_persistence, \
              patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(
+                         base_settings.discovery,
+                         always_full_scan=False,
+                         mode="listing",
+                     ),
+                     celery=replace(
+                         base_settings.celery,
+                         parallel_segments=False,
+                     ),
+                 ),
+             ), \
              patch.object(tasks, "_acquire_lock", return_value=True), \
              patch.object(tasks, "_is_full_scan_done", return_value=False), \
              patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             redis_client.get.side_effect = lambda key: (
@@ -295,7 +343,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-beyond")
                 try:
                     result = tasks.sync_listing_task.run()
@@ -507,7 +555,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
              patch.object(tasks.sync_listing_task, "update_state"), \
              patch.object(tasks, "_run_browser_job") as run_job, \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             redis_client.get.return_value = None
@@ -554,7 +602,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
              patch.object(tasks, "_release_lock_if_owner") as release_lock, \
              patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
              patch.object(tasks.sync_listing_task, "update_state"), \
-             patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+             patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
             get_persistence.return_value = MagicMock()
             redis_client = MagicMock()
             redis_client.get.side_effect = lambda key: (
@@ -579,7 +627,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
             scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
             scraper_ctx.__exit__.return_value = None
 
-            with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
+            with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
                 tasks.sync_listing_task.push_request(id="task-always-full-resume")
                 try:
                     result = tasks.sync_listing_task.run()
@@ -591,6 +639,85 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
         self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
         release_lock.assert_called_once()
 
+    def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None:
+        segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                     celery=replace(base_settings.celery, parallel_segments=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks, "_reset_segments_progress") as reset_progress, \
+             patch.object(tasks, "_set_segmented_scan_active") as set_active, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            redis_client = MagicMock()
+            redis_client.smembers.return_value = {"0", "1"}
+            redis_client.get.return_value = None
+            get_redis.return_value = redis_client
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
+                 patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+                tasks.sync_listing_task.push_request(id="task-always-hourly")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "success")
+        self.assertEqual(result["segments_dispatched"], 2)
+        reset_progress.assert_called_once_with(redis_client, len(segments))
+        set_active.assert_called_once_with(redis_client, total=len(segments))
+        self.assertEqual(task_app.send_task.call_count, 2)
+        release_lock.assert_called_once()
+
+    def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None:
+        segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
+        with patch.object(tasks, "_get_persistence") as get_persistence, \
+             patch.object(tasks, "_get_redis") as get_redis, \
+             patch.object(
+                 tasks,
+                 "Settings",
+                 return_value=replace(
+                     base_settings,
+                     discovery=replace(base_settings.discovery, always_full_scan=True),
+                     celery=replace(base_settings.celery, parallel_segments=True),
+                 ),
+             ), \
+             patch.object(tasks, "_acquire_lock", return_value=True), \
+             patch.object(tasks, "_is_full_scan_done", return_value=True), \
+             patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \
+             patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
+             patch.object(tasks, "_release_lock_if_owner") as release_lock, \
+             patch.object(tasks.sync_listing_task, "update_state"):
+            get_persistence.return_value = MagicMock()
+            get_redis.return_value = MagicMock()
+            start_heartbeat.return_value = (MagicMock(), MagicMock())
+
+            with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
+                 patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
+                tasks.sync_listing_task.push_request(id="task-always-skip-duplicate")
+                try:
+                    result = tasks.sync_listing_task.run()
+                finally:
+                    tasks.sync_listing_task.pop_request()
+
+        self.assertEqual(result["status"], "skipped")
+        self.assertEqual(result["reason"], "segmented_scan_in_progress")
+        task_app.send_task.assert_not_called()
+        release_lock.assert_called_once()
+
 
 if __name__ == "__main__":
     unittest.main()

uv.lock

@@ -342,7 +342,7 @@ wheels = [
 ]
 
 [[package]]
-name = "iaai-scraper"
+name = "dubizzle-scraper"
 version = "0.1.0"
 source = { editable = "." }
 dependencies = [