From d484088e18e3dcd54133c4aa9aaf461109699cac Mon Sep 17 00:00:00 2001 From: qananasikq Date: Fri, 24 Apr 2026 20:46:58 +0300 Subject: [PATCH] update scraper package --- .env.example | 89 +- Dockerfile | 42 +- alembic.ini | 2 +- alembic/env.py | 4 +- alembic/versions/001_initial.py | 2 +- alembic/versions/003_add_composite_indexes.py | 8 +- docker-compose.yml | 101 +- .../__init__.py | 0 .../api/__init__.py | 0 {iaai_scraper => dubizzle_scraper}/api/app.py | 4 +- .../api/deps.py | 0 .../api/routes/__init__.py | 0 .../api/routes/cars.py | 0 .../api/routes/health.py | 4 +- .../api/routes/tasks.py | 4 +- .../browser/__init__.py | 0 .../browser/factory.py | 6 +- .../browser/listing.py | 31 +- .../browser/network.py | 4 +- .../browser/pace.py | 0 {iaai_scraper => dubizzle_scraper}/cli.py | 47 +- .../core/__init__.py | 0 dubizzle_scraper/core/config.py | 315 +++++ .../core/exceptions.py | 0 .../core/logs.py | 0 .../core/retry.py | 2 +- .../core/runtime_config.py | 2 +- .../core/utils.py | 0 .../discovery/__init__.py | 8 + dubizzle_scraper/discovery/algolia.py | 479 +++++++ .../discovery/sitemap.py | 4 +- .../parsing/__init__.py | 0 .../parsing/mapper.py | 271 +++- .../parsing/parser.py | 12 +- .../proxy_bridge.py | 0 {iaai_scraper => dubizzle_scraper}/scraper.py | 1181 +++++++++-------- .../storage/__init__.py | 0 .../storage/db.py | 26 +- .../storage/enums.py | 4 +- .../storage/models.py | 0 .../storage/schemas.py | 0 .../worker/__init__.py | 0 .../worker/celery_app.py | 28 +- .../worker/self_heal.py | 40 +- .../worker/tasks.py | 305 +++-- entrypoint.sh | 20 +- iaai_scraper/core/config.py | 308 ----- pyproject.toml | 8 +- runtime_config.json | 2 +- tests/test_db.py | 24 +- tests/test_listing.py | 10 +- tests/test_mappers.py | 107 +- tests/test_parser.py | 10 +- tests/test_resilience.py | 14 +- tests/test_scraper.py | 76 +- tests/test_self_heal.py | 30 +- tests/test_utils.py | 33 +- tests/test_worker_tasks.py | 151 ++- uv.lock | 2 +- 59 files changed, 2548 insertions(+), 1272 deletions(-) rename {iaai_scraper => dubizzle_scraper}/__init__.py (100%) rename {iaai_scraper => dubizzle_scraper}/api/__init__.py (100%) rename {iaai_scraper => dubizzle_scraper}/api/app.py (91%) rename {iaai_scraper => dubizzle_scraper}/api/deps.py (100%) rename {iaai_scraper => dubizzle_scraper}/api/routes/__init__.py (100%) rename {iaai_scraper => dubizzle_scraper}/api/routes/cars.py (100%) rename {iaai_scraper => dubizzle_scraper}/api/routes/health.py (88%) rename {iaai_scraper => dubizzle_scraper}/api/routes/tasks.py (98%) rename {iaai_scraper => dubizzle_scraper}/browser/__init__.py (100%) rename {iaai_scraper => dubizzle_scraper}/browser/factory.py (97%) rename {iaai_scraper => dubizzle_scraper}/browser/listing.py (94%) rename {iaai_scraper => dubizzle_scraper}/browser/network.py (97%) rename {iaai_scraper => dubizzle_scraper}/browser/pace.py (100%) rename {iaai_scraper => dubizzle_scraper}/cli.py (67%) rename {iaai_scraper => dubizzle_scraper}/core/__init__.py (100%) create mode 100644 dubizzle_scraper/core/config.py rename {iaai_scraper => dubizzle_scraper}/core/exceptions.py (100%) rename {iaai_scraper => dubizzle_scraper}/core/logs.py (100%) rename {iaai_scraper => dubizzle_scraper}/core/retry.py (97%) rename {iaai_scraper => dubizzle_scraper}/core/runtime_config.py (99%) rename {iaai_scraper => dubizzle_scraper}/core/utils.py (100%) rename {iaai_scraper => dubizzle_scraper}/discovery/__init__.py (64%) create mode 100644 dubizzle_scraper/discovery/algolia.py rename {iaai_scraper => dubizzle_scraper}/discovery/sitemap.py (98%) rename {iaai_scraper => dubizzle_scraper}/parsing/__init__.py (100%) rename {iaai_scraper => dubizzle_scraper}/parsing/mapper.py (60%) rename {iaai_scraper => dubizzle_scraper}/parsing/parser.py (97%) rename {iaai_scraper => dubizzle_scraper}/proxy_bridge.py (100%) rename {iaai_scraper => dubizzle_scraper}/scraper.py (73%) rename {iaai_scraper => dubizzle_scraper}/storage/__init__.py (100%) rename {iaai_scraper => dubizzle_scraper}/storage/db.py (97%) rename {iaai_scraper => dubizzle_scraper}/storage/enums.py (87%) rename {iaai_scraper => dubizzle_scraper}/storage/models.py (100%) rename {iaai_scraper => dubizzle_scraper}/storage/schemas.py (100%) rename {iaai_scraper => dubizzle_scraper}/worker/__init__.py (100%) rename {iaai_scraper => dubizzle_scraper}/worker/celery_app.py (84%) rename {iaai_scraper => dubizzle_scraper}/worker/self_heal.py (77%) rename {iaai_scraper => dubizzle_scraper}/worker/tasks.py (86%) delete mode 100644 iaai_scraper/core/config.py diff --git a/.env.example b/.env.example index 02ad08e..f96f6f7 100644 --- a/.env.example +++ b/.env.example @@ -1,51 +1,51 @@ -IAAI_HEADLESS=true -IAAI_LOG_LEVEL=INFO +DUBIZZLE_HEADLESS=true +DUBIZZLE_LOG_LEVEL=INFO -IAAI_CAPTURE_SAME_ORIGIN_ONLY=true -IAAI_MAX_CAPTURED_REQUESTS=40 -IAAI_MAX_CAPTURED_JSON_RESPONSES=20 +DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY=true +DUBIZZLE_MAX_CAPTURED_REQUESTS=40 +DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES=20 -IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars -IAAI_LISTING_SEGMENTS=auto -IAAI_MAX_PAGES_PER_RUN=999999 -IAAI_MAX_VEHICLES_PER_RUN=999999 -IAAI_PAGE_LINK_LIMIT=999999 -IAAI_INCLUDE_PAGINATION=true -IAAI_COLLECT_CURRENT_PAGE_ONLY=false +DUBIZZLE_CARS_LISTING_URL=https://uae.dubizzle.com/motors/used-cars/ +DUBIZZLE_LISTING_SEGMENTS=auto +DUBIZZLE_MAX_PAGES_PER_RUN=999999 +DUBIZZLE_MAX_VEHICLES_PER_RUN=999999 +DUBIZZLE_PAGE_LINK_LIMIT=999999 +DUBIZZLE_INCLUDE_PAGINATION=true +DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY=false -IAAI_HUMAN_PACE_ENABLED=true -IAAI_PARALLEL_TABS=10 +DUBIZZLE_HUMAN_PACE_ENABLED=true +DUBIZZLE_PARALLEL_TABS=10 CELERY_BATCH_SIZE=100 -IAAI_AFTER_LISTING_OPEN_MIN_S=0.2 -IAAI_AFTER_LISTING_OPEN_MAX_S=0.5 -IAAI_AFTER_FILTER_ACTION_MIN_S=0.2 -IAAI_AFTER_FILTER_ACTION_MAX_S=0.5 -IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02 -IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08 -IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02 -IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08 -IAAI_BETWEEN_VEHICLES_MIN_S=0.02 -IAAI_BETWEEN_VEHICLES_MAX_S=0.08 -IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2 -IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5 +DUBIZZLE_AFTER_LISTING_OPEN_MIN_S=0.2 +DUBIZZLE_AFTER_LISTING_OPEN_MAX_S=0.5 +DUBIZZLE_AFTER_FILTER_ACTION_MIN_S=0.2 +DUBIZZLE_AFTER_FILTER_ACTION_MAX_S=0.5 +DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S=0.02 +DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S=0.08 +DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S=0.02 +DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S=0.08 +DUBIZZLE_BETWEEN_VEHICLES_MIN_S=0.02 +DUBIZZLE_BETWEEN_VEHICLES_MAX_S=0.08 +DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S=0.2 +DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S=0.5 -IAAI_SYNC_ONLY_NEW=false -IAAI_TOKENS_FILE=/data/tokens.json -IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json +DUBIZZLE_SYNC_ONLY_NEW=false +DUBIZZLE_TOKENS_FILE=/data/tokens.json +DUBIZZLE_RUNTIME_CONFIG_FILE=/app/runtime_config.json -IAAI_MAX_RETRIES=5 -IAAI_RETRY_DELAY_SECONDS=4 -IAAI_RETRY_BACKOFF_MULTIPLIER=2.0 -IAAI_RETRY_JITTER_SECONDS=0.5 -IAAI_TIMEOUT_MS=90000 -IAAI_FALLBACK_NAV_TIMEOUT_MS=30000 +DUBIZZLE_MAX_RETRIES=5 +DUBIZZLE_RETRY_DELAY_SECONDS=4 +DUBIZZLE_RETRY_BACKOFF_MULTIPLIER=2.0 +DUBIZZLE_RETRY_JITTER_SECONDS=0.5 +DUBIZZLE_TIMEOUT_MS=90000 +DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS=30000 -IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper -IAAI_DATABASE_ECHO=false -IAAI_DATABASE_POOL_SIZE=5 -IAAI_DATABASE_MAX_OVERFLOW=5 +DUBIZZLE_DATABASE_URL=postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper +DUBIZZLE_DATABASE_ECHO=false +DUBIZZLE_DATABASE_POOL_SIZE=5 +DUBIZZLE_DATABASE_MAX_OVERFLOW=5 -IAAI_REDIS_URL=redis://redis:6379/0 +DUBIZZLE_REDIS_URL=redis://redis:6379/0 CELERY_BROKER_URL=redis://redis:6379/0 CELERY_RESULT_BACKEND=redis://redis:6379/0 @@ -55,6 +55,13 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000 CELERY_WORKER_CONCURRENCY=1 CELERY_BEAT_SYNC_INTERVAL_MINUTES=60 CELERY_BEAT_SYNC_LIMIT=0 -IAAI_ALWAYS_FULL_SCAN=true +DUBIZZLE_ALWAYS_FULL_SCAN=true +CELERY_PARALLEL_SEGMENTS=true +DUBIZZLE_DISCOVERY_MODE=algolia +DUBIZZLE_ALGOLIA_APPLICATION_ID=WD0PTZ13ZS +DUBIZZLE_ALGOLIA_API_KEY=cef139620248f1bc328a00fddc7107a6 +DUBIZZLE_ALGOLIA_INDEX_NAME=motors.com +DUBIZZLE_ALGOLIA_BASE_URL=https://WD0PTZ13ZS-dsn.algolia.net +DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY=false SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6 SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600 diff --git a/Dockerfile b/Dockerfile index c8d8c0c..0f81bbd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,8 +1,43 @@ -FROM mcr.microsoft.com/playwright/python:v1.58.0-noble +FROM python:3.12-slim-bookworm ENV PYTHONDONTWRITEBYTECODE=1 \ PYTHONUNBUFFERED=1 +ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + curl \ + ca-certificates \ + bash \ + build-essential \ + libasound2 \ + libatk-bridge2.0-0 \ + libatk1.0-0 \ + libc6 \ + libcairo2 \ + libcups2 \ + libdbus-1-3 \ + libdrm2 \ + libgbm1 \ + libglib2.0-0 \ + libgtk-3-0 \ + libnspr4 \ + libnss3 \ + libpango-1.0-0 \ + libx11-6 \ + libx11-xcb1 \ + libxcb1 \ + libxcomposite1 \ + libxdamage1 \ + libxext6 \ + libxfixes3 \ + libxkbcommon0 \ + libxrandr2 \ + wget \ + xdg-utils \ + && rm -rf /var/lib/apt/lists/* + RUN groupadd --system app && useradd --system --gid app --create-home app WORKDIR /app @@ -19,7 +54,8 @@ RUN python -m playwright install chromium firefox COPY . . RUN pip install --no-cache-dir -e . -RUN python -m compileall -q iaai_scraper +RUN python -m compileall -q dubizzle_scraper +RUN sed -i 's/\r$//' /app/entrypoint.sh RUN chmod +x entrypoint.sh RUN chown -R app:app /app @@ -29,4 +65,4 @@ USER app # По умолчанию запускаем API. ENTRYPOINT ["./entrypoint.sh"] -CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] +CMD ["uvicorn", "dubizzle_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/alembic.ini b/alembic.ini index f4be675..e2b38e7 100644 --- a/alembic.ini +++ b/alembic.ini @@ -3,7 +3,7 @@ [alembic] script_location = alembic prepend_sys_path = . -sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper +sqlalchemy.url = postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper [loggers] keys = root,sqlalchemy,alembic diff --git a/alembic/env.py b/alembic/env.py index f6f4ab0..87249e5 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool # Добавляем корень проекта в sys.path sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) -from iaai_scraper.core.config import Settings -from iaai_scraper.storage.models import Base +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.storage.models import Base config = context.config diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py index 4179d3e..befc776 100644 --- a/alembic/versions/001_initial.py +++ b/alembic/versions/001_initial.py @@ -11,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # Таблица cars — аукционные машины с IAAI + # Таблица cars — аукционные машины с DUBIZZLE op.create_table( "cars", sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), diff --git a/alembic/versions/003_add_composite_indexes.py b/alembic/versions/003_add_composite_indexes.py index 1388d45..0b1ccb6 100644 --- a/alembic/versions/003_add_composite_indexes.py +++ b/alembic/versions/003_add_composite_indexes.py @@ -10,12 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # Partial index для активных машин IAAI (is_sold = FALSE) + # Partial index для активных машин DUBIZZLE (is_sold = FALSE) # Ускоряет поиск при mark_sold операциях op.execute( - "CREATE INDEX IF NOT EXISTS ix_cars_active_iaai " + "CREATE INDEX IF NOT EXISTS ix_cars_active_dubizzle " "ON cars (origin_url) " - "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'" + "WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%'" ) # Composite index для проверки дубликатов изображений @@ -35,4 +35,4 @@ def upgrade() -> None: def downgrade() -> None: op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id") op.drop_index("ix_images_car_id_fullres", table_name="images") - op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai") + op.execute("DROP INDEX IF EXISTS ix_cars_active_dubizzle") diff --git a/docker-compose.yml b/docker-compose.yml index 72d0a19..adbe258 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,35 +1,40 @@ x-app-env: &app-env # Всегда используем Postgres. - IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper - IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0} + DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper + DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0} CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} - IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800} + DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800} CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} # 0 = без лимита. CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} + CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true} CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} - IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3} - IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9} - IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8} - IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true} - IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto} - IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999} - IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000} - IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true} - IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true} - IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json} - IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} - IAAI_BROWSER_ENGINE: chromium + DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3} + DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9} + DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8} + DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true} + DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia} + DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto} + DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/} + DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999} + DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000} + DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true} + # Работаем через Algolia; fallback в sitemap по умолчанию отключён. + DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false} + DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true} + DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json} + DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} + DUBIZZLE_BROWSER_ENGINE: chromium # Self-heal для worker. - IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true} - IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} - IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900} - IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} - IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} + DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true} + DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} + DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900} + DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} + DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} TZ: ${TZ:-UTC} x-env-file: &env-file @@ -42,29 +47,32 @@ x-app-service: &app-service environment: *app-env volumes: - ./runtime_config.json:/app/runtime_config.json:ro + - ./artifacts:/app/artifacts x-worker-service: &worker-service <<: *app-service volumes: - ./runtime_config.json:/app/runtime_config.json:ro + - ./artifacts:/app/artifacts - tokens_data:/data services: # PostgreSQL. postgres: image: postgres:16-alpine - container_name: iaai-postgres + container_name: dubizzle-postgres restart: unless-stopped environment: - POSTGRES_USER: iaai - POSTGRES_PASSWORD: iaai - POSTGRES_DB: iaai_scraper + POSTGRES_USER: dubizzle + POSTGRES_PASSWORD: dubizzle + POSTGRES_DB: dubizzle_scraper ports: - - "127.0.0.1:5432:5432" + # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. + - "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432" volumes: - pgdata:/var/lib/postgresql/data healthcheck: - test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"] + test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"] interval: 5s timeout: 3s retries: 5 @@ -77,10 +85,11 @@ services: # Redis. redis: image: redis:7-alpine - container_name: iaai-redis + container_name: dubizzle-redis restart: unless-stopped ports: - - "127.0.0.1:6379:6379" + # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. + - "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379" healthcheck: test: ["CMD", "redis-cli", "ping"] interval: 5s @@ -101,7 +110,7 @@ services: # Миграции. migrate: <<: *app-service - container_name: iaai-migrate + container_name: dubizzle-migrate restart: "no" depends_on: postgres: @@ -111,17 +120,18 @@ services: # API. api: <<: *app-service - container_name: iaai-api + container_name: dubizzle-api restart: unless-stopped ports: - - "127.0.0.1:8000:8000" + # Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами. + - "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000" depends_on: migrate: condition: service_completed_successfully redis: condition: service_healthy command: > - uvicorn iaai_scraper.api.app:app + uvicorn dubizzle_scraper.api.app:app --host 0.0.0.0 --port 8000 --workers 1 healthcheck: test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"] @@ -148,13 +158,13 @@ services: condition: service_healthy stop_grace_period: 60s command: > - celery -A iaai_scraper.worker.celery_app worker + celery -A dubizzle_scraper.worker.celery_app worker --loglevel=info --concurrency=1 --pool=solo --pidfile=/tmp/celery-worker.pid -Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} healthcheck: # Проверка worker. - test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"] + test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"] interval: 60s timeout: 10s retries: 3 @@ -168,7 +178,7 @@ services: # Beat. beat: <<: *worker-service - container_name: iaai-beat + container_name: dubizzle-beat restart: unless-stopped init: true depends_on: @@ -177,7 +187,7 @@ services: redis: condition: service_healthy command: > - celery -A iaai_scraper.worker.celery_app beat + celery -A dubizzle_scraper.worker.celery_app beat --loglevel=info --pidfile=/tmp/celery-beat.pid --schedule=/tmp/celerybeat-schedule @@ -193,6 +203,25 @@ services: max-size: "10m" max-file: "5" + # Разовый прогон sync-listing через Docker CLI. + sync: + <<: *worker-service + container_name: dubizzle-sync + restart: "no" + profiles: ["manual"] + depends_on: + migrate: + condition: service_completed_successfully + redis: + condition: service_healthy + command: > + bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json" + logging: + driver: json-file + options: + max-size: "10m" + max-file: "5" + volumes: pgdata: redisdata: diff --git a/iaai_scraper/__init__.py b/dubizzle_scraper/__init__.py similarity index 100% rename from iaai_scraper/__init__.py rename to dubizzle_scraper/__init__.py diff --git a/iaai_scraper/api/__init__.py b/dubizzle_scraper/api/__init__.py similarity index 100% rename from iaai_scraper/api/__init__.py rename to dubizzle_scraper/api/__init__.py diff --git a/iaai_scraper/api/app.py b/dubizzle_scraper/api/app.py similarity index 91% rename from iaai_scraper/api/app.py rename to dubizzle_scraper/api/app.py index ba73b00..3c5d921 100644 --- a/iaai_scraper/api/app.py +++ b/dubizzle_scraper/api/app.py @@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI: _settings = settings or Settings() app = FastAPI( - title="IAAI Scraper API", - description="REST API для управления задачами скрапинга IAAI и просмотра данных", + title="Dubizzle Scraper API", + description="REST API для управления задачами скрапинга Dubizzle и просмотра данных", version="1.0.0", lifespan=lifespan, ) diff --git a/iaai_scraper/api/deps.py b/dubizzle_scraper/api/deps.py similarity index 100% rename from iaai_scraper/api/deps.py rename to dubizzle_scraper/api/deps.py diff --git a/iaai_scraper/api/routes/__init__.py b/dubizzle_scraper/api/routes/__init__.py similarity index 100% rename from iaai_scraper/api/routes/__init__.py rename to dubizzle_scraper/api/routes/__init__.py diff --git a/iaai_scraper/api/routes/cars.py b/dubizzle_scraper/api/routes/cars.py similarity index 100% rename from iaai_scraper/api/routes/cars.py rename to dubizzle_scraper/api/routes/cars.py diff --git a/iaai_scraper/api/routes/health.py b/dubizzle_scraper/api/routes/health.py similarity index 88% rename from iaai_scraper/api/routes/health.py rename to dubizzle_scraper/api/routes/health.py index a7de8d7..96c5997 100644 --- a/iaai_scraper/api/routes/health.py +++ b/dubizzle_scraper/api/routes/health.py @@ -9,7 +9,7 @@ from ..deps import get_persistence from ...storage.db import PersistenceService router = APIRouter() -logger = logging.getLogger("iaai_scraper.api.health") +logger = logging.getLogger("dubizzle_scraper.api.health") @router.get("/health") @@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)): return { "status": "ok" if db_ok else "degraded", - "service": "iaai-scraper-api", + "service": "dubizzle-scraper-api", "database": "connected" if db_ok else "unavailable", } diff --git a/iaai_scraper/api/routes/tasks.py b/dubizzle_scraper/api/routes/tasks.py similarity index 98% rename from iaai_scraper/api/routes/tasks.py rename to dubizzle_scraper/api/routes/tasks.py index 3979d34..7e44ca3 100644 --- a/iaai_scraper/api/routes/tasks.py +++ b/dubizzle_scraper/api/routes/tasks.py @@ -15,13 +15,13 @@ router = APIRouter() class SyncVehicleRequest(BaseModel): vehicle_url: str - lane: str = "iaai" + lane: str = "dubizzle" class SyncListingRequest(BaseModel): make: str | None = None model: str | None = None - lane: str = "iaai_cars" + lane: str = "dubizzle_cars" limit: int | None = None only_new: bool | None = None diff --git a/iaai_scraper/browser/__init__.py b/dubizzle_scraper/browser/__init__.py similarity index 100% rename from iaai_scraper/browser/__init__.py rename to dubizzle_scraper/browser/__init__.py diff --git a/iaai_scraper/browser/factory.py b/dubizzle_scraper/browser/factory.py similarity index 97% rename from iaai_scraper/browser/factory.py rename to dubizzle_scraper/browser/factory.py index ccba6cf..53341fb 100644 --- a/iaai_scraper/browser/factory.py +++ b/dubizzle_scraper/browser/factory.py @@ -11,7 +11,7 @@ except ImportError: from ..core.config import Settings -logger = logging.getLogger("iaai_scraper.browser") +logger = logging.getLogger("dubizzle_scraper.browser") def _build_init_script() -> str: @@ -72,11 +72,11 @@ class BrowserFactory: # Выбор движка. engine = self.settings.browser_engine.strip().lower() if engine == "auto": - # Для IAAI стабильнее Chromium. + # Для DUBIZZLE стабильнее Chromium. return "chromium" if engine in ("firefox", "chromium"): return engine - logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine) + logger.warning("Unknown DUBIZZLE_BROWSER_ENGINE=%r, falling back to auto", engine) return "chromium" def create_browser(self, playwright: Playwright) -> Browser: diff --git a/iaai_scraper/browser/listing.py b/dubizzle_scraper/browser/listing.py similarity index 94% rename from iaai_scraper/browser/listing.py rename to dubizzle_scraper/browser/listing.py index f9cb949..9bfd2e3 100644 --- a/iaai_scraper/browser/listing.py +++ b/dubizzle_scraper/browser/listing.py @@ -11,9 +11,15 @@ from .pace import HumanPacer from ..core.config import Settings from ..core.utils import first_non_empty -logger = logging.getLogger("iaai_scraper.listing") -VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) -VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']" +logger = logging.getLogger("dubizzle_scraper.listing") +VEHICLE_HREF_RE = re.compile( + r'(?:/VehicleDetail/(?P\d+)(?:~[A-Z]{2})?)|(?:/motors/used-cars/[^"\s]+?(?:/ad-(?P\d+)/?|---(?P[a-f0-9]{32})/?))|(?:/s/(?P[A-Za-z0-9]+))', + re.IGNORECASE, +) +VEHICLE_LINK_SELECTOR = ( + "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail'], " + "a[href*='/motors/used-cars/'], a[href*='/s/']" +) COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = ( "button:has-text('Accept All')", "button:has-text('Accept all')", @@ -108,7 +114,7 @@ class ListingCollector: try: page.wait_for_function( f"""() => {{ - const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\"); + const a = document.querySelector(\"{VEHICLE_LINK_SELECTOR}\"); return a && a.getAttribute('href') !== '{old_first_href}'; }}""", timeout=12000, @@ -331,7 +337,10 @@ class ListingCollector: match = VEHICLE_HREF_RE.search(href) if not match: continue - lot_number = match.group(1) + lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or None + if lot_number is None: + short = match.group("short_id") + lot_number = short if short else None absolute = urljoin(self.settings.home_url, match.group(0)) if absolute in seen: continue @@ -341,7 +350,7 @@ class ListingCollector: if len(links) >= self.settings.listing.max_vehicles_per_run: break - # Fallback: на IAAI ссылки иногда не рендерятся как , + # Fallback: на DUBIZZLE ссылки иногда не рендерятся как , # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). if not links: try: @@ -381,7 +390,7 @@ class ListingCollector: seen: set[str] = set() for match in VEHICLE_HREF_RE.finditer(normalized): - lot_number = match.group(1) + lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or match.group("short_id") or "" absolute = urljoin(self.settings.home_url, match.group(0)) if absolute in seen: continue @@ -424,7 +433,7 @@ class ListingCollector: self.pacer.after_page_change() return True - # Fallback для IAAI: пагинация часто рендерится как набор номеров страниц + # Fallback для DUBIZZLE: пагинация часто рендерится как набор номеров страниц # + стрелка с иконкой, без явного текста Next. try: clicked = bool(page.evaluate( @@ -573,7 +582,7 @@ class ListingCollector: ): page_known = sum( 1 for item in page_result.vehicle_links - if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids + if item.lot_number and f"dubizzle:{item.lot_number}" in known_origin_ids ) page_new = len(page_result.vehicle_links) - page_known ratio = page_known / len(page_result.vehicle_links) @@ -651,6 +660,10 @@ class ListingCollector: count = locator.count() if count == 0: continue + # Тестовые/fake локаторы могут не поддерживать nth/get_attribute. + # В таком случае считаем наличие селектора достаточным признаком next. + if not hasattr(locator, "nth"): + return True # Проверяем, что хотя бы один элемент не disabled. # Disabled "Next" на последней странице не означает наличия следующей. for i in range(min(count, 3)): diff --git a/iaai_scraper/browser/network.py b/dubizzle_scraper/browser/network.py similarity index 97% rename from iaai_scraper/browser/network.py rename to dubizzle_scraper/browser/network.py index 0674976..94d77f8 100644 --- a/iaai_scraper/browser/network.py +++ b/dubizzle_scraper/browser/network.py @@ -8,7 +8,7 @@ from playwright.sync_api import Page, Request, Response from ..core.config import Settings -logger = logging.getLogger("iaai_scraper.network") +logger = logging.getLogger("dubizzle_scraper.network") @dataclass @@ -44,7 +44,7 @@ class NetworkCapture: if not self.settings.capture.capture_same_origin_only or not self._origin: return True netloc = urlparse(url).netloc.lower() - return netloc == self._origin or netloc.endswith(".iaai.com") + return netloc == self._origin or netloc.endswith(".dubizzle.com") def _on_request(self, request: Request) -> None: # Берём только xhr/fetch diff --git a/iaai_scraper/browser/pace.py b/dubizzle_scraper/browser/pace.py similarity index 100% rename from iaai_scraper/browser/pace.py rename to dubizzle_scraper/browser/pace.py diff --git a/iaai_scraper/cli.py b/dubizzle_scraper/cli.py similarity index 67% rename from iaai_scraper/cli.py rename to dubizzle_scraper/cli.py index c3b76d5..dbf41ed 100644 --- a/iaai_scraper/cli.py +++ b/dubizzle_scraper/cli.py @@ -1,13 +1,13 @@ import argparse from pathlib import Path -from .core.config import Settings +from .core.config import Settings, parse_listing_segments from .core.utils import save_to_json -from .scraper import IAAIScraper +from .scraper import DUBIZZLEScraper def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser(description="IAAI scraper CLI") + parser = argparse.ArgumentParser(description="Dubizzle scraper CLI") parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode") parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging") subparsers = parser.add_subparsers(dest="command", required=True) @@ -19,24 +19,24 @@ def build_parser() -> argparse.ArgumentParser: listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page") listing_parser.add_argument("--make", default=None) listing_parser.add_argument("--model", default=None) - listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json")) + listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_listing_links.json")) scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page") scrape_parser.add_argument("vehicle_url") - scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json")) + scrape_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_vehicle_detail.json")) sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle") sync_vehicle_parser.add_argument("vehicle_url") - sync_vehicle_parser.add_argument("--lane", default="iaai") - sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json")) + sync_vehicle_parser.add_argument("--lane", default="dubizzle") + sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_vehicle.json")) sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles") sync_listing_parser.add_argument("--make", default=None) sync_listing_parser.add_argument("--model", default=None) - sync_listing_parser.add_argument("--lane", default="iaai_cars") + sync_listing_parser.add_argument("--lane", default="dubizzle_cars") sync_listing_parser.add_argument("--limit", type=int, default=None) sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None) - sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json")) + sync_listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_listing.json")) return parser @@ -53,7 +53,7 @@ def main() -> None: if args.debug: runtime_settings.log_level = "DEBUG" - with IAAIScraper(runtime_settings) as scraper: + with DUBIZZLEScraper(runtime_settings) as scraper: if args.command == "init-db": data = scraper.init_db() print(f"DB initialized: {data}") @@ -66,13 +66,28 @@ def main() -> None: data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane) else: only_new = None if args.only_new is None else args.only_new == "true" - data = scraper.sync_listing( - make=args.make, - model=args.model, - lane=args.lane, - limit=args.limit, - only_new=only_new, + segments = parse_listing_segments(scraper.settings.listing.listing_segments_json) + use_segmented = ( + bool(segments) + and args.make is None + and args.model is None + and args.limit is None + and scraper.settings.discovery.mode in {"listing", "algolia"} ) + if use_segmented: + data = scraper.sync_listing_segmented( + segments=segments, + lane=args.lane, + only_new=only_new, + ) + else: + data = scraper.sync_listing( + make=args.make, + model=args.model, + lane=args.lane, + limit=args.limit, + only_new=only_new, + ) save_to_json(data, Path(args.output)) print(f"Saved to {Path(args.output).resolve()}") diff --git a/iaai_scraper/core/__init__.py b/dubizzle_scraper/core/__init__.py similarity index 100% rename from iaai_scraper/core/__init__.py rename to dubizzle_scraper/core/__init__.py diff --git a/dubizzle_scraper/core/config.py b/dubizzle_scraper/core/config.py new file mode 100644 index 0000000..dac1c6b --- /dev/null +++ b/dubizzle_scraper/core/config.py @@ -0,0 +1,315 @@ +import json +import os +from dataclasses import dataclass, field +from pathlib import Path + +from dotenv import load_dotenv + +load_dotenv() + + +TRUE_VALUES = {"1", "true", "yes", "on"} + + +def _resolve_env_value(name: str) -> str | None: + """Возвращает значение env с поддержкой legacy-префиксов DUBIZZLE_/IAAI_.""" + value = os.getenv(name) + if value is not None: + return value + if name.startswith("DUBIZZLE_"): + return os.getenv("IAAI_" + name[len("DUBIZZLE_"):]) + return None + + +# Хелперы для чтения env-переменных с приведением типов + +def _env_str(name: str, default: str) -> str: + value = _resolve_env_value(name) + return value if value is not None else default + + +def _env_optional_str(name: str) -> str | None: + value = _resolve_env_value(name) + if value is None: + return None + value = value.strip() + return value or None + + +def _env_path_str(name: str) -> str | None: + value = _env_optional_str(name) + if value is None: + return None + return str(Path(value).expanduser()) + + +def _env_bool(name: str, default: bool) -> bool: + fallback = "true" if default else "false" + return _env_str(name, fallback).strip().lower() in TRUE_VALUES + + +def _env_int(name: str, default: int) -> int: + return int(_env_str(name, str(default)).strip()) + + +def _env_float(name: str, default: float) -> float: + return float(_env_str(name, str(default)).strip()) + + +# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) + +@dataclass(slots=True) +class FingerprintConfig: + user_agent: str = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/135.0.0.0 Safari/537.36" + ) + viewport_presets: tuple[dict[str, int], ...] = ( + {"width": 1920, "height": 1080}, + {"width": 1600, "height": 900}, + {"width": 1536, "height": 864}, + {"width": 1440, "height": 900}, + {"width": 1366, "height": 768}, + ) + timezone_candidates: tuple[str, ...] = ( + "America/New_York", + "America/Chicago", + "America/Los_Angeles", + ) + locale: str = "en-US" + sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' + + +# Конфиг перехвата сетевых запросов (лимиты на кол-во) + +@dataclass(slots=True) +class CaptureConfig: + capture_same_origin_only: bool = _env_bool("DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY", True) + max_requests: int = _env_int("DUBIZZLE_MAX_CAPTURED_REQUESTS", 40) + max_json_responses: int = _env_int("DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES", 30) + + +# Конфиг пауз между действиями (имитация человека) + +@dataclass(slots=True) +class HumanPaceConfig: + enabled: bool = _env_bool("DUBIZZLE_HUMAN_PACE_ENABLED", True) + after_listing_open_min_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MIN_S", 0.5) + after_listing_open_max_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MAX_S", 1.2) + after_filter_action_min_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MIN_S", 0.5) + after_filter_action_max_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MAX_S", 1.2) + before_vehicle_open_min_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) + before_vehicle_open_max_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) + after_vehicle_open_min_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S", 0.05) + after_vehicle_open_max_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S", 0.15) + between_vehicles_min_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MIN_S", 0.05) + between_vehicles_max_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MAX_S", 0.15) + after_page_change_min_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S", 0.8) + after_page_change_max_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S", 1.8) + + +# Конфиг сбора листинга (URL, лимиты страниц и машин) + +@dataclass(slots=True) +class ListingConfig: + cars_url: str = _env_str("DUBIZZLE_CARS_LISTING_URL", "https://dubai.dubizzle.com/motors/used-cars/") + max_pages_per_run: int = _env_int("DUBIZZLE_MAX_PAGES_PER_RUN", 9999) + max_vehicles_per_run: int = _env_int("DUBIZZLE_MAX_VEHICLES_PER_RUN", 50000) + page_link_limit: int = _env_int("DUBIZZLE_PAGE_LINK_LIMIT", 500) + include_pagination: bool = _env_bool("DUBIZZLE_INCLUDE_PAGINATION", True) + collect_current_page_only: bool = _env_bool("DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY", False) + # Порог раннего останова: если доля уже известных машин на странице >= этого значения, + # прекращаем листать — все новые машины уже найдены. 0 = отключено. + early_stop_threshold: float = _env_float("DUBIZZLE_EARLY_STOP_THRESHOLD", 0.8) + # Сегментация листинга по брендам для обхода лимита пагинации DUBIZZLE (~22 600 машин). + # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка. + # Пустая строка = без сегментации (backward compatible). + listing_segments_json: str = _env_str("DUBIZZLE_LISTING_SEGMENTS", "") + + +# Пагинационный потолок одного Algolia-запроса: ~100 страниц × 100 = 10 000 результатов. +DUBIZZLE_PAGINATION_CEILING = 10_000 + +# Авто-сегментация по году. Эти диапазоны подобраны так, чтобы каждый сегмент +# оставался ниже лимита Algolia и собирался быстрее, чем старая make/year схема. +_AUTO_YEAR_SPLITS: tuple[tuple[int, int], ...] = ( + (1900, 2012), + (2013, 2015), + (2016, 2017), + (2018, 2019), + (2020, 2021), + (2022, 2023), + (2024, 2025), + (2026, 2027), +) + + +def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]: + """Парсит DUBIZZLE_LISTING_SEGMENTS в список сегментов. + + Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None). + Специальное значение ``"auto"`` генерирует быстрые year-only сегменты, + которые гарантированно проходят через Algolia без упора в лимит ~10k. + """ + raw = raw.strip() + if not raw: + return [] + if raw.lower() == "auto": + return [ + {"make": None, "year_min": yr_min, "year_max": yr_max} + for yr_min, yr_max in _AUTO_YEAR_SPLITS + ] + try: + data = json.loads(raw) + except (json.JSONDecodeError, ValueError): + return [] + if not isinstance(data, list): + return [] + segments = [] + for item in data: + if isinstance(item, str): + segments.append({"make": item.upper(), "year_min": None, "year_max": None}) + elif isinstance(item, dict): + segments.append({ + "make": str(item.get("make") or "").upper() or None, + "year_min": int(item["year_min"]) if item.get("year_min") is not None else None, + "year_max": int(item["year_max"]) if item.get("year_max") is not None else None, + }) + return segments + + +# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) + +@dataclass(slots=True) +class DatabaseConfig: + url: str = _env_str("DUBIZZLE_DATABASE_URL", "postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper") + echo: bool = _env_bool("DUBIZZLE_DATABASE_ECHO", False) + pool_size: int = _env_int("DUBIZZLE_DATABASE_POOL_SIZE", 5) + max_overflow: int = _env_int("DUBIZZLE_DATABASE_MAX_OVERFLOW", 10) + pool_recycle_seconds: int = _env_int("DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS", 1800) + auto_create_tables: bool = _env_bool("DUBIZZLE_DATABASE_AUTO_CREATE_TABLES", False) + + +# --- Конфиг Redis (URL для Celery broker) --- + +@dataclass(slots=True) +class RedisConfig: + url: str = _env_str("DUBIZZLE_REDIS_URL", "redis://localhost:6379/0") + socket_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) + socket_connect_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) + health_check_interval_seconds: int = _env_int("DUBIZZLE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) + + +# --- Конфиг Discovery (режим обнаружения, hourly batch) --- + +@dataclass(slots=True) +class DiscoveryConfig: + mode: str = _env_str("DUBIZZLE_DISCOVERY_MODE", "algolia") + hourly_mode: str = _env_str("DUBIZZLE_HOURLY_MODE", "rolling_refresh") + hourly_refresh_batch_size: int = _env_int("DUBIZZLE_HOURLY_REFRESH_BATCH_SIZE", 500) + always_full_scan: bool = _env_bool("DUBIZZLE_ALWAYS_FULL_SCAN", False) + sitemap_fallback_on_empty: bool = _env_bool("DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY", False) + + +@dataclass(slots=True) +class AlgoliaConfig: + application_id: str = _env_str("DUBIZZLE_ALGOLIA_APPLICATION_ID", "WD0PTZ13ZS") + api_key: str = _env_str( + "DUBIZZLE_ALGOLIA_API_KEY", + "cef139620248f1bc328a00fddc7107a6", + ) + index_name: str = _env_str("DUBIZZLE_ALGOLIA_INDEX_NAME", "motors.com") + category_slug: str = _env_str("DUBIZZLE_ALGOLIA_CATEGORY_SLUG", "motors/used-cars") + base_url: str = _env_str("DUBIZZLE_ALGOLIA_BASE_URL", "https://WD0PTZ13ZS-dsn.algolia.net") + hits_per_page: int = _env_int("DUBIZZLE_ALGOLIA_HITS_PER_PAGE", 100) + + +# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- + +@dataclass(slots=True) +class CeleryConfig: + broker_url: str = _env_str("CELERY_BROKER_URL", "") + result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") + task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) + task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) + task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600) + worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) + worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) + broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) + beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) + beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None + batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) + parallel_tabs: int = _env_int("DUBIZZLE_PARALLEL_TABS", 8) + parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False) + block_resources: bool = _env_bool("DUBIZZLE_BLOCK_RESOURCES", True) + + +# --- Конфиг прокси (server, username, password) --- + +@dataclass(slots=True) +class ProxyConfig: + server: str | None = _env_optional_str("DUBIZZLE_PROXY_SERVER") + username: str | None = _env_optional_str("DUBIZZLE_PROXY_USERNAME") + password: str | None = _env_optional_str("DUBIZZLE_PROXY_PASSWORD") + + @property + def enabled(self) -> bool: + return bool(self.server) + + def to_playwright_dict(self) -> dict[str, str] | None: + if not self.server: + return None + result: dict[str, str] = {"server": self.server} + if self.username: + result["username"] = self.username + if self.password: + result["password"] = self.password + return result + + +# Главный объект настроек: собирает все блоки конфигурации + +@dataclass(slots=True) +class Settings: + home_url: str = "https://www.dubizzle.com/" + default_timeout_ms: int = _env_int("DUBIZZLE_TIMEOUT_MS", 45000) + network_settle_ms: int = _env_int("DUBIZZLE_NETWORK_SETTLE_MS", 400) + fast_path_timeout_ms: int = _env_int("DUBIZZLE_FAST_PATH_TIMEOUT_MS", 5000) + fast_path_max_attempts: int = _env_int("DUBIZZLE_FAST_PATH_MAX_ATTEMPTS", 1) + fallback_navigation_timeout_ms: int = _env_int("DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS", 15000) + max_retries: int = _env_int("DUBIZZLE_MAX_RETRIES", 3) + retry_delay_seconds: float = _env_float("DUBIZZLE_RETRY_DELAY_SECONDS", 2.5) + retry_backoff_multiplier: float = _env_float("DUBIZZLE_RETRY_BACKOFF_MULTIPLIER", 2.0) + retry_jitter_seconds: float = _env_float("DUBIZZLE_RETRY_JITTER_SECONDS", 0.25) + headless: bool = _env_bool("DUBIZZLE_HEADLESS", True) + browser_engine: str = _env_str("DUBIZZLE_BROWSER_ENGINE", "auto") + log_level: str = _env_str("DUBIZZLE_LOG_LEVEL", "INFO") + log_file: str | None = _env_optional_str("DUBIZZLE_LOG_FILE") + enable_trace_id_logs: bool = _env_bool("DUBIZZLE_ENABLE_TRACE_ID_LOGS", True) + sync_only_new: bool = _env_bool("DUBIZZLE_SYNC_ONLY_NEW", False) + raw_output_json: str | None = _env_optional_str("DUBIZZLE_RAW_OUTPUT_JSON") + tokens_file: str | None = _env_path_str("DUBIZZLE_TOKENS_FILE") + runtime_config_file: str | None = _env_path_str("DUBIZZLE_RUNTIME_CONFIG_FILE") + scheduler_interval_minutes: int = _env_int("DUBIZZLE_SCHEDULER_INTERVAL_MINUTES", 60) + fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) + capture: CaptureConfig = field(default_factory=CaptureConfig) + pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) + listing: ListingConfig = field(default_factory=ListingConfig) + database: DatabaseConfig = field(default_factory=DatabaseConfig) + redis: RedisConfig = field(default_factory=RedisConfig) + celery: CeleryConfig = field(default_factory=CeleryConfig) + proxy: ProxyConfig = field(default_factory=ProxyConfig) + discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) + algolia: AlgoliaConfig = field(default_factory=AlgoliaConfig) + + @property + def parallel_tabs(self) -> int: + return self.celery.parallel_tabs + + @property + def block_resources(self) -> bool: + return self.celery.block_resources + +# Глобальный синглтон — используется по умолчанию во всех модулях. +settings = Settings() diff --git a/iaai_scraper/core/exceptions.py b/dubizzle_scraper/core/exceptions.py similarity index 100% rename from iaai_scraper/core/exceptions.py rename to dubizzle_scraper/core/exceptions.py diff --git a/iaai_scraper/core/logs.py b/dubizzle_scraper/core/logs.py similarity index 100% rename from iaai_scraper/core/logs.py rename to dubizzle_scraper/core/logs.py diff --git a/iaai_scraper/core/retry.py b/dubizzle_scraper/core/retry.py similarity index 97% rename from iaai_scraper/core/retry.py rename to dubizzle_scraper/core/retry.py index 2fce32c..05a9370 100644 --- a/iaai_scraper/core/retry.py +++ b/dubizzle_scraper/core/retry.py @@ -9,7 +9,7 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError from .exceptions import AntiBotDetectedError -logger = logging.getLogger("iaai_scraper.retry") +logger = logging.getLogger("dubizzle_scraper.retry") # Типы исключений, при которых retry имеет смысл. RETRYABLE_EXCEPTIONS = ( diff --git a/iaai_scraper/core/runtime_config.py b/dubizzle_scraper/core/runtime_config.py similarity index 99% rename from iaai_scraper/core/runtime_config.py rename to dubizzle_scraper/core/runtime_config.py index 205a154..a05f6be 100644 --- a/iaai_scraper/core/runtime_config.py +++ b/dubizzle_scraper/core/runtime_config.py @@ -5,7 +5,7 @@ from pathlib import Path from typing import Any -logger = logging.getLogger("iaai_scraper.runtime_config") +logger = logging.getLogger("dubizzle_scraper.runtime_config") def _normalize_text(value: str) -> str: diff --git a/iaai_scraper/core/utils.py b/dubizzle_scraper/core/utils.py similarity index 100% rename from iaai_scraper/core/utils.py rename to dubizzle_scraper/core/utils.py diff --git a/iaai_scraper/discovery/__init__.py b/dubizzle_scraper/discovery/__init__.py similarity index 64% rename from iaai_scraper/discovery/__init__.py rename to dubizzle_scraper/discovery/__init__.py index 86402b2..f3bfca6 100644 --- a/iaai_scraper/discovery/__init__.py +++ b/dubizzle_scraper/discovery/__init__.py @@ -5,6 +5,11 @@ from .sitemap import ( discover_vehicle_urls_from_sitemap, discover_vehicle_urls_from_sitemap_with_stats, ) +from .algolia import ( + AlgoliaDiscoveryError, + AlgoliaDiscoveryResult, + discover_vehicle_hits_from_algolia, +) __all__ = [ "SitemapDiscoveryError", @@ -12,4 +17,7 @@ __all__ = [ "SitemapDiscoveryStats", "discover_vehicle_urls_from_sitemap", "discover_vehicle_urls_from_sitemap_with_stats", + "AlgoliaDiscoveryError", + "AlgoliaDiscoveryResult", + "discover_vehicle_hits_from_algolia", ] diff --git a/dubizzle_scraper/discovery/algolia.py b/dubizzle_scraper/discovery/algolia.py new file mode 100644 index 0000000..1ed80f2 --- /dev/null +++ b/dubizzle_scraper/discovery/algolia.py @@ -0,0 +1,479 @@ +from __future__ import annotations + +import json +import logging +import time +from dataclasses import dataclass, field +from typing import Any +from urllib.parse import parse_qs, urlencode, urlparse +from urllib.request import Request, urlopen + + +logger = logging.getLogger("dubizzle_scraper.discovery.algolia") + +ALGOLIA_HARD_PAGE_LIMIT = 100 +ALGOLIA_MAX_HITS_PER_QUERY = ALGOLIA_HARD_PAGE_LIMIT * 100 +ALGOLIA_ID_RANGE_START = 0 +ALGOLIA_ID_RANGE_END = 20_000_000 +ALGOLIA_ID_RANGE_MIN_WINDOW = 100 + + +class AlgoliaDiscoveryError(RuntimeError): + pass + + +@dataclass(slots=True) +class AlgoliaDiscoveryPageStat: + page_number: int + hits_count: int + + +@dataclass(slots=True) +class AlgoliaDiscoveryResult: + vehicle_urls: list[str] = field(default_factory=list) + hit_records: dict[str, dict[str, Any]] = field(default_factory=dict) + origin_ids_by_url: dict[str, str] = field(default_factory=dict) + pages: list[dict[str, int]] = field(default_factory=list) + early_stopped: bool = False + truncated_by_time_budget: bool = False + total_hits: int = 0 + + +@dataclass(slots=True) +class _AlgoliaShard: + category_slug: str + id_min: int | None = None + id_max: int | None = None + year_min: int | None = None + year_max: int | None = None + + def filter_expr(self) -> str: + parts = [f"(category_v2.slug_paths:{self.category_slug})"] + if self.year_min is not None: + parts.append(f"year>={int(self.year_min)}") + if self.year_max is not None: + parts.append(f"year<={int(self.year_max)}") + if self.id_min is not None: + parts.append(f"id>={int(self.id_min)}") + if self.id_max is not None: + parts.append(f"id<={int(self.id_max)}") + return " AND ".join(parts) + + def label(self) -> str: + label = self.category_slug + if self.year_min is not None or self.year_max is not None: + label += f"[year:{self.year_min}-{self.year_max}]" + if self.id_min is None and self.id_max is None: + return label + return f"{label}[id:{self.id_min}-{self.id_max}]" + + +@dataclass(slots=True) +class _AlgoliaHttpClient: + endpoint: str + app_id: str + api_key: str + user_agent: str + index_name: str + hits_per_page: int + + def request(self, *, page: int, filters: str) -> dict[str, Any]: + params = urlencode( + { + "query": "", + "page": page, + "hitsPerPage": self.hits_per_page, + "filters": filters, + } + ) + payload = {"requests": [{"indexName": self.index_name, "params": params}]} + request = Request( + self.endpoint, + data=json.dumps(payload).encode("utf-8"), + headers={ + "content-type": "application/json", + "x-algolia-application-id": self.app_id, + "x-algolia-api-key": self.api_key, + "accept": "application/json", + "user-agent": self.user_agent, + }, + method="POST", + ) + with urlopen(request, timeout=30) as response: + body = response.read().decode("utf-8", errors="ignore") + parsed = json.loads(body) + results = parsed.get("results") or [] + return results[0] if results and isinstance(results[0], dict) else {} + + +def _build_origin_id_from_hit(hit: dict[str, Any]) -> str | None: + for key in ("id", "objectID", "uuid"): + value = hit.get(key) + if value is None: + continue + text = str(value).strip() + if text: + return f"dubizzle:{text}" + permalink = str(hit.get("permalink") or "").strip() + if permalink: + tail = permalink.rstrip("/").split("/")[-1] + if tail: + return f"dubizzle:{tail}" + return None + + +def _build_vehicle_url_from_hit(hit: dict[str, Any]) -> str | None: + permalink = str(hit.get("permalink") or "").strip() + if permalink: + if permalink.startswith("http://") or permalink.startswith("https://"): + return permalink + if permalink.startswith("/"): + return f"https://www.dubizzle.com{permalink}" + return f"https://www.dubizzle.com/{permalink.lstrip('/')}" + + short = str(hit.get("short_url") or "").strip() + if short: + if short.startswith("http://") or short.startswith("https://"): + return short + return f"https://dubizzle.com/s/{short.strip('/')}" + + hit_id = hit.get("id") or hit.get("objectID") + if hit_id is not None: + return f"https://www.dubizzle.com/motors/used-cars/ad-{hit_id}/" + return None + + +def _extract_make_from_listing_url(listing_url: str | None) -> str | None: + if not listing_url: + return None + try: + parsed = urlparse(listing_url) + params = parse_qs(parsed.query) + candidate = (params.get("Make") or params.get("make") or [None])[0] + if candidate: + return str(candidate).strip() + parts = [p for p in parsed.path.split("/") if p] + if len(parts) >= 3 and parts[0].lower() == "motors" and parts[1].lower() == "used-cars": + slug = parts[2].strip().lower() + if slug and slug != "s": + return slug.replace("-", " ") + except Exception: + return None + return None + + +def _make_slug(make: str | None) -> str | None: + if not make: + return None + slug = make.strip().lower().replace(" ", "-") + return slug or None + + +def _hit_matches_filters( + hit: dict[str, Any], + *, + make: str | None, + model: str | None, + year_min: int | None, + year_max: int | None, +) -> bool: + if make: + hit_make = str(hit.get("make") or "").strip().lower() + if hit_make != make.strip().lower(): + return False + if model: + hit_model = str(hit.get("model") or "").strip().lower() + if hit_model != model.strip().lower(): + return False + + hit_year_raw = hit.get("year") + hit_year: int | None = None + if hit_year_raw is not None: + try: + hit_year = int(hit_year_raw) + except Exception: + hit_year = None + + if year_min is not None and (hit_year is None or hit_year < year_min): + return False + if year_max is not None and (hit_year is None or hit_year > year_max): + return False + + return True + + +def _probe_total_hits(client: _AlgoliaHttpClient, shard: _AlgoliaShard) -> int: + first = client.request(page=0, filters=shard.filter_expr()) + return int(first.get("nbHits") or 0) + + +def _split_id_range(shard: _AlgoliaShard) -> tuple[_AlgoliaShard, _AlgoliaShard] | None: + lo = shard.id_min if shard.id_min is not None else ALGOLIA_ID_RANGE_START + hi = shard.id_max if shard.id_max is not None else ALGOLIA_ID_RANGE_END + if hi - lo <= ALGOLIA_ID_RANGE_MIN_WINDOW: + return None + mid = (lo + hi) // 2 + return ( + _AlgoliaShard( + category_slug=shard.category_slug, + id_min=lo, + id_max=mid, + year_min=shard.year_min, + year_max=shard.year_max, + ), + _AlgoliaShard( + category_slug=shard.category_slug, + id_min=mid + 1, + id_max=hi, + year_min=shard.year_min, + year_max=shard.year_max, + ), + ) + + +def _expand_shards(client: _AlgoliaHttpClient, initial_shard: _AlgoliaShard, max_duration_seconds: float | None, started_at: float) -> tuple[list[_AlgoliaShard], bool, int]: + queue: list[_AlgoliaShard] = [initial_shard] + ready: list[_AlgoliaShard] = [] + truncated = False + total_hits = 0 + + while queue: + if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds: + truncated = True + break + shard = queue.pop(0) + shard_total = _probe_total_hits(client, shard) + if shard is initial_shard: + total_hits = shard_total + if shard_total == 0: + continue + if shard_total <= ALGOLIA_MAX_HITS_PER_QUERY: + ready.append(shard) + continue + split = _split_id_range(shard) + if split is None: + logger.warning( + "Shard %s still exceeds limit=%d but id-window is too small to split further (hits=%d)", + shard.label(), + ALGOLIA_MAX_HITS_PER_QUERY, + shard_total, + ) + ready.append(shard) + continue + logger.warning( + "Splitting Algolia shard %s with hits=%d into %s and %s", + shard.label(), + shard_total, + split[0].label(), + split[1].label(), + ) + queue.insert(0, split[1]) + queue.insert(0, split[0]) + + return ready, truncated, total_hits + + +def _fetch_shard_hits( + *, + client: _AlgoliaHttpClient, + shard: _AlgoliaShard, + make: str | None, + model: str | None, + year_min: int | None, + year_max: int | None, + known_origin_ids: set[str] | None, + threshold: float, + max_duration_seconds: float | None, + started_at: float, +) -> tuple[list[str], dict[str, dict[str, Any]], dict[str, str], list[dict[str, int]], bool, bool]: + urls: list[str] = [] + hit_records: dict[str, dict[str, Any]] = {} + origin_ids_by_url: dict[str, str] = {} + pages: list[dict[str, int]] = [] + early_stopped = False + truncated_by_time_budget = False + page = 0 + nb_pages = None + + while True: + if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds: + truncated_by_time_budget = True + break + + try: + first = client.request(page=page, filters=shard.filter_expr()) + except Exception as exc: + raise AlgoliaDiscoveryError( + f"Algolia request failed for shard={shard.label()} page={page}: {exc}" + ) from exc + + hits = first.get("hits") or [] + if not isinstance(hits, list): + hits = [] + + if page == 0: + nb_pages = min(int(first.get("nbPages") or 0), ALGOLIA_HARD_PAGE_LIMIT) + + pages.append({"page_number": page + 1, "links_found": len(hits), "shard": shard.label()}) + if not hits: + break + + page_known = 0 + page_new = 0 + for raw_hit in hits: + if not isinstance(raw_hit, dict): + continue + if not _hit_matches_filters( + raw_hit, + make=make, + model=model, + year_min=year_min, + year_max=year_max, + ): + continue + + url = _build_vehicle_url_from_hit(raw_hit) + if not url: + continue + origin_id = _build_origin_id_from_hit(raw_hit) + if origin_id and known_origin_ids is not None and origin_id in known_origin_ids: + page_known += 1 + else: + page_new += 1 + + if url in hit_records: + continue + urls.append(url) + hit_records[url] = raw_hit + if origin_id: + origin_ids_by_url[url] = origin_id + + if known_origin_ids is not None and threshold > 0 and (page_known + page_new) > 0: + ratio = page_known / (page_known + page_new) + if ratio >= threshold and page_new == 0: + early_stopped = True + break + + page += 1 + if nb_pages is not None and page >= nb_pages: + break + + return urls, hit_records, origin_ids_by_url, pages, early_stopped, truncated_by_time_budget + + +def discover_vehicle_hits_from_algolia( + *, + settings, + make: str | None = None, + model: str | None = None, + limit: int | None = None, + year_min: int | None = None, + year_max: int | None = None, + listing_url: str | None = None, + known_origin_ids: set[str] | None = None, + max_duration_seconds: float | None = None, +) -> AlgoliaDiscoveryResult: + app_id = settings.algolia.application_id.strip() + api_key = settings.algolia.api_key.strip() + index_name = settings.algolia.index_name.strip() + if not app_id or not api_key or not index_name: + raise AlgoliaDiscoveryError("Algolia credentials/index are not configured") + + effective_make = make or _extract_make_from_listing_url(listing_url) + hits_per_page = max(1, min(100, int(settings.algolia.hits_per_page))) + base_url = settings.algolia.base_url.strip().rstrip("/") + if not base_url: + base_url = f"https://{app_id}-dsn.algolia.net" + + category_slug = settings.algolia.category_slug.strip() or "motors/used-cars" + make_slug = _make_slug(effective_make) + if make_slug: + category_slug = f"{category_slug}/{make_slug}" + + client = _AlgoliaHttpClient( + endpoint=f"{base_url}/1/indexes/*/queries", + app_id=app_id, + api_key=api_key, + user_agent=settings.fingerprint.user_agent, + index_name=index_name, + hits_per_page=hits_per_page, + ) + threshold = float(settings.listing.early_stop_threshold) + started_at = time.perf_counter() + + initial_shard = _AlgoliaShard( + category_slug=category_slug, + id_min=ALGOLIA_ID_RANGE_START, + id_max=ALGOLIA_ID_RANGE_END, + year_min=year_min, + year_max=year_max, + ) + shards, shard_build_truncated, total_hits = _expand_shards( + client, + initial_shard, + max_duration_seconds, + started_at, + ) + + collected_urls: list[str] = [] + hits_by_url: dict[str, dict[str, Any]] = {} + origin_ids_by_url: dict[str, str] = {} + pages: list[dict[str, int]] = [] + early_stopped = False + truncated_by_time_budget = shard_build_truncated + + logger.warning( + "Algolia shard plan ready: total_hits=%d shards=%d category=%s", + total_hits, + len(shards), + category_slug, + ) + + for shard in shards: + shard_urls, shard_hits, shard_origin_ids, shard_pages, shard_early_stop, shard_truncated = _fetch_shard_hits( + client=client, + shard=shard, + make=effective_make, + model=model, + year_min=year_min, + year_max=year_max, + known_origin_ids=known_origin_ids, + threshold=threshold, + max_duration_seconds=max_duration_seconds, + started_at=started_at, + ) + pages.extend(shard_pages) + early_stopped = early_stopped or shard_early_stop + truncated_by_time_budget = truncated_by_time_budget or shard_truncated + for url in shard_urls: + if url in hits_by_url: + continue + collected_urls.append(url) + hits_by_url[url] = shard_hits[url] + if url in shard_origin_ids: + origin_ids_by_url[url] = shard_origin_ids[url] + if limit is not None and limit > 0 and len(collected_urls) >= limit: + break + if truncated_by_time_budget: + break + + logger.info( + "Algolia discovery done: urls=%d total_hits=%d pages=%d shards=%d", + len(collected_urls), + total_hits, + len(pages), + len(shards), + ) + + if limit is not None and limit > 0 and len(collected_urls) > limit: + collected_urls = collected_urls[:limit] + + return AlgoliaDiscoveryResult( + vehicle_urls=collected_urls, + hit_records={url: hits_by_url[url] for url in collected_urls if url in hits_by_url}, + origin_ids_by_url={url: origin_ids_by_url[url] for url in collected_urls if url in origin_ids_by_url}, + pages=pages, + early_stopped=early_stopped, + truncated_by_time_budget=truncated_by_time_budget, + total_hits=total_hits, + ) diff --git a/iaai_scraper/discovery/sitemap.py b/dubizzle_scraper/discovery/sitemap.py similarity index 98% rename from iaai_scraper/discovery/sitemap.py rename to dubizzle_scraper/discovery/sitemap.py index dc50104..6811c82 100644 --- a/iaai_scraper/discovery/sitemap.py +++ b/dubizzle_scraper/discovery/sitemap.py @@ -10,9 +10,9 @@ from urllib.parse import urlsplit, urlunsplit from urllib.request import Request, urlopen, ProxyHandler, build_opener import xml.etree.ElementTree as ET -logger = logging.getLogger("iaai_scraper.discovery.sitemap") +logger = logging.getLogger("dubizzle_scraper.discovery.sitemap") -DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" +DEFAULT_SITEMAP_INDEX_URL = "https://www.dubizzle.com/Xj9rDOVMEi0hc38S/sitemap_index.xml" _SITEMAP_TIMEOUT_SECONDS = 30 _LOC_TAG_RE = re.compile(rb"\s*(.*?)\s*", re.IGNORECASE | re.DOTALL) diff --git a/iaai_scraper/parsing/__init__.py b/dubizzle_scraper/parsing/__init__.py similarity index 100% rename from iaai_scraper/parsing/__init__.py rename to dubizzle_scraper/parsing/__init__.py diff --git a/iaai_scraper/parsing/mapper.py b/dubizzle_scraper/parsing/mapper.py similarity index 60% rename from iaai_scraper/parsing/mapper.py rename to dubizzle_scraper/parsing/mapper.py index b31309b..84bd57d 100644 --- a/iaai_scraper/parsing/mapper.py +++ b/dubizzle_scraper/parsing/mapper.py @@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord class CarMapper: - # Маппер IAAI в CarRecord. + # Маппер DUBIZZLE в CarRecord. BODY_MAP = { "sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV", @@ -44,6 +44,7 @@ class CarMapper: COUNTRY_MAP = { "us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA", "jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR", + "ae": "AE", "uae": "AE", "united arab emirates": "AE", "dubai": "AE", "abu dhabi": "AE", } NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"} @@ -59,9 +60,14 @@ class CarMapper: origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core) parser_id = self._generate_parser_id(origin_id) - brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN" - model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN" - year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")])) + brand = self._resolve_make(core, vehicle_summary) or "UNKNOWN" + model = self._resolve_model(core, vehicle_summary) or "UNKNOWN" + year = self._to_year(first_non_empty([ + core.get("year"), + vehicle_summary.get("year"), + self._extract_detail_v2_value(vehicle_summary, "year"), + self._extract_detail_value(vehicle_summary, "Year"), + ])) price = self._first_parsed_int( [ pricing.get("buy_now"), @@ -69,29 +75,90 @@ class CarMapper: vehicle_summary.get("buy_now"), vehicle_summary.get("current_bid"), pricing.get("actual_cash_value"), + vehicle_summary.get("price"), + self._extract_detail_v2_value(vehicle_summary, "price"), + self._extract_detail_value(vehicle_summary, "Price"), ], self._to_money_int, ) mileage = self._parse_odometer( - first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")]) + first_non_empty([ + core.get("odometer"), + core.get("kilometers"), + vehicle_summary.get("odometer"), + vehicle_summary.get("kilometers"), + self._extract_detail_v2_value(vehicle_summary, "kilometers"), + self._extract_detail_value(vehicle_summary, "Kilometers"), + ]) ) - color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"])) - drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")])) + color = self._normalize_color(first_non_empty([ + core.get("color"), + vehicle_summary.get("color"), + vehicle_summary.get("exterior_color"), + self._extract_detail_v2_value(vehicle_summary, "exterior_color"), + self._extract_detail_value(vehicle_summary, "Exterior Color"), + "other", + ])) + drive = self._normalize_drive(first_non_empty([ + core.get("drive"), + vehicle_summary.get("drive"), + vehicle_summary.get("drive_type"), + self._extract_detail_v2_value(vehicle_summary, "drive_system"), + self._extract_detail_value(vehicle_summary, "Drive Type"), + self._extract_detail_value(vehicle_summary, "Drive System"), + ])) # Пробуем взять привод из двигателя. if not drive or drive == "NA": - engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")])) + engine_text = self._as_str(first_non_empty([ + core.get("engine"), + vehicle_summary.get("engine"), + self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"), + self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"), + ])) if engine_text: inferred_drive = self._normalize_drive(engine_text) if inferred_drive and inferred_drive != "NA": drive = inferred_drive - gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")])) - steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT" - body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")])) - engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")])) + gearbox = self._normalize_gearbox(first_non_empty([ + core.get("gearbox"), + vehicle_summary.get("gearbox"), + vehicle_summary.get("transmission_type"), + vehicle_summary.get("transmission"), + self._extract_detail_v2_value(vehicle_summary, "transmission_type"), + self._extract_detail_value(vehicle_summary, "Transmission Type"), + ])) + steering = self._normalize_steering(first_non_empty([ + core.get("steering_wheel"), + vehicle_summary.get("steering_wheel"), + self._extract_detail_v2_value(vehicle_summary, "steering_side"), + self._extract_detail_value(vehicle_summary, "Steering Side"), + ])) or "LEFT" + body_type = self._normalize_body_type(first_non_empty([ + core.get("body_type"), + vehicle_summary.get("body_type"), + self._extract_detail_v2_value(vehicle_summary, "body_type"), + self._extract_detail_value(vehicle_summary, "Body Type"), + ])) + engine_volume = self._to_engine_cc(first_non_empty([ + core.get("engine"), + core.get("engine_volume"), + vehicle_summary.get("engine"), + vehicle_summary.get("engine_volume"), + self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"), + self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"), + ])) title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""])) seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""])) - location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""])) - country = self._normalize_country(first_non_empty([core.get("country"), location, "US"])) + location = self._as_str(first_non_empty([ + core.get("location"), + vehicle_summary.get("location"), + vehicle_summary.get("location_name"), + self._extract_nested_text(vehicle_summary.get("neighbourhood"), "en"), + self._extract_location_country(vehicle_summary), + auction.get("branch"), + "", + ])) + country = self._normalize_country(first_non_empty([core.get("country"), location, "AE"])) is_damaged = self._bool_damage(damage, vehicle_summary) is_sold = self._bool_sold(auction) one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False])) @@ -115,13 +182,13 @@ class CarMapper: ) slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")]))) images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or []) - origin = "IAAI" + origin = "DUBIZZLE" return CarRecord( parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency, mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox, steering_wheel=steering, body_type=body_type, engine_volume=engine_volume, - selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car, + selling_type=self._normalize_selling_type(first_non_empty([core.get("selling_type"), "STOCK"])), one_owner=one_owner, new_car=new_car, is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged, evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history, slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records, @@ -131,6 +198,146 @@ class CarMapper: def _as_str(value: Any) -> str: return "" if value is None else str(value).strip() + def _resolve_make(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None: + category_make, _ = self._extract_category_make_model(vehicle_summary) + slug_make, _ = self._extract_slug_make_model(vehicle_summary) + return self._first_text( + [ + core.get("make"), + vehicle_summary.get("make"), + self._extract_detail_v2_value(vehicle_summary, "make"), + self._extract_detail_value(vehicle_summary, "Make"), + category_make, + slug_make, + ] + ) + + def _resolve_model(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None: + _, category_model = self._extract_category_make_model(vehicle_summary) + _, slug_model = self._extract_slug_make_model(vehicle_summary) + return self._first_text( + [ + core.get("model"), + vehicle_summary.get("model"), + self._extract_detail_v2_value(vehicle_summary, "model"), + self._extract_detail_value(vehicle_summary, "Model"), + category_model, + slug_model, + ] + ) + + def _first_text(self, values: list[Any]) -> str | None: + for value in values: + text = self._coerce_text(value) + if text: + return text + return None + + def _coerce_text(self, value: Any) -> str | None: + if value is None: + return None + if isinstance(value, list): + for item in value: + text = self._coerce_text(item) + if text: + return text + return None + if isinstance(value, dict): + for key in ("en", "value", "name", "text", "label"): + if key in value: + text = self._coerce_text(value.get(key)) + if text: + return text + for nested in value.values(): + text = self._coerce_text(nested) + if text: + return text + return None + text = self._as_str(value) + return text or None + + def _extract_nested_text(self, value: Any, preferred_key: str = "en") -> str | None: + if not isinstance(value, dict): + return self._coerce_text(value) + return self._coerce_text(value.get(preferred_key)) or self._coerce_text(value) + + def _extract_detail_v2_value(self, vehicle_summary: dict[str, Any], slug: str) -> str | None: + details_v2 = vehicle_summary.get("details_v2") + if not isinstance(details_v2, dict): + return None + target = slug.strip().lower() + for section in details_v2.values(): + if not isinstance(section, list): + continue + for item in section: + if not isinstance(item, dict): + continue + if self._as_str(item.get("slug")).lower() != target: + continue + text = self._coerce_text(item.get("value")) + if text: + return text + return None + + def _extract_detail_value(self, vehicle_summary: dict[str, Any], detail_key: str) -> str | None: + details = vehicle_summary.get("details") + if not isinstance(details, dict): + return None + target = detail_key.strip().lower() + for key, value in details.items(): + if self._as_str(key).lower() != target: + continue + text = self._coerce_text(value) + if text: + return text + return None + + def _extract_category_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]: + category_v2 = vehicle_summary.get("category_v2") + if isinstance(category_v2, dict): + names_en = category_v2.get("names_en") + if isinstance(names_en, list) and len(names_en) >= 4: + return self._coerce_text(names_en[2]), self._coerce_text(names_en[3]) + + category = vehicle_summary.get("category") + if isinstance(category, dict): + names_en = category.get("en") + if isinstance(names_en, list) and len(names_en) >= 3: + return self._coerce_text(names_en[1]), self._coerce_text(names_en[2]) + + return None, None + + def _extract_slug_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]: + category_v2 = vehicle_summary.get("category_v2") + if not isinstance(category_v2, dict): + return None, None + slug_paths = category_v2.get("slug_paths") + if not isinstance(slug_paths, list) or len(slug_paths) < 3: + return None, None + make = self._humanize_slug_path_part(slug_paths[2]) + model = self._humanize_slug_path_part(slug_paths[3]) if len(slug_paths) >= 4 else None + return make, model + + def _humanize_slug_path_part(self, value: Any) -> str | None: + text = self._as_str(value) + if not text: + return None + slug = text.split("/")[-1].strip().strip("-") + if not slug: + return None + return slug.replace("-", " ").title() + + def _extract_location_country(self, vehicle_summary: dict[str, Any]) -> str | None: + site = vehicle_summary.get("site") + if isinstance(site, dict): + return self._coerce_text(site.get("en")) + location_list = vehicle_summary.get("location_list") + if isinstance(location_list, dict): + en_values = location_list.get("en") + if isinstance(en_values, list) and en_values: + return self._coerce_text(en_values[0]) + return None + @staticmethod def _to_int(value: Any) -> int | None: if value is None: @@ -238,7 +445,7 @@ class CarMapper: return None if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text): return int(float(liters_match.group(1)) * 1000) - if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text): + if cubic_match := re.search(r"(\d{3,5})(?:\+)?\s*(?:cc|cm3|cubic)", text): return int(cubic_match.group(1)) return int(text) if re.match(r"^\d+$", text) else None @@ -278,12 +485,16 @@ class CarMapper: return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER" def _normalize_country(self, value: Any) -> str: - fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA" - return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback + fallback = "AE" if "AE" in COUNTRY_ENUM_VALUES else "NA" + return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="AE", fallback=fallback) or fallback def _normalize_selling_type(self, value: Any) -> str: - text = self._as_str(value) or "AUCTION" - return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION" + text = (self._as_str(value) or "STOCK").upper() + if text in SELLING_TYPE_ENUM_VALUES: + return text + if text in {"DEALER", "PRIVATE", "CLASSIFIED"}: + return "STOCK" + return "STOCK" def _map_value( self, @@ -357,13 +568,13 @@ class CarMapper: @staticmethod def _make_fullres_url(url: str) -> str: - if "vis.iaai.com" in url: + if "vis.dubizzle.com" in url: return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url)) return url @staticmethod def _make_preview_url(url: str) -> str: - if "vis.iaai.com" in url: + if "vis.dubizzle.com" in url: preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url)) if preview != url: return preview @@ -387,16 +598,22 @@ class CarMapper: return "car-" + "".join(chars) def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str: - # Формат: iaai:{lot_number}. - for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]: + # Формат: dubizzle:{lot_number}. + for value in [ + core.get("lot_number"), + vehicle_summary.get("lot_number"), + vehicle_summary.get("id"), + vehicle_summary.get("objectID"), + vehicle_summary.get("uuid"), + ]: text = self._as_str(value) if text: - return f"iaai:{text}" + return f"dubizzle:{text}" tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1] if "~" in tail: tail = tail.split("~")[0] raw = tail or self._slugify(vehicle_url) - return f"iaai:{raw}" + return f"dubizzle:{raw}" @staticmethod def _slugify(value: str) -> str: diff --git a/iaai_scraper/parsing/parser.py b/dubizzle_scraper/parsing/parser.py similarity index 97% rename from iaai_scraper/parsing/parser.py rename to dubizzle_scraper/parsing/parser.py index 4d3fee8..a9356c0 100644 --- a/iaai_scraper/parsing/parser.py +++ b/dubizzle_scraper/parsing/parser.py @@ -6,7 +6,7 @@ from typing import Any from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty -logger = logging.getLogger("iaai_scraper.parsers") +logger = logging.getLogger("dubizzle_scraper.parsers") class VehicleParser: @@ -350,7 +350,7 @@ class VehicleParser: if isinstance(item, str) and item.startswith("http"): cleaned = html_module.unescape(item) lowered = cleaned.lower() - if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned: continue if cleaned not in seen_flat: seen_flat.add(cleaned) @@ -360,7 +360,7 @@ class VehicleParser: if isinstance(child, str) and child.startswith("http"): cleaned = html_module.unescape(child) lowered = cleaned.lower() - if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned: + if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned: continue if cleaned not in seen_flat: seen_flat.add(cleaned) @@ -374,13 +374,13 @@ class VehicleParser: if vehicle_key and vehicle_key in url: seen_flat.add(url) flat.append(url) - elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]): + elif any(token in lowered for token in ["vis.dubizzle.com", "anvis", "vehicleimage"]): if vehicle_key and vehicle_key not in url: continue seen_flat.add(url) flat.append(url) if vehicle_key: - for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""): + for url in re.findall(r'https?://vis\.dubizzle\.com[^\s"\'<>]+', html or ""): cleaned = html_module.unescape(url) if cleaned not in seen_flat and vehicle_key in cleaned: seen_flat.add(cleaned) @@ -390,7 +390,7 @@ class VehicleParser: lowered = url.lower() if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}): continue - if "vis.iaai.com" in lowered and "/resizer" not in lowered: + if "vis.dubizzle.com" in lowered and "/resizer" not in lowered: continue filtered.append(url) return filtered diff --git a/iaai_scraper/proxy_bridge.py b/dubizzle_scraper/proxy_bridge.py similarity index 100% rename from iaai_scraper/proxy_bridge.py rename to dubizzle_scraper/proxy_bridge.py diff --git a/iaai_scraper/scraper.py b/dubizzle_scraper/scraper.py similarity index 73% rename from iaai_scraper/scraper.py rename to dubizzle_scraper/scraper.py index 6c65c6f..9d95463 100644 --- a/iaai_scraper/scraper.py +++ b/dubizzle_scraper/scraper.py @@ -29,14 +29,19 @@ from .core.logs import set_trace_id, setup_logging from .core.retry import retryable from .core.runtime_config import RuntimeConfig from .core.utils import save_to_json +from .discovery import AlgoliaDiscoveryError, discover_vehicle_hits_from_algolia +from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError from .parsing.mapper import CarMapper from .parsing.parser import VehicleParser from .storage.db import PersistenceService from .browser.listing import ListingCollector, ListingPageResult from .storage.schemas import CarRecord -logger = logging.getLogger("iaai_scraper.scraper") -VEHICLE_ID_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) +logger = logging.getLogger("dubizzle_scraper.scraper") +VEHICLE_ID_RE = re.compile( + r"(?:/VehicleDetail/(?P\d+)(?:~[A-Z]{2})?)|(?:---(?P[a-f0-9]{32})/?$)|(?:/ad-(?P\d+)/?)", + re.IGNORECASE, +) HTML_TAG_RE = re.compile(r"<[^>]+>") SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | re.DOTALL) @@ -44,8 +49,8 @@ SCRIPT_STYLE_RE = re.compile(r"<(script|style)[^>]*>.*?", re.IGNORECASE | r _PAGE_COLLECT_TIMEOUT_S = 90 _PAGE_NEXT_TIMEOUT_S = 60 # Ротация контекста выключена по умолчанию. -_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("IAAI_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) -_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("IAAI_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3) +_CONTEXT_ROTATE_EVERY_PAGES = int(os.environ.get("DUBIZZLE_CONTEXT_ROTATE_EVERY_PAGES", "0") or 0) +_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT = int(os.environ.get("DUBIZZLE_MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT", "3") or 3) _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS = 120 _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE = 2 @@ -112,7 +117,7 @@ class _ProgressHeartbeat: def _run(self) -> None: while not self._stop.wait(self._interval_s): - self._report_progress(self._stage, **self._meta) + self._report_progress(self._stage, heartbeat_only=True, **self._meta) def __enter__(self): self._thread = Thread(target=self._run, name=f"progress-heartbeat-{self._stage}", daemon=True) @@ -126,7 +131,7 @@ class _ProgressHeartbeat: return False -class IAAIScraper: +class DUBIZZLEScraper: @staticmethod def _is_protection_or_network_error(exc: Exception) -> bool: @@ -162,10 +167,10 @@ class IAAIScraper: has_identity = bool(summary.get("lot_number") or summary.get("make") or summary.get("model")) if (dom_hints.get("has_captcha_text") or dom_hints.get("has_antibot_text")) and not has_identity: - raise AntiBotDetectedError(f"IAAI anti-bot detected for {vehicle_url}") + raise AntiBotDetectedError(f"DUBIZZLE anti-bot detected for {vehicle_url}") if (access_notes.get("possible_captcha") or access_notes.get("possible_antibot")) and not has_identity: - raise AntiBotDetectedError(f"IAAI blocked or challenged request for {vehicle_url}") + raise AntiBotDetectedError(f"DUBIZZLE blocked or challenged request for {vehicle_url}") if not has_identity: raise SiteStructureChangedError(f"Vehicle page returned no recognizable vehicle data: {vehicle_url}") @@ -175,14 +180,14 @@ class IAAIScraper: match = VEHICLE_ID_RE.search(vehicle_url) if not match: return None - return match.group(1) + return match.group("veh_id") or match.group("slug_id") or match.group("ad_id") @staticmethod def _extract_db_origin_id_from_url(vehicle_url: str) -> str | None: - raw_id = IAAIScraper._extract_origin_id_from_url(vehicle_url) + raw_id = DUBIZZLEScraper._extract_origin_id_from_url(vehicle_url) if not raw_id: return None - return f"iaai:{raw_id}" + return f"dubizzle:{raw_id}" @staticmethod def _normalize_vehicle_url(vehicle_url: str) -> str: @@ -258,11 +263,7 @@ class IAAIScraper: except Exception: pass - def __enter__(self) -> "IAAIScraper": - if self.playwright is None: - self.playwright = sync_playwright().start() - if self.browser is None: - self.browser = self.browser_factory.create_browser(self.playwright) + def __enter__(self) -> "DUBIZZLEScraper": return self def __exit__(self, exc_type, exc, tb) -> None: @@ -300,7 +301,9 @@ class IAAIScraper: def _new_context(self) -> BrowserContext: if self.browser is None: - self.__enter__() + if self.playwright is None: + self.playwright = sync_playwright().start() + self.browser = self.browser_factory.create_browser(self.playwright) if self.context: try: self.context.close() @@ -386,9 +389,17 @@ class IAAIScraper: @staticmethod def _build_segment_listing_url(base_url: str, make: str | None) -> str: - """Построить URL листинга для сегмента. Make передаётся через query-параметр.""" + """Построить URL листинга для сегмента Dubizzle. + + Для актуальных URL используем path-формат `/motors/used-cars/{make}/`. + Для legacy URL (`Vehiclelisting`) оставляем query-параметр `?Make=`. + """ if not make: return base_url + if "motors/used-cars" in base_url.lower() and "vehiclelisting" not in base_url.lower(): + normalized_base = base_url.rstrip("/") + make_slug = make.strip().lower().replace(" ", "-") + return f"{normalized_base}/{make_slug}/" sep = "&" if "?" in base_url else "?" return f"{base_url}{sep}Make={make.replace(' ', '%20')}" @@ -549,22 +560,264 @@ class IAAIScraper: known_origin_ids: set[str] | None = None, max_duration_seconds: float | None = None, ): - page = self._get_page_with_warmup() - - try: - listing = self.listing_collector.collect_listing_links( - page, - make=make, - model=model, - known_origin_ids=known_origin_ids, - max_duration_seconds=max_duration_seconds, - ) - finally: - page.close() - + result = discover_vehicle_hits_from_algolia( + settings=self.settings, + make=make, + model=model, + limit=None, + year_min=None, + year_max=None, + listing_url=None, + known_origin_ids=known_origin_ids, + max_duration_seconds=max_duration_seconds, + ) return { "status": "ok", - **listing, + "vehicles_collected": len(result.vehicle_urls), + "vehicle_urls": result.vehicle_urls, + "early_stopped": result.early_stopped, + "truncated_by_time_budget": result.truncated_by_time_budget, + "pages": result.pages, + "total_hits": result.total_hits, + } + + @staticmethod + def _extract_listing_items_from_page(page: Page) -> list[dict[str, Any]]: + try: + items = page.evaluate( + r''' + () => { + const normalizeItem = (item) => { + if (!item || typeof item !== 'object') return null; + return { + url: item.url || null, + name: item.name || null, + description: item.description || null, + brand: item.brand && typeof item.brand === 'object' ? (item.brand.name || null) : null, + model: item.model || null, + year: item.vehicleModelDate || null, + mileage_km: item.mileageFromOdometer && typeof item.mileageFromOdometer === 'object' + ? (item.mileageFromOdometer.value ?? null) + : null, + price_aed: item.offers && typeof item.offers === 'object' ? (item.offers.price ?? null) : null, + currency: item.offers && typeof item.offers === 'object' ? (item.offers.priceCurrency || null) : null, + location: item.offers && item.offers.areaServed && item.offers.areaServed.address + ? (item.offers.areaServed.address.addressLocality || null) + : null, + image: item.image || null, + }; + }; + + const out = []; + const seen = new Set(); + + const pushItem = (item) => { + const normalized = normalizeItem(item); + if (!normalized || !normalized.url || seen.has(normalized.url)) return; + seen.add(normalized.url); + out.push(normalized); + }; + + for (const el of document.querySelectorAll('script[type="application/ld+json"]')) { + try { + const data = JSON.parse(el.textContent || '{}'); + const list = data && data.mainEntity && Array.isArray(data.mainEntity.itemListElement) + ? data.mainEntity.itemListElement + : []; + for (const entry of list) { + pushItem(entry && typeof entry === 'object' ? (entry.item || entry) : null); + } + } catch (_) {} + } + + if (out.length) return out; + + const nextDataEl = document.querySelector('#__NEXT_DATA__'); + if (!nextDataEl) return out; + try { + const nextData = JSON.parse(nextDataEl.textContent || '{}'); + const serialized = JSON.stringify(nextData); + const matches = serialized.match(/https:\/\/[^\"]+\/motors\/used-cars\/[^\"]+?(?:---[a-f0-9]{32}|\/ad-\d+\/?)/ig) || []; + for (const url of matches) { + if (seen.has(url)) continue; + seen.add(url); + out.push({ + url, + name: null, + description: null, + brand: null, + model: null, + year: null, + mileage_km: null, + price_aed: null, + currency: 'AED', + location: null, + image: null, + }); + } + } catch (_) {} + return out; + } + ''' + ) + except Exception: + return [] + if not isinstance(items, list): + return [] + return [item for item in items if isinstance(item, dict) and item.get("url")] + + def _build_record_from_listing_item(self, item: dict[str, Any]) -> CarRecord: + url = str(item.get("url") or "").strip() + image = str(item.get("image") or "").strip() + summary = { + "source_url": url, + "make": item.get("brand"), + "model": item.get("model"), + "year": item.get("year"), + "odometer": item.get("mileage_km"), + "buy_now": item.get("price_aed"), + "currency": item.get("currency") or "AED", + "location": item.get("location"), + "title": item.get("name"), + "image_urls": [image] if image else [], + } + payload_insights = { + "vehicle_core": { + "make": item.get("brand"), + "model": item.get("model"), + "year": item.get("year"), + "odometer": item.get("mileage_km"), + "location": item.get("location"), + "title": item.get("name"), + "country": "AE", + "selling_type": "STOCK", + }, + "pricing": { + "buy_now": item.get("price_aed"), + "current_bid": None, + "actual_cash_value": None, + "currency": item.get("currency") or "AED", + }, + "damage": {}, + "auction": {"sale_status": "active"}, + "images": {"urls": [image] if image else []}, + } + return self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=summary, + payload_insights=payload_insights, + ) + + @staticmethod + def _extract_listing_items_with_wait(page: Page, attempts: int = 3) -> list[dict[str, Any]]: + for attempt in range(max(1, attempts)): + items = DUBIZZLEScraper._extract_listing_items_from_page(page) + if items: + return items + try: + page.wait_for_function( + r'''() => { + const hasLdJson = Array.from(document.querySelectorAll('script[type="application/ld+json"]')) + .some((el) => { + const text = el.textContent || ''; + return text.includes('itemListElement') || text.includes('SearchResultsPage'); + }); + const nextData = document.querySelector('#__NEXT_DATA__')?.textContent || ''; + return hasLdJson || nextData.length > 1000; + }''', + timeout=5_000, + ) + except Exception: + pass + if attempt < attempts - 1: + try: + page.wait_for_timeout(1_500) + except Exception: + time.sleep(1.5) + return [] + + def _build_sync_result_from_listing_items( + self, + *, + listing_items: list[dict[str, Any]], + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + applied_filters: dict[str, str | int | None], + listing_url: str | None, + ) -> dict[str, Any] | None: + filtered_items = listing_items + if make: + filtered_items = [item for item in filtered_items if str(item.get("brand") or "").strip().lower() == make.strip().lower()] + if model: + filtered_items = [item for item in filtered_items if str(item.get("model") or "").strip().lower() == model.strip().lower()] + + all_raw_urls = [str(item.get("url")) for item in filtered_items if item.get("url")] + if not all_raw_urls: + return None + + pages_info = [{"page_number": 1, "links_found": len(all_raw_urls), "source": "listing_jsonld"}] + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + skipped_existing = 0 + failures: list[dict[str, str]] = [] + + records: list[CarRecord] = [] + for item in filtered_items[:limit if limit is not None and limit > 0 else None]: + try: + records.append(self._build_record_from_listing_item(item)) + except Exception as exc: + cars_failed += 1 + failures.append({"vehicle_url": str(item.get("url") or "listing_jsonld"), "error": str(exc)}) + + if effective_only_new and records: + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids( + [record.origin_url for record in records], + [record.origin_id for record in records], + ) + fresh_records: list[CarRecord] = [] + for record in records: + if record.origin_url in existing_urls or record.origin_id in existing_ids: + skipped_existing += 1 + continue + fresh_records.append(record) + records = fresh_records + + if records: + try: + db_result = self.persistence.upsert_cars_batch(records) + cars_upserted += db_result.get("inserted", 0) + db_result.get("updated", 0) + images_upserted += db_result.get("images_upserted", 0) + except Exception as exc: + cars_failed += len(records) + failures.append({"vehicle_url": "listing_jsonld_batch", "error": str(exc)}) + + total = len(records) if effective_only_new else len(all_raw_urls) + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url} + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": applied_filters, + "pages_collected": 1, + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": False, + "truncated_by_time_budget": False, + "pages": pages_info, + "discovery_source": "listing_jsonld", + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": 0, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, } def _sync_listing_streaming( @@ -581,11 +834,6 @@ class IAAIScraper: year_max: int | None = None, ) -> dict[str, Any]: batch_size = self.settings.celery.batch_size - pending_urls: list[str] = [] - seen_urls: set[str] = set() - all_raw_urls: list[str] = [] - all_listing_origin_urls: set[str] = set() - pages_info: list[dict[str, Any]] = [] skipped_existing = 0 total = 0 cars_upserted = 0 @@ -593,24 +841,16 @@ class IAAIScraper: protection_events = 0 images_upserted = 0 failures: list[dict[str, str]] = [] - early_stopped = False - truncated_by_time_budget = False - listing_recovery_attempts = 0 known_origin_ids: set[str] | None = None threshold = self.settings.listing.early_stop_threshold if effective_only_new and threshold > 0.0: try: - known_origin_ids = self.persistence.get_all_origin_ids_for_lane("iaai:") - logger.info( - "Loaded %d known origin_ids for early-stop listing", - len(known_origin_ids), - ) + known_origin_ids = self.persistence.get_all_origin_ids_for_lane("dubizzle:") except Exception as exc: logger.warning("Could not load known origin_ids for early-stop: %s", exc) - # Совместимость: если only_new без limit и без сегментного URL — старая схема. - # При сегментированном скрапинге (listing_url/year фильтры) всегда streaming. + # Legacy only_new path для совместимости с текущими тестами и старым поведением. if effective_only_new and (limit is None or limit <= 0) and not listing_url and year_min is None and year_max is None: listing_payload = self.collect_listing( make=make, @@ -622,49 +862,37 @@ class IAAIScraper: deduped_urls = self._dedupe_urls(raw_urls) fresh_urls, page_skipped = self._filter_known_urls(deduped_urls) skipped_existing += page_skipped - pending_urls.extend(fresh_urls) total = len(fresh_urls) - for raw in raw_urls: - normalized = self._normalize_vehicle_url(raw) - if normalized: - all_listing_origin_urls.add(normalized) - - logger.info( - "Starting sync: %d vehicles to process (batch mode, only_new legacy path)", - total, - ) - - while len(pending_urls) >= batch_size: - batch_urls = pending_urls[:batch_size] + if fresh_urls: try: - batch_result = self.sync_batch(batch_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - protection_events += int(batch_result.get("protection_events", 0)) - images_upserted += batch_result.get("images_upserted", 0) + batch_result = self.sync_batch(fresh_urls, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) failures.extend(batch_result.get("failures", [])) - except Exception as batch_exc: - logger.error("Legacy only_new batch failed: %s", batch_exc) - cars_failed += len(batch_urls) - failures.append({"vehicle_url": "legacy_only_new_batch", "error": str(batch_exc)}) - pending_urls = pending_urls[batch_size:] - - if pending_urls: - try: - batch_result = self.sync_batch(pending_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - protection_events += int(batch_result.get("protection_events", 0)) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - except Exception as batch_exc: - logger.error("Legacy only_new final batch failed: %s", batch_exc) - cars_failed += len(pending_urls) - failures.append({"vehicle_url": "legacy_only_new_final_batch", "error": str(batch_exc)}) + except Exception as exc: + cars_failed += len(fresh_urls) + failures.append({"vehicle_url": "legacy_only_new", "error": str(exc)}) + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in raw_urls if url} return { - "listing": listing_payload, + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": { + "make": make, + "model": model, + "year_min": year_min, + "year_max": year_max, + }, + "pages_collected": len(listing_payload.get("pages", [])) if isinstance(listing_payload.get("pages"), list) else 0, + "vehicles_collected": len(raw_urls), + "vehicle_urls": raw_urls, + "early_stopped": bool(listing_payload.get("early_stopped", False)), + "truncated_by_time_budget": bool(listing_payload.get("truncated_by_time_budget", False)), + "pages": listing_payload.get("pages", []), + }, "total": total, "skipped_existing": skipped_existing, "cars_upserted": cars_upserted, @@ -675,74 +903,279 @@ class IAAIScraper: "all_listing_origin_urls": all_listing_origin_urls, } - def _process_pending_batch(batch_urls: list[str], batch_start: int) -> bool: - nonlocal cars_upserted, cars_failed, protection_events, images_upserted, failures - if not batch_urls: - return True + elapsed = max(0.0, time.perf_counter() - started_at) + remaining_budget = max(60.0, float(self.settings.celery.task_soft_time_limit) - elapsed - 60.0) - logger.info( - "Processing batch %d-%d of %d", - batch_start + 1, - batch_start + len(batch_urls), - total, + try: + algolia_result = discover_vehicle_hits_from_algolia( + settings=self.settings, + make=make, + model=model, + limit=limit, + year_min=year_min, + year_max=year_max, + listing_url=listing_url, + known_origin_ids=known_origin_ids, + max_duration_seconds=remaining_budget, ) - try: - batch_result = self.sync_batch(batch_urls, lane=lane) - cars_upserted += batch_result.get("cars_upserted", 0) - cars_failed += batch_result.get("cars_failed", 0) - protection_events += int(batch_result.get("protection_events", 0)) - images_upserted += batch_result.get("images_upserted", 0) - failures.extend(batch_result.get("failures", [])) - self._report_progress( - "batch_upserted", - cars_upserted=cars_upserted, - cars_failed=cars_failed, - total_discovered=total, - ) - return True - except PlaywrightError as pw_exc: - logger.warning( - "Batch %d-%d browser crashed: %s. Reinitializing browser context.", - batch_start + 1, - batch_start + len(batch_urls), - pw_exc, - ) - cars_failed += len(batch_urls) - failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(pw_exc)}) + except AlgoliaDiscoveryError as exc: + logger.warning("Algolia discovery failed, fallback to browser listing: %s", exc) + browser_result = self._sync_listing_streaming_browser_fallback( + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + listing_url=listing_url, + year_min=year_min, + year_max=year_max, + ) + # Если после браузерного fallback всё равно 0 и включён флаг — пробуем sitemap discovery. + if ( + int(browser_result.get("total") or 0) == 0 + and self.settings.discovery.sitemap_fallback_on_empty + and make is None + and model is None + and year_min is None + and year_max is None + ): try: - self._new_context() - logger.info("Browser context reinitialized successfully after crash") - return True - except Exception as reinit_exc: - logger.error("Failed to reinitialize browser: %s. Aborting remaining batches.", reinit_exc) - return False - except Exception as batch_exc: - logger.error( - "Batch %d-%d failed: %s. Continuing with next batch.", - batch_start + 1, - batch_start + len(batch_urls), - batch_exc, - ) - cars_failed += len(batch_urls) - failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)}) - return True + logger.warning("Browser fallback returned 0 vehicles, trying sitemap fallback") + return self._sync_listing_streaming_sitemap_fallback( + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + ) + except Exception as sitemap_exc: + logger.warning("Sitemap fallback failed: %s", sitemap_exc) + return browser_result - def _consume_listing_recovery_attempt(*, page_number: int, reason: str) -> None: - nonlocal listing_recovery_attempts - listing_recovery_attempts += 1 - logger.warning( - "Listing recovery attempt %d/%d at page %d (%s)", - listing_recovery_attempts, - _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT, - page_number, - reason, - ) - if listing_recovery_attempts > _MAX_LISTING_RECOVERY_ATTEMPTS_PER_SEGMENT: - raise ListingResumeError( - f"Listing recovery exhausted at page {page_number}: {reason}" - ) + all_raw_urls = list(algolia_result.vehicle_urls) + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url} + pages_info = list(algolia_result.pages) + effective_urls = list(all_raw_urls) + if effective_only_new and effective_urls: + candidate_ids = [ + algolia_result.origin_ids_by_url[url] + for url in effective_urls + if url in algolia_result.origin_ids_by_url + ] + existing_urls, existing_ids = self.persistence.get_existing_urls_and_ids(effective_urls, candidate_ids) + filtered_urls: list[str] = [] + for url in effective_urls: + oid = algolia_result.origin_ids_by_url.get(url) + if url in existing_urls or (oid and oid in existing_ids): + skipped_existing += 1 + continue + filtered_urls.append(url) + effective_urls = filtered_urls + + if limit is not None and limit > 0: + effective_urls = effective_urls[:limit] + + total = len(effective_urls) + logger.info("Algolia listing discovered %d vehicles, processing %d", len(all_raw_urls), total) + + for batch_start in range(0, len(effective_urls), batch_size): + batch_urls = effective_urls[batch_start: batch_start + batch_size] + records: list[CarRecord] = [] + + for url in batch_urls: + hit = algolia_result.hit_records.get(url) + if not hit: + cars_failed += 1 + failures.append({"vehicle_url": url, "error": "Missing Algolia hit for URL"}) + continue + try: + record = self.car_mapper.map_to_car_record( + vehicle_url=url, + vehicle_summary=hit, + payload_insights={ + "vehicle_core": { + "lot_number": hit.get("id") or hit.get("objectID"), + "year": hit.get("year"), + "make": hit.get("make"), + "model": hit.get("model"), + "trim": hit.get("trim") or hit.get("motors_trim"), + "odometer": hit.get("kilometers") or hit.get("odometer"), + "body_type": hit.get("body_type"), + "gearbox": hit.get("transmission_type") or hit.get("transmission"), + "drive": hit.get("drive_type"), + "fuel_type": hit.get("fuel_type"), + "color": hit.get("exterior_color") or hit.get("color"), + "seller": hit.get("seller_type"), + "location": hit.get("location_name") or hit.get("location"), + "title": hit.get("title") or hit.get("name"), + "country": "AE", + "selling_type": "STOCK", + }, + "pricing": { + "buy_now": hit.get("price"), + "current_bid": None, + "actual_cash_value": None, + "currency": hit.get("price_currency") or "AED", + }, + "damage": {}, + "auction": {"sale_status": hit.get("status")}, + "images": { + "urls": hit.get("photo_mains") or hit.get("photo_thumbnails") or [], + }, + }, + ) + records.append(record) + except Exception as rec_exc: + cars_failed += 1 + failures.append({"vehicle_url": url, "error": str(rec_exc)}) + + if not records: + continue + + try: + db_result = self.persistence.upsert_cars_batch(records) + cars_upserted += db_result.get("inserted", 0) + db_result.get("updated", 0) + images_upserted += db_result.get("images_upserted", 0) + except Exception as db_exc: + cars_failed += len(records) + failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(db_exc)}) + + return { + "listing": { + "status": "ok", + "listing_url": listing_url or self.settings.listing.cars_url, + "applied_filters": { + "make": make, + "model": model, + "year_min": year_min, + "year_max": year_max, + }, + "pages_collected": len(pages_info), + "vehicles_collected": len(all_raw_urls), + "vehicle_urls": all_raw_urls, + "early_stopped": algolia_result.early_stopped, + "truncated_by_time_budget": algolia_result.truncated_by_time_budget, + "pages": pages_info, + "total_hits": algolia_result.total_hits, + }, + "total": total, + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": protection_events, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _sync_listing_streaming_sitemap_fallback( + self, + *, + lane: str, + limit: int | None, + effective_only_new: bool, + ) -> dict[str, Any]: + try: + discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=self.settings) + except SitemapDiscoveryError as exc: + raise RuntimeError(f"Sitemap discovery failed: {exc}") from exc + + all_urls = list(discovery_result.vehicle_urls) + urls = list(all_urls) + skipped_existing = 0 + if effective_only_new and urls: + urls, skipped_existing = self._filter_known_urls(urls) + + if limit is not None and limit > 0: + urls = urls[:limit] + + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] + batch_size = self.settings.celery.batch_size + + for i in range(0, len(urls), batch_size): + chunk = urls[i:i + batch_size] + try: + result = self.sync_batch(chunk, lane=lane) + cars_upserted += int(result.get("cars_upserted", 0)) + cars_failed += int(result.get("cars_failed", 0)) + images_upserted += int(result.get("images_upserted", 0)) + failures.extend(result.get("failures", [])) + except Exception as exc: + cars_failed += len(chunk) + failures.append({"vehicle_url": f"sitemap_batch_{i}", "error": str(exc)}) + + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_urls if url} + return { + "listing": { + "status": "ok", + "listing_url": self.settings.listing.cars_url, + "applied_filters": {"make": None, "model": None, "year_min": None, "year_max": None}, + "pages_collected": 1, + "vehicles_collected": len(all_urls), + "vehicle_urls": all_urls, + "early_stopped": False, + "truncated_by_time_budget": False, + "pages": [{"page_number": 1, "links_found": len(all_urls)}], + "discovery_source": "sitemap", + }, + "total": len(urls), + "skipped_existing": skipped_existing, + "cars_upserted": cars_upserted, + "cars_failed": cars_failed, + "images_upserted": images_upserted, + "protection_events": 0, + "failures": failures, + "all_listing_origin_urls": all_listing_origin_urls, + } + + def _sync_listing_streaming_browser_fallback( + self, + *, + make: str | None, + model: str | None, + lane: str, + limit: int | None, + effective_only_new: bool, + listing_url: str | None = None, + year_min: int | None = None, + year_max: int | None = None, + ) -> dict[str, Any]: + batch_size = self.settings.celery.batch_size + all_raw_urls: list[str] = [] + seen_urls: set[str] = set() + pending_urls: list[str] = [] + pages_info: list[dict[str, Any]] = [] + skipped_existing = 0 + cars_upserted = 0 + cars_failed = 0 + images_upserted = 0 + failures: list[dict[str, str]] = [] page = None + applied_filters: dict[str, str | int | None] = { + "make": make, + "model": model, + "year_min": year_min, + "year_max": year_max, + } + listing_items_count = 0 + + def _flush_pending() -> None: + nonlocal cars_upserted, cars_failed, images_upserted, failures, pending_urls + while len(pending_urls) >= batch_size: + chunk = pending_urls[:batch_size] + pending_urls = pending_urls[batch_size:] + try: + batch_result = self.sync_batch(chunk, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + except Exception as exc: + cars_failed += len(chunk) + failures.append({"vehicle_url": "browser_fallback_batch", "error": str(exc)}) + try: page, applied_filters = self._open_listing_for_stream( make=make, @@ -752,104 +1185,28 @@ class IAAIScraper: year_max=year_max, ) - pages_since_rotation = 0 + listing_items = self._extract_listing_items_with_wait(page) + listing_result = self._build_sync_result_from_listing_items( + listing_items=listing_items, + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + applied_filters=applied_filters, + listing_url=listing_url, + ) + if listing_result is not None: + listing_items_count = int(listing_result["listing"]["vehicles_collected"]) + return listing_result + for page_number in range(1, self.settings.listing.max_pages_per_run + 1): - # Heartbeat для worker stall-watchdog: при малом числе ссылок на странице - # batch_upserted может долго не вызываться, поэтому пульсуем прогресс - # на каждом шаге пагинации. - self._report_progress( - "listing_page_scan_started", - page_number=page_number, - total_discovered=total, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - - # Проактивная ротация контекста (опционально, по умолчанию выключена). - # Если включена — требует долистывания до page_number после reopen, - # поэтому max_nav_pages поднят под реальную глубину. - if _CONTEXT_ROTATE_EVERY_PAGES > 0 and pages_since_rotation >= _CONTEXT_ROTATE_EVERY_PAGES: - logger.warning( - "Rotating browser context at page %d (every %d pages) to reset anti-bot state", - page_number, _CONTEXT_ROTATE_EVERY_PAGES, - ) - try: - page.close() - except Exception: - pass - page = None - try: - _consume_listing_recovery_attempt( - page_number=page_number, - reason="context_rotation", - ) - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=300, - ) - pages_since_rotation = 0 - except ListingResumeError as resume_exc: - logger.warning( - "Context rotation could not resume at page %d (%s) — stopping segment", - page_number, resume_exc, - ) - break - - try: - with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number}"): - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - except (PageOperationTimeoutError, PlaywrightError) as op_exc: - logger.warning( - "Page %d collect stalled/failed (%s) — reopening listing and resuming", - page_number, op_exc, - ) - try: - page.close() - except Exception: - pass - page = None - try: - _consume_listing_recovery_attempt( - page_number=page_number, - reason=f"collect_failed:{type(op_exc).__name__}", - ) - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=300, - ) - pages_since_rotation = 0 - with _PageOpWatchdog(_PAGE_COLLECT_TIMEOUT_S, f"collect page {page_number} (after reopen)"): - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - except (ListingResumeError, PageOperationTimeoutError, PlaywrightError) as resume_exc: - logger.warning( - "Cannot recover at page %d (%s) — stopping pagination for this segment", - page_number, resume_exc, - ) - break - - pages_info.append({ - "page_number": page_result.page_number, - "links_found": len(page_result.vehicle_links), - }) - pages_since_rotation += 1 - - page_urls = self._extract_page_urls( - page_result, - all_raw_urls, - seen_urls, - all_listing_origin_urls, - ) + page_result = self.listing_collector.collect_current_page(page, page_number=page_number) + pages_info.append({"page_number": page_number, "links_found": len(page_result.vehicle_links)}) + page_urls = self._extract_page_urls(page_result, all_raw_urls, seen_urls) + for discovered in page_urls: + if discovered not in all_raw_urls: + all_raw_urls.append(discovered) if not page_urls: page_result, page_urls = self._recover_empty_listing_page( @@ -857,309 +1214,78 @@ class IAAIScraper: page_number=page_number, all_raw_urls=all_raw_urls, seen_urls=seen_urls, - all_listing_origin_urls=all_listing_origin_urls, - listing_url=listing_url, ) - if not page_urls and page_number > 1: - logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number) - page.close() - try: - _consume_listing_recovery_attempt( - page_number=page_number, - reason="empty_page_after_retry", - ) - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=300, - ) - page_result = self.listing_collector.collect_current_page(page, page_number=page_number) - page_urls = self._extract_page_urls( - page_result, - all_raw_urls, - seen_urls, - all_listing_origin_urls, - ) - except ListingResumeError as resume_exc: - logger.warning( - "Cannot resume at page %d (%s) — stopping pagination for this segment", - page_number, resume_exc, - ) - page = None - page_urls = [] - if not page_urls: - logger.info("Page %d: 0 new links, stopping pagination", page_number) - break - - if known_origin_ids is not None and threshold > 0.0 and page_result.vehicle_links: - page_known = sum( - 1 - for item in page_result.vehicle_links - if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids - ) - page_new = len(page_result.vehicle_links) - page_known - ratio = page_known / len(page_result.vehicle_links) - if ratio >= threshold and page_new == 0: - logger.info( - "Early stop on page %d: %.0f%% known (%d/%d), 0 new >= threshold %.0f%%", - page_number, - ratio * 100, - page_known, - len(page_result.vehicle_links), - threshold * 100, + pages_info[-1]["links_found"] = len(page_result.vehicle_links) + if page_number == 1: + listing_result = self._build_sync_result_from_listing_items( + listing_items=self._extract_listing_items_with_wait(page), + make=make, + model=model, + lane=lane, + limit=limit, + effective_only_new=effective_only_new, + applied_filters=applied_filters, + listing_url=listing_url, ) - early_stopped = True + if listing_result is not None: + listing_items_count = int(listing_result["listing"]["vehicles_collected"]) + return listing_result + for discovered in page_urls: + if discovered not in all_raw_urls: + all_raw_urls.append(discovered) + if not page_urls: break fresh_urls = page_urls - page_skipped = 0 if effective_only_new: fresh_urls, page_skipped = self._filter_known_urls(page_urls) skipped_existing += page_skipped if limit is not None and limit > 0: - remaining_limit = max(0, limit - total - len(pending_urls)) - if remaining_limit <= 0: + remaining = max(0, limit - len(pending_urls) - (cars_upserted + cars_failed)) + if remaining <= 0: break - fresh_urls = fresh_urls[:remaining_limit] + fresh_urls = fresh_urls[:remaining] pending_urls.extend(fresh_urls) - total += len(fresh_urls) + _flush_pending() - logger.info( - "Page %d: %d links, %d new, %d known (total new: %d/%s)", - page_number, - len(page_urls), - len(fresh_urls), - page_skipped, - total, - limit if limit is not None else "∞", - ) - self._report_progress( - "listing_page_scan_done", - page_number=page_number, - page_links=len(page_urls), - page_new=len(fresh_urls), - page_known=page_skipped, - total_discovered=total, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - - # Прогресс каждые 10 страниц на уровне WARNING. - if page_number % 10 == 0: - logger.warning( - "sync_listing progress: pages=%d, discovered=%d, upserted=%d, failed=%d, pending=%d", - page_number, - total, - cars_upserted, - cars_failed, - len(pending_urls), - ) - - while len(pending_urls) >= batch_size: - batch_urls = pending_urls[:batch_size] - self._report_progress( - "listing_batch_dispatch_started", - page_number=page_number, - batch_size=len(batch_urls), - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - if not _process_pending_batch(batch_urls, cars_upserted + cars_failed): - pending_urls = [] - break - pending_urls = pending_urls[batch_size:] - self._report_progress( - "listing_batch_dispatch_done", - page_number=page_number, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - # Пауза между батчами: снижает нагрузку на IAAI и риск бана. - if pending_urls: - time.sleep(random.uniform(0.5, 1.5)) - - # Anti-stall: не держим хвост pending до следующей страницы/конца сегмента. - # Если после scan остались URL меньше batch_size — отправляем их сразу. - if pending_urls: - self._report_progress( - "listing_tail_batch_started", - page_number=page_number, - batch_size=len(pending_urls), - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - if not _process_pending_batch(pending_urls, cars_upserted + cars_failed): - pending_urls = [] - break - pending_urls = [] - self._report_progress( - "listing_tail_batch_done", - page_number=page_number, - pending_urls=0, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - - if limit is not None and limit > 0 and total >= limit: + if limit is not None and limit > 0 and (cars_upserted + cars_failed + len(pending_urls)) >= limit: break - if ( - self.settings.listing.collect_current_page_only - or not self.settings.listing.include_pagination - or not page_result.next_page_detected - ): + if not page_result.next_page_detected: + break + if self.settings.listing.collect_current_page_only: break - if page_number == 1 and not self.listing_collector.has_page_number(page, 2): - logger.info( - "Page 2 not found on page 1 — treating segment as single-page", - ) - self._report_progress( - "listing_single_page_no_page2", - page_number=page_number, - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) + if not self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1): break - suspicious_small_segment = ( - total <= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_LINKS - and page_number >= _SMALL_SEGMENT_SUSPICIOUS_PAGINATION_MAX_PAGE - ) - self._report_progress( - "listing_next_page_started", - page_number=page_number, - next_page_number=page_number + 1, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - try: - with _ProgressHeartbeat( - self._report_progress, - "listing_next_page_started", - page_number=page_number, - next_page_number=page_number + 1, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ): - with _PageOpWatchdog(_PAGE_NEXT_TIMEOUT_S, f"next page {page_number + 1}"): - next_ok = self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1) - except (PageOperationTimeoutError, PlaywrightError) as nav_exc: - logger.warning( - "go_to_next_page stalled/failed at page %d (%s) — will reopen", - page_number + 1, nav_exc, - ) - next_ok = False - self._report_progress( - "listing_next_page_done", - page_number=page_number, - next_page_number=page_number + 1, - next_ok=bool(next_ok), - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - if not next_ok: - if page_number == 1: - logger.warning( - "Page 2 is not reachable from page 1 — treating segment as single-page and moving on", - ) - break - if suspicious_small_segment: - logger.warning( - "Small segment pagination looks exhausted at page %d: total=%d, next page navigation failed — stopping segment", - page_number, - total, - ) - break - logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1) - self._report_progress( - "listing_reopen_started", - page_number=page_number, - target_page_number=page_number + 1, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - try: - page.close() - except Exception: - pass - page = None - try: - _consume_listing_recovery_attempt( - page_number=page_number + 1, - reason=f"next_page_failed:{type(nav_exc).__name__}", - ) - page, _ = self._reopen_listing_and_resume( - target_page_number=page_number + 1, - make=make, - model=model, - listing_url=listing_url, - year_min=year_min, - year_max=year_max, - max_nav_pages=300, - ) - pages_since_rotation = 0 - self._report_progress( - "listing_reopen_done", - page_number=page_number, - target_page_number=page_number + 1, - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - except ListingResumeError as resume_exc: - logger.warning( - "Cannot resume at page %d (%s) — treating pagination as exhausted", - page_number + 1, resume_exc, - ) - self._report_progress( - "listing_reopen_failed", - page_number=page_number, - target_page_number=page_number + 1, - error=str(resume_exc), - pending_urls=len(pending_urls), - cars_upserted=cars_upserted, - cars_failed=cars_failed, - ) - break - if pending_urls: - _process_pending_batch(pending_urls, cars_upserted + cars_failed) - - logger.warning( - "sync_listing final progress: pages=%d, discovered=%d, upserted=%d, failed=%d", - len(pages_info), - total, - cars_upserted, - cars_failed, - ) + try: + batch_result = self.sync_batch(pending_urls, lane=lane) + cars_upserted += int(batch_result.get("cars_upserted", 0)) + cars_failed += int(batch_result.get("cars_failed", 0)) + images_upserted += int(batch_result.get("images_upserted", 0)) + failures.extend(batch_result.get("failures", [])) + except Exception as exc: + cars_failed += len(pending_urls) + failures.append({"vehicle_url": "browser_fallback_final", "error": str(exc)}) finally: if page is not None: page.close() + all_listing_origin_urls = {self._normalize_vehicle_url(url) for url in all_raw_urls if url} + total = len(all_raw_urls) if not effective_only_new else max(0, len(all_raw_urls) - skipped_existing) return { "listing": { "status": "ok", - "listing_url": self.settings.listing.cars_url, + "listing_url": listing_url or self.settings.listing.cars_url, "applied_filters": applied_filters, "pages_collected": len(pages_info), "vehicles_collected": len(all_raw_urls), "vehicle_urls": all_raw_urls, - "early_stopped": early_stopped, - "truncated_by_time_budget": truncated_by_time_budget, + "early_stopped": False, + "truncated_by_time_budget": False, "pages": pages_info, }, "total": total, @@ -1167,7 +1293,7 @@ class IAAIScraper: "cars_upserted": cars_upserted, "cars_failed": cars_failed, "images_upserted": images_upserted, - "protection_events": protection_events, + "protection_events": 0, "failures": failures, "all_listing_origin_urls": all_listing_origin_urls, } @@ -1260,7 +1386,7 @@ class IAAIScraper: brnch = str(js_data.get("BranchNumber", "") or "").strip() img_keys = js_data.get("imageKeys") or [] image_urls = [ - f"https://vis.iaai.com/resizer?imageKeys={k}&width=845&height=633" + f"https://vis.dubizzle.com/resizer?imageKeys={k}&width=845&height=633" for k in img_keys ] @@ -1398,8 +1524,8 @@ class IAAIScraper: return result @staticmethod - def _extract_iaai_json_from_html(html: str, vehicle_url: str) -> dict | None: - """Извлекает IAAI inventoryView.attributes из HTML без браузера. + def _extract_dubizzle_json_from_html(html: str, vehicle_url: str) -> dict | None: + """Извлекает DUBIZZLE inventoryView.attributes из HTML без браузера. Использует json.JSONDecoder.raw_decode для быстрого поиска JSON вместо посимвольного сканирования скобок. @@ -1496,7 +1622,7 @@ class IAAIScraper: html = response.text() - js_data = self._extract_iaai_json_from_html(html, vehicle_url) + js_data = self._extract_dubizzle_json_from_html(html, vehicle_url) if not js_data: raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") @@ -1559,7 +1685,7 @@ class IAAIScraper: raise RuntimeError(f"HTTP fetch failed for {vehicle_url}: {response.status}") html = response.data.decode("utf-8", errors="ignore") - js_data = self._extract_iaai_json_from_html(html, vehicle_url) + js_data = self._extract_dubizzle_json_from_html(html, vehicle_url) if not js_data: raise RuntimeError(f"HTTP fast-path missing embedded JSON for {vehicle_url}") @@ -1575,7 +1701,7 @@ class IAAIScraper: ) return CarRecord.model_validate(db_record.model_dump(mode="json")) - def sync_vehicle(self, vehicle_url: str, lane: str = "iaai"): + def sync_vehicle(self, vehicle_url: str, lane: str = "dubizzle"): trace_id = self._new_trace_id("sync-vehicle") started_at = time.perf_counter() self.persistence.create_tables() @@ -1807,7 +1933,7 @@ class IAAIScraper: def sync_batch( self, vehicle_urls: list[str], - lane: str = "iaai_cars", + lane: str = "dubizzle_cars", parallel_tabs: int | None = None, ) -> dict: # Runtime config может меняться на лету (добавили/убрали бренды, диапазоны и т.п.). @@ -2075,7 +2201,7 @@ class IAAIScraper: self, make: str | None = None, model: str | None = None, - lane: str = "iaai_cars", + lane: str = "dubizzle_cars", limit: int | None = None, only_new: bool | None = None, listing_url: str | None = None, @@ -2093,7 +2219,7 @@ class IAAIScraper: limit = rc.limit if only_new is None and rc.only_new is not None: only_new = rc.only_new - if lane == "iaai_cars" and rc.lane is not None: + if lane == "dubizzle_cars" and rc.lane is not None: lane = rc.lane trace_id = self._new_trace_id("sync-listing") @@ -2209,7 +2335,7 @@ class IAAIScraper: def sync_listing_segmented( self, segments: list[dict[str, Any]], - lane: str = "iaai_cars", + lane: str = "dubizzle_cars", only_new: bool | None = None, start_segment: int = 0, start_page: int = 1, @@ -2317,6 +2443,8 @@ class IAAIScraper: }) segment_done = bool(result.get("full_scan_completed", False)) + if not segment_done: + completed_all = False # Даже если сегмент завершился неполно (например, страница/пагинация сломалась), # в bootstrap-режиме не зацикливаемся на нём: двигаем чекпоинт дальше. if not segment_done: @@ -2353,6 +2481,7 @@ class IAAIScraper: segment_full_scan_completed=segment_done, ) except Exception as exc: + completed_all = False logger.error("Segment %d/%d failed: %s — %s", seg_idx + 1, len(segments), seg_label, exc) all_failures.append({"vehicle_url": f"segment_{seg_idx}_{seg_label}", "error": str(exc)}) skipped_segments += 1 diff --git a/iaai_scraper/storage/__init__.py b/dubizzle_scraper/storage/__init__.py similarity index 100% rename from iaai_scraper/storage/__init__.py rename to dubizzle_scraper/storage/__init__.py diff --git a/iaai_scraper/storage/db.py b/dubizzle_scraper/storage/db.py similarity index 97% rename from iaai_scraper/storage/db.py rename to dubizzle_scraper/storage/db.py index a1740f7..1ae3b7c 100644 --- a/iaai_scraper/storage/db.py +++ b/dubizzle_scraper/storage/db.py @@ -11,7 +11,7 @@ from ..core.config import Settings from .models import Base, Car, Image, SyncRun from .schemas import CarRecord -logger = logging.getLogger("iaai_scraper.db") +logger = logging.getLogger("dubizzle_scraper.db") CAR_DB_FIELDS = { @@ -492,7 +492,7 @@ class PersistenceService: logger.error("Individual upsert failed for %s: %s", record.origin_id, exc) return {"inserted": inserted, "updated": updated, "images_upserted": images_total} - def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int: + def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "dubizzle") -> int: """Помечает авто как проданные, если их нет в активном листинге (по origin_id).""" if not active_origin_ids: return 0 @@ -501,7 +501,7 @@ class PersistenceService: update(Car) .where(Car.origin_id.notin_(active_origin_ids)) .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like("iaai:%")) + .where(Car.origin_id.like("dubizzle:%")) .values(is_sold=True) ) result = session.execute(stmt) @@ -510,7 +510,7 @@ class PersistenceService: logger.info("Marked %d cars as sold (no longer in listing)", count) return count - def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int: + def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "dubizzle") -> int: """Помечает авто как проданные, если их URL нет в активном листинге. Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN, @@ -532,7 +532,7 @@ class PersistenceService: if is_postgres: # Считаем кандидатов на sold. total_active = session.execute( - text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'") + text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%%'") ).scalar() or 0 if total_active == 0: @@ -560,7 +560,7 @@ class PersistenceService: LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url WHERE a.url IS NULL AND c.is_sold = FALSE - AND c.origin_id LIKE 'iaai:%%' + AND c.origin_id LIKE 'dubizzle:%%' """)).scalar() or 0 # Защита от аномалии. @@ -581,7 +581,7 @@ class PersistenceService: LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url WHERE a.url IS NULL AND c.is_sold = FALSE - AND c.origin_id LIKE 'iaai:%%' + AND c.origin_id LIKE 'dubizzle:%%' ) sub WHERE cars.id = sub.id """)) @@ -591,13 +591,13 @@ class PersistenceService: stmt = ( update(Car) .where(Car.is_sold == False) # noqa: E712 - .where(Car.origin_id.like("iaai:%")) + .where(Car.origin_id.like("dubizzle:%")) .values(is_sold=True) ) # Загружаем active URL. all_active = session.execute( select(Car.id, Car.origin_url).where( - Car.is_sold == False, Car.origin_id.like("iaai:%") # noqa: E712 + Car.is_sold == False, Car.origin_id.like("dubizzle:%") # noqa: E712 ) ).all() mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls] @@ -621,7 +621,7 @@ class PersistenceService: logger.info("Marked %d cars as sold by URL (no longer in listing)", count) return count - def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]: + def get_all_origin_ids_for_lane(self, prefix: str = "dubizzle:") -> set[str]: """Возвращает все известные origin_id для заданного префикса. Использует yield_per для потоковой загрузки при большом количестве записей. @@ -632,7 +632,7 @@ class PersistenceService: ) return {str(row[0]) for row in result if row and row[0]} - def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]: + def get_all_active_origin_urls_for_lane(self, prefix: str = "dubizzle:") -> set[str]: """Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса.""" with self.session_scope() as session: result = session.execute( @@ -643,7 +643,7 @@ class PersistenceService: ) return {str(row[0]) for row in result if row and row[0]} - def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int: + def count_active_cars_for_lane(self, prefix: str = "dubizzle:") -> int: """Возвращает количество активных (не проданных) авто для заданного lane-префикса.""" from sqlalchemy import func as sa_func with self.session_scope() as session: @@ -657,7 +657,7 @@ class PersistenceService: def get_active_origin_urls_batch_for_refresh( self, - prefix: str = "iaai:", + prefix: str = "dubizzle:", offset: int = 0, limit: int = 500, ) -> list[str]: diff --git a/iaai_scraper/storage/enums.py b/dubizzle_scraper/storage/enums.py similarity index 87% rename from iaai_scraper/storage/enums.py rename to dubizzle_scraper/storage/enums.py index d25d8a5..371272e 100644 --- a/iaai_scraper/storage/enums.py +++ b/dubizzle_scraper/storage/enums.py @@ -16,9 +16,9 @@ BODY_TYPE_ENUM_VALUES = ( "OTHER", "NA", ) -COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA") +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "AE", "NA") ORIGIN_ENUM_VALUES = ( - "IAAI", + "DUBIZZLE", "NA", ) SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA") diff --git a/iaai_scraper/storage/models.py b/dubizzle_scraper/storage/models.py similarity index 100% rename from iaai_scraper/storage/models.py rename to dubizzle_scraper/storage/models.py diff --git a/iaai_scraper/storage/schemas.py b/dubizzle_scraper/storage/schemas.py similarity index 100% rename from iaai_scraper/storage/schemas.py rename to dubizzle_scraper/storage/schemas.py diff --git a/iaai_scraper/worker/__init__.py b/dubizzle_scraper/worker/__init__.py similarity index 100% rename from iaai_scraper/worker/__init__.py rename to dubizzle_scraper/worker/__init__.py diff --git a/iaai_scraper/worker/celery_app.py b/dubizzle_scraper/worker/celery_app.py similarity index 84% rename from iaai_scraper/worker/celery_app.py rename to dubizzle_scraper/worker/celery_app.py index 66363f2..242791d 100644 --- a/iaai_scraper/worker/celery_app.py +++ b/dubizzle_scraper/worker/celery_app.py @@ -9,8 +9,7 @@ from redis import Redis from ..core.config import settings from ..core.logs import setup_logging -logger = logging.getLogger("iaai_scraper.worker.celery_app") -STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched" +logger = logging.getLogger("dubizzle_scraper.worker.celery_app") @celery_setup_logging.connect @@ -37,7 +36,7 @@ def _result_backend() -> str: celery_app = Celery( - "iaai_scraper", + "dubizzle_scraper", broker=_broker_url(), backend=_result_backend(), ) @@ -79,10 +78,13 @@ celery_app.conf.update( worker_hijack_root_logger=False, beat_schedule={ "periodic-sync-listing": { - "task": "iaai_scraper.worker.tasks.sync_listing_task", + "task": "dubizzle_scraper.worker.tasks.sync_listing_task", "schedule": settings.celery.beat_sync_interval_minutes * 60.0, "args": (), - "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True}, + "kwargs": { + "limit": settings.celery.beat_sync_limit, + "only_new": False if settings.discovery.always_full_scan else True, + }, "options": { "queue": "scraping", "expires": settings.celery.beat_sync_interval_minutes * 60.0, @@ -90,11 +92,11 @@ celery_app.conf.update( } }, task_routes={ - "iaai_scraper.worker.tasks.*": {"queue": "scraping"} + "dubizzle_scraper.worker.tasks.*": {"queue": "scraping"} }, ) -celery_app.autodiscover_tasks(["iaai_scraper.worker"]) +celery_app.autodiscover_tasks(["dubizzle_scraper.worker"]) @worker_ready.connect @@ -111,7 +113,7 @@ def _on_worker_ready(**kwargs): retry_on_timeout=True, ) - for stale_key in ("iaai:locks:sync_listing",): + for stale_key in ("dubizzle:locks:sync_listing",): try: ttl = redis_client.ttl(stale_key) if ttl is not None and ttl != -2: @@ -127,10 +129,8 @@ def _on_worker_ready(**kwargs): if queue_len > 0: logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len) return - - should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) except Exception: - logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True) + logger.warning("Worker ready startup sync check failed; skipping immediate dispatch", exc_info=True) return finally: if redis_client is not None: @@ -139,13 +139,9 @@ def _on_worker_ready(**kwargs): except Exception: pass - if not should_dispatch: - logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue") - return - logger.info("Worker ready — dispatching initial sync_listing task") celery_app.send_task( - "iaai_scraper.worker.tasks.sync_listing_task", + "dubizzle_scraper.worker.tasks.sync_listing_task", kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False}, queue="scraping", expires=settings.celery.beat_sync_interval_minutes * 60.0, diff --git a/iaai_scraper/worker/self_heal.py b/dubizzle_scraper/worker/self_heal.py similarity index 77% rename from iaai_scraper/worker/self_heal.py rename to dubizzle_scraper/worker/self_heal.py index d337032..13f90cd 100644 --- a/iaai_scraper/worker/self_heal.py +++ b/dubizzle_scraper/worker/self_heal.py @@ -8,21 +8,28 @@ import time from redis import Redis -logger = logging.getLogger("iaai_scraper.worker.self_heal") +logger = logging.getLogger("dubizzle_scraper.worker.self_heal") -GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts" -SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress" +GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts" +SELF_HEAL_RESTART_LOCK_KEY = "dubizzle:state:self_heal_restart_in_progress" + + +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + if value is None and name.startswith("DUBIZZLE_"): + value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):]) + return value if value is not None else default def _env_bool(name: str, default: bool) -> bool: - value = os.getenv(name) + value = _env_str(name, "true" if default else "false") if value is None: return default return value.strip().lower() in {"1", "true", "yes", "on"} def _env_int(name: str, default: int) -> int: - value = os.getenv(name) + value = _env_str(name, str(default)) if value is None: return default try: @@ -32,7 +39,7 @@ def _env_int(name: str, default: int) -> int: def _get_redis() -> Redis: - url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0") + url = _env_str("DUBIZZLE_REDIS_URL", "redis://redis:6379/0") return Redis.from_url( url, decode_responses=True, @@ -62,7 +69,7 @@ def _read_last_progress_ts(redis_client: Redis) -> int | None: # Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*. # Это дороже, но выполняется только при отсутствии основного маркера. max_ts = 0 - for key in redis_client.scan_iter(match="iaai:state:task_progress:*"): + for key in redis_client.scan_iter(match="dubizzle:state:task_progress:*"): try: payload = redis_client.get(key) if not payload: @@ -101,7 +108,8 @@ def _kill_worker_process() -> None: # Если процесс ещё жив — принудительно убиваем. os.kill(pid, 0) logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid) - os.kill(pid, signal.SIGKILL) + kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM) + os.kill(pid, kill_signal) except ProcessLookupError: pass except Exception: @@ -109,15 +117,15 @@ def _kill_worker_process() -> None: def main() -> None: - if not _env_bool("IAAI_SELF_HEAL_ENABLED", True): - logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED") + if not _env_bool("DUBIZZLE_SELF_HEAL_ENABLED", True): + logger.info("Self-heal watchdog disabled via DUBIZZLE_SELF_HEAL_ENABLED/DUBIZZLE_SELF_HEAL_ENABLED") return - queue_name = os.getenv("IAAI_CELERY_QUEUE", "scraping") - check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30)) - stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720)) - startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300)) - restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300)) + queue_name = _env_str("DUBIZZLE_CELERY_QUEUE", "scraping") + check_interval = max(5, _env_int("DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30)) + stall_seconds = max(180, _env_int("DUBIZZLE_SELF_HEAL_STALL_SECONDS", 720)) + startup_grace = max(30, _env_int("DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300)) + restart_cooldown = max(60, _env_int("DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300)) logger.warning( "Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss", @@ -195,7 +203,7 @@ def main() -> None: if __name__ == "__main__": logging.basicConfig( - level=os.getenv("IAAI_LOG_LEVEL", "INFO"), + level=_env_str("DUBIZZLE_LOG_LEVEL", "INFO"), format="%(asctime)s | %(levelname)s | %(name)s | %(message)s", ) main() diff --git a/iaai_scraper/worker/tasks.py b/dubizzle_scraper/worker/tasks.py similarity index 86% rename from iaai_scraper/worker/tasks.py rename to dubizzle_scraper/worker/tasks.py index 5edaf46..239a738 100644 --- a/iaai_scraper/worker/tasks.py +++ b/dubizzle_scraper/worker/tasks.py @@ -1,4 +1,4 @@ -# Задачи Celery для синхронизации автомобилей и листинга IAAI. +# Задачи Celery для синхронизации автомобилей и листинга Dubizzle. import json import logging @@ -13,25 +13,39 @@ from celery import shared_task from redis import Redis from ..core.config import Settings, parse_listing_segments -from ..scraper import IAAIScraper +from ..scraper import DUBIZZLEScraper from ..storage.db import PersistenceService from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats -logger = logging.getLogger("iaai_scraper.worker.tasks") +logger = logging.getLogger("dubizzle_scraper.worker.tasks") -# Минимальная пауза между батчами (секунды) — не давит IAAI. -_INTER_BATCH_DELAY = max(float(os.getenv("IAAI_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0) -# Если доля failed в батче превышает порог — прерываем (IAAI блокирует). -_FAIL_RATE_THRESHOLD = min(max(float(os.getenv("IAAI_FAIL_RATE_THRESHOLD", "0.9")), 0.0), 1.0) -SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing" -SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done" -SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint" +def _env_str(name: str, default: str) -> str: + value = os.getenv(name) + if value is None and name.startswith("DUBIZZLE_"): + value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):]) + return value if value is not None else default + + +def _env_float(name: str, default: float) -> float: + try: + return float(_env_str(name, str(default)).strip()) + except Exception: + return default + +# Минимальная пауза между батчами (секунды). +_INTER_BATCH_DELAY = max(_env_float("DUBIZZLE_INTER_BATCH_DELAY_SECONDS", 0.3), 0.0) +# Если доля failed в батче превышает порог — прерываем. +_FAIL_RATE_THRESHOLD = min(max(_env_float("DUBIZZLE_FAIL_RATE_THRESHOLD", 0.9), 0.0), 1.0) + +SYNC_LISTING_LOCK_KEY = "dubizzle:locks:sync_listing" +SYNC_FULL_SCAN_DONE_KEY = "dubizzle:state:sync_full_scan_done" +SYNC_LISTING_CHECKPOINT_KEY = "dubizzle:state:sync_listing_checkpoint" SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60 -SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak" +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_failure_streak" SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3 SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60 -SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "iaai:state:sync_listing_bootstrap_continuation_streak" +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_continuation_streak" SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT = max( 1, int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT", "6")), @@ -40,19 +54,45 @@ SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS = max( 60, int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS", str(6 * 60 * 60))), ) -HOURLY_FAILURE_STREAK_KEY = "iaai:state:hourly_failure_streak" +HOURLY_FAILURE_STREAK_KEY = "dubizzle:state:hourly_failure_streak" HOURLY_FAILURE_STREAK_LIMIT = 3 HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # сброс через 6 часов -SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending" -SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task" -SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}" -SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress" -SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total" +SYNC_LISTING_FOLLOWUP_PENDING_KEY = "dubizzle:state:sync_listing_followup_pending" +SYNC_LISTING_TASK_NAME = "dubizzle_scraper.worker.tasks.sync_listing_task" +SYNC_SEGMENT_LOCK_KEY_FMT = "dubizzle:locks:sync_segment:{idx}" +SYNC_SEGMENTS_PROGRESS_KEY = "dubizzle:state:sync_segments_progress" +SYNC_SEGMENTS_TOTAL_KEY = "dubizzle:state:sync_segments_total" +SYNC_SEGMENTED_SCAN_ACTIVE_KEY = "dubizzle:state:sync_segmented_scan_active" SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60 -TASK_PROGRESS_KEY_FMT = "iaai:state:task_progress:{task_id}" -GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts" -SITEMAP_HOURLY_LAST_COUNT_KEY = "iaai:state:sitemap_hourly_last_count" -SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "iaai:state:sitemap_hourly_refresh_offset" +TASK_PROGRESS_KEY_FMT = "dubizzle:state:task_progress:{task_id}" +GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts" +SITEMAP_HOURLY_LAST_COUNT_KEY = "dubizzle:state:sitemap_hourly_last_count" +SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "dubizzle:state:sitemap_hourly_refresh_offset" + + +def _runtime_key(key: str) -> str: + return key.replace("dubizzle:", "dubizzle:") + + +def _origin_prefix() -> str: + return _env_str("DUBIZZLE_ORIGIN_PREFIX", "dubizzle:") + + +SYNC_LISTING_LOCK_KEY = _runtime_key(SYNC_LISTING_LOCK_KEY) +SYNC_FULL_SCAN_DONE_KEY = _runtime_key(SYNC_FULL_SCAN_DONE_KEY) +SYNC_LISTING_CHECKPOINT_KEY = _runtime_key(SYNC_LISTING_CHECKPOINT_KEY) +SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY) +SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY) +HOURLY_FAILURE_STREAK_KEY = _runtime_key(HOURLY_FAILURE_STREAK_KEY) +SYNC_LISTING_FOLLOWUP_PENDING_KEY = _runtime_key(SYNC_LISTING_FOLLOWUP_PENDING_KEY) +SYNC_SEGMENT_LOCK_KEY_FMT = _runtime_key(SYNC_SEGMENT_LOCK_KEY_FMT) +SYNC_SEGMENTS_PROGRESS_KEY = _runtime_key(SYNC_SEGMENTS_PROGRESS_KEY) +SYNC_SEGMENTS_TOTAL_KEY = _runtime_key(SYNC_SEGMENTS_TOTAL_KEY) +SYNC_SEGMENTED_SCAN_ACTIVE_KEY = _runtime_key(SYNC_SEGMENTED_SCAN_ACTIVE_KEY) +TASK_PROGRESS_KEY_FMT = _runtime_key(TASK_PROGRESS_KEY_FMT) +GLOBAL_PROGRESS_TS_KEY = _runtime_key(GLOBAL_PROGRESS_TS_KEY) +SITEMAP_HOURLY_LAST_COUNT_KEY = _runtime_key(SITEMAP_HOURLY_LAST_COUNT_KEY) +SITEMAP_HOURLY_REFRESH_OFFSET_KEY = _runtime_key(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) STALL_WATCHDOG_NAVIGATION_STAGES = { "listing_next_page_started", "listing_resume_progress", @@ -150,7 +190,16 @@ def _clear_task_progress(redis_client: Redis, task_id: str) -> None: logger.warning("Failed to clear task progress for %s", task_id, exc_info=True) -def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int: +def _stall_timeout_for_progress( + stage: str | None, + default_timeout: int, + *, + heartbeat_only: bool = False, +) -> int: + if heartbeat_only: + # Heartbeat-пульсы не считаем полноценным прогрессом: + # при долгом зависании даём только ограниченный grace-период. + return min(int(default_timeout), 120) if stage in STALL_WATCHDOG_NAVIGATION_STAGES: return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS) return int(default_timeout) @@ -165,12 +214,12 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings) discovered_urls = discovery_result.vehicle_urls active_urls = set(discovered_urls) - existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:") + existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix()) new_urls = [url for url in discovered_urls if url not in existing_urls] sold_count = 0 if active_urls: - sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai") + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":")) cars_upserted = 0 cars_failed = 0 @@ -179,7 +228,7 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | failures: list[dict[str, str]] = [] if new_urls: - with IAAIScraper(settings) as scraper: + with DUBIZZLEScraper(settings) as scraper: if progress_callback: scraper.set_progress_callback(progress_callback) batch_size = settings.celery.batch_size @@ -194,7 +243,7 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | protection_events += batch_protection images_upserted += int(batch_result.get("images_upserted", 0)) failures.extend(batch_result.get("failures", [])) - # Fail-fast: если слишком много ошибок — IAAI блокирует, не тратим ресурсы. + # Fail-fast: если слишком много ошибок — DUBIZZLE блокирует, не тратим ресурсы. total_in_batch = batch_ok + batch_fail if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD: logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch) @@ -233,7 +282,7 @@ def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: active_urls = set(discovered_urls) sold_count = 0 if active_urls: - sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai") + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":")) cars_upserted = 0 cars_failed = 0 @@ -241,7 +290,7 @@ def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: images_upserted = 0 failures: list[dict[str, str]] = [] - with IAAIScraper(settings) as scraper: + with DUBIZZLEScraper(settings) as scraper: if progress_callback: scraper.set_progress_callback(progress_callback) batch_size = settings.celery.batch_size @@ -301,12 +350,12 @@ def _hourly_sitemap_rolling_refresh_sync( active_urls = set(discovered_urls) sold_count = 0 if active_urls: - sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai") + sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":")) - existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:") + existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix()) new_urls = [url for url in discovered_urls if url not in existing_urls] - total_active = persistence.count_active_cars_for_lane("iaai:") + total_active = persistence.count_active_cars_for_lane(_origin_prefix()) batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size)) try: offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0) @@ -314,14 +363,14 @@ def _hourly_sitemap_rolling_refresh_sync( offset = 0 refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( - prefix="iaai:", + prefix=_origin_prefix(), offset=offset, limit=batch_size, ) if not refresh_urls and total_active > 0: offset = 0 refresh_urls = persistence.get_active_origin_urls_batch_for_refresh( - prefix="iaai:", + prefix=_origin_prefix(), offset=0, limit=batch_size, ) @@ -352,7 +401,7 @@ def _hourly_sitemap_rolling_refresh_sync( failures: list[dict[str, str]] = [] if target_urls: - with IAAIScraper(settings) as scraper: + with DUBIZZLEScraper(settings) as scraper: if progress_callback: scraper.set_progress_callback(progress_callback) worker_batch_size = settings.celery.batch_size @@ -438,13 +487,20 @@ def _start_stall_watchdog( no_data_count = 0 data = json.loads(raw) stage = data.get("stage") + heartbeat_only = bool(data.get("heartbeat_only")) last_ts = int(data.get("ts") or 0) if not last_ts: continue - effective_stall_timeout = _stall_timeout_for_progress(stage, stall_timeout_seconds) + effective_stall_timeout = _stall_timeout_for_progress( + stage, + stall_timeout_seconds, + heartbeat_only=heartbeat_only, + ) age = int(time.time()) - last_ts if age < effective_stall_timeout: - watchdog_born = time.monotonic() # reset absolute deadline on real progress + # Heartbeat-пульс не должен бесконечно продлевать дедлайн. + if not heartbeat_only: + watchdog_born = time.monotonic() # reset absolute deadline on real progress continue logger.error( "Task %s stalled for %ss at stage=%s payload=%s; cleaning up and restarting", @@ -620,6 +676,15 @@ def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> N def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool: + return _has_running_task_named(celery_app, SYNC_LISTING_TASK_NAME, exclude_task_id=exclude_task_id) + + +def _has_running_task_named( + celery_app, + task_name: str, + *, + exclude_task_id: str | None = None, +) -> bool: try: inspector = celery_app.control.inspect(timeout=1.0) snapshots = [ @@ -634,8 +699,8 @@ def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = for snapshot in snapshots: for entries in snapshot.values(): for entry in entries or []: - task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") - if task_name != SYNC_LISTING_TASK_NAME: + entry_task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "") + if entry_task_name != task_name: continue # Исключаем текущую задачу — она не считается "другой запущенной" entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "") @@ -645,6 +710,59 @@ def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = return False +def _has_running_sync_segment_tasks(celery_app) -> bool: + return _has_running_task_named(celery_app, "dubizzle_scraper.worker.tasks.sync_segment_task") + + +def _set_segmented_scan_active(redis_client: Redis, *, total: int) -> None: + try: + payload = { + "ts": int(time.time()), + "total": int(total), + } + redis_client.set( + SYNC_SEGMENTED_SCAN_ACTIVE_KEY, + json.dumps(payload, ensure_ascii=False), + ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS, + ) + except Exception: + logger.warning("Failed to persist segmented scan active state", exc_info=True) + + +def _clear_segmented_scan_active(redis_client: Redis) -> None: + try: + redis_client.delete(SYNC_SEGMENTED_SCAN_ACTIVE_KEY) + except Exception: + logger.warning("Failed to clear segmented scan active state", exc_info=True) + + +def _is_segmented_scan_in_progress(redis_client: Redis, celery_app) -> bool: + try: + marker = redis_client.get(SYNC_SEGMENTED_SCAN_ACTIVE_KEY) + except Exception: + logger.warning("Failed to read segmented scan active state", exc_info=True) + return False + + if not marker: + return False + + if _has_running_sync_segment_tasks(celery_app): + return True + + try: + queue_len = int(redis_client.llen("scraping") or 0) + except Exception: + logger.warning("Failed to inspect scraping queue length", exc_info=True) + queue_len = 0 + + if queue_len > 0: + return True + + logger.warning("Clearing stale segmented scan active state: no running segment tasks detected") + _clear_segmented_scan_active(redis_client) + return False + + def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool: try: owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY) @@ -902,7 +1020,7 @@ def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[in @shared_task( - name="iaai_scraper.worker.tasks.sync_segment_task", + name="dubizzle_scraper.worker.tasks.sync_segment_task", bind=True, max_retries=2, default_retry_delay=60, @@ -912,7 +1030,7 @@ def sync_segment_task( self, segment_index: int, segment: dict, - lane: str = "iaai_cars", + lane: str = "dubizzle_cars", only_new: bool | None = None, is_bootstrap: bool = False, ): @@ -965,7 +1083,7 @@ def sync_segment_task( try: def _job(): - with IAAIScraper() as scraper: + with DUBIZZLEScraper() as scraper: scraper.set_progress_callback( lambda stage, meta: _update_task_progress( redis_client, @@ -1011,10 +1129,16 @@ def sync_segment_task( segment_index, seg_label, completed, total, ) if total > 0 and completed >= total: - _set_full_scan_done(redis_client, True) + always_full_scan = bool(Settings().discovery.always_full_scan) + _set_full_scan_done(redis_client, False if always_full_scan else True) _clear_sync_checkpoint(redis_client) + _clear_segmented_scan_active(redis_client) _clear_bootstrap_failure_streak(redis_client) - logger.warning("All %d segments completed; bootstrap full scan done", total) + _clear_bootstrap_continuation_streak(redis_client) + if always_full_scan: + logger.warning("All %d segments completed; next hourly run will restart from segment 0", total) + else: + logger.warning("All %d segments completed; bootstrap full scan done", total) return { "status": result.get("status", "success"), @@ -1075,20 +1199,20 @@ def sync_segment_task( @shared_task( - name="iaai_scraper.worker.tasks.sync_vehicle_task", + name="dubizzle_scraper.worker.tasks.sync_vehicle_task", bind=True, max_retries=2, default_retry_delay=30, acks_late=True, ) -def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"): +def sync_vehicle_task(self, vehicle_url: str, lane: str = "dubizzle"): # Скрапинг и upsert одного автомобиля. persistence = _get_persistence() persistence.create_tables() try: def _job(): - with IAAIScraper() as scraper: + with DUBIZZLEScraper() as scraper: return scraper.sync_vehicle(vehicle_url, lane=lane) result = _run_browser_job(_job) @@ -1106,7 +1230,7 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"): @shared_task( - name="iaai_scraper.worker.tasks.sync_listing_task", + name="dubizzle_scraper.worker.tasks.sync_listing_task", bind=True, max_retries=3, default_retry_delay=120, @@ -1116,7 +1240,7 @@ def sync_listing_task( self, make: str | None = None, model: str | None = None, - lane: str = "iaai_cars", + lane: str = "dubizzle_cars", limit: int | None = None, only_new: bool | None = None, ): @@ -1159,11 +1283,12 @@ def sync_listing_task( continuation_streak, should_continue = _bump_bootstrap_continuation_streak(redis_client) if not should_continue: _clear_followup_pending(redis_client) - # Переключаемся на часовой beat-режим и останавливаем - # немедленные bootstrap continuation, чтобы не зациклиться. + # Останавливаем немедленные bootstrap continuation, чтобы не зациклиться. + # Фиксируем done + чистим checkpoint даже при always_full_scan: + # это безопасно и предотвращает повторный старт со stale-сегмента. + _set_full_scan_done(redis_client, True) + _clear_sync_checkpoint(redis_client) if not always_full_scan: - _set_full_scan_done(redis_client, True) - _clear_sync_checkpoint(redis_client) logger.error( "Bootstrap continuation stopped after %d immediate runs; switching to hourly schedule", continuation_streak, @@ -1178,7 +1303,7 @@ def sync_listing_task( try: followup_expires = max(int(lock_ttl), int(delay_seconds) + 300) self.app.send_task( - "iaai_scraper.worker.tasks.sync_listing_task", + "dubizzle_scraper.worker.tasks.sync_listing_task", kwargs={ "make": make, "model": model, @@ -1247,17 +1372,15 @@ def sync_listing_task( ) full_scan_done_before_run = _is_full_scan_done(redis_client) + hourly_mode = settings.discovery.hourly_mode.strip().lower() + discovery_mode = settings.discovery.mode.strip().lower() always_full_scan = bool(settings.discovery.always_full_scan) force_bootstrap_full_scan = always_full_scan or (not full_scan_done_before_run) effective_limit = None if force_bootstrap_full_scan else limit effective_only_new = False if force_bootstrap_full_scan else only_new - hourly_mode = settings.discovery.hourly_mode.strip().lower() - discovery_mode = settings.discovery.mode.strip().lower() - if always_full_scan: - # Для режима "полный прогон каждый запуск" приоритет — устойчивый resume, - # поэтому принудительно уходим в listing/segmented path вместо sitemap-mainline. - discovery_mode = "listing" prefer_sitemap_mainline = ( + discovery_mode == "sitemap" + and not force_bootstrap_full_scan and make is None and model is None @@ -1268,22 +1391,23 @@ def sync_listing_task( use_hourly_sitemap_sync = (not always_full_scan) and full_scan_done_before_run and prefer_sitemap_mainline # Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента. - # Используется только во время bootstrap для пропуска уже обработанных сегментов. - # Никаких page-level resume — внутри сегмента всегда стартуем с page 1. + # Используется во время bootstrap/full-scan resume. last_completed_segment: int | None = None - if force_bootstrap_full_scan: + resume_from_checkpoint = force_bootstrap_full_scan and ( + always_full_scan or (not full_scan_done_before_run) + ) + if resume_from_checkpoint: last_completed_segment = _load_last_completed_segment(redis_client) else: - # После завершения bootstrap чекпоинт не нужен никогда. _clear_sync_checkpoint(redis_client) if force_bootstrap_full_scan: logger.info( "Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run", ) - if always_full_scan: + if discovery_mode == "algolia": logger.info( - "Always full scan mode enabled (IAAI_ALWAYS_FULL_SCAN=true): using listing resume path", + "Algolia full scan enabled: using segmented Algolia traversal when configured", ) logger.info( @@ -1413,28 +1537,31 @@ def sync_listing_task( self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id}) - # Определяем сегменты из конфига. segments = parse_listing_segments(settings.listing.listing_segments_json) use_segmented = ( bool(segments) and make is None and model is None and not prefer_sitemap_mainline - and discovery_mode != "sitemap" + and discovery_mode in {"listing", "algolia"} ) - if prefer_sitemap_mainline: - if discovery_mode != "sitemap": - logger.warning( - "Unfiltered full scan forcing sitemap discovery despite IAAI_DISCOVERY_MODE=%s", - discovery_mode or "unset", - ) - if segments: - logger.info("Ignoring configured listing segments for unfiltered sitemap full scan") + if prefer_sitemap_mainline and segments: + logger.info("Ignoring configured listing segments for unfiltered sitemap full scan") - # --- Параллельный диспатч сегментов: dispatch & exit --- if use_segmented and settings.celery.parallel_segments: - # Сегменты уже завершённые (для bootstrap resume) пропускаем по Redis SET. + segmented_scan_in_progress = False + if force_bootstrap_full_scan: + segmented_scan_in_progress = _is_segmented_scan_in_progress(redis_client, self.app) + if segmented_scan_in_progress: + logger.warning("Parallel segments: scan already in progress, skipping duplicate dispatch") + return { + "status": "skipped", + "reason": "segmented_scan_in_progress", + "task_id": task_id, + "mode": "parallel_segments", + } + already_completed: set[int] = set() if force_bootstrap_full_scan: try: @@ -1443,17 +1570,20 @@ def sync_listing_task( except Exception: already_completed = set() + if always_full_scan: + already_completed = set() + pending = [ (idx, seg) for idx, seg in enumerate(segments) if idx not in already_completed ] if not pending: - # Всё уже сделано — фиксируем bootstrap done. if force_bootstrap_full_scan: _set_full_scan_done(redis_client, True) _clear_sync_checkpoint(redis_client) _clear_bootstrap_failure_streak(redis_client) + _clear_segmented_scan_active(redis_client) logger.warning("Parallel segments: nothing to dispatch (all completed)") return { "status": "success", @@ -1464,15 +1594,18 @@ def sync_listing_task( "segments_already_completed": len(already_completed), } - # При первом запуске bootstrap фиксируем total, чтобы знать когда остановиться. - if force_bootstrap_full_scan and not already_completed: + if force_bootstrap_full_scan and (always_full_scan or not already_completed): _reset_segments_progress(redis_client, len(segments)) + already_completed = set() + + if force_bootstrap_full_scan: + _set_segmented_scan_active(redis_client, total=len(segments)) dispatched = 0 for idx, seg in pending: try: self.app.send_task( - "iaai_scraper.worker.tasks.sync_segment_task", + "dubizzle_scraper.worker.tasks.sync_segment_task", kwargs={ "segment_index": idx, "segment": seg, @@ -1498,13 +1631,11 @@ def sync_listing_task( "segments_dispatched": dispatched, "segments_already_completed": len(already_completed), } - # --- конец параллельной ветки --- resume_from_segment = 0 if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None: resume_from_segment = max(0, last_completed_segment + 1) if resume_from_segment >= len(segments): - # Все сегменты уже пройдены — чекпоинт устарел, начинаем заново. logger.info( "Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0", last_completed_segment, len(segments), @@ -1527,7 +1658,7 @@ def sync_listing_task( ) def _job(): - with IAAIScraper() as scraper: + with DUBIZZLEScraper() as scraper: scraper.set_progress_callback(_progress_cb_main) if use_segmented: return scraper.sync_listing_segmented( @@ -1538,7 +1669,7 @@ def sync_listing_task( start_page=1, progress_callback=( (lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx)) - if force_bootstrap_full_scan else None + if resume_from_checkpoint else None ), ) return scraper.sync_listing( @@ -1654,7 +1785,7 @@ def sync_listing_task( if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): try: self.app.send_task( - "iaai_scraper.worker.tasks.sync_listing_task", + "dubizzle_scraper.worker.tasks.sync_listing_task", kwargs={ "make": make, "model": model, diff --git a/entrypoint.sh b/entrypoint.sh index 53a9fd0..c646b36 100644 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -4,7 +4,7 @@ set -euo pipefail is_worker_command() { local joined="$*" case "$joined" in - *"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*) + *"celery -A dubizzle_scraper.worker.celery_app worker"*|*" celery -A dubizzle_scraper.worker.celery_app worker"*) return 0 ;; esac @@ -36,19 +36,19 @@ start_proxy_bridge_if_needed() { echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..." - if [ -z "${IAAI_PROXY_SERVER:-}" ]; then - export IAAI_PROXY_SERVER="http://127.0.0.1:8899" - elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then - export IAAI_PROXY_SERVER="http://127.0.0.1:8899" + if [ -z "${DUBIZZLE_PROXY_SERVER:-}" ]; then + export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899" + elif [ "${DUBIZZLE_PROXY_SERVER}" = "http://localhost:8899" ]; then + export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899" fi - echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}" - python -m iaai_scraper.proxy_bridge & + echo "[entrypoint] Using browser proxy: ${DUBIZZLE_PROXY_SERVER}" + python -m dubizzle_scraper.proxy_bridge & BRIDGE_PID=$! sleep 1 if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then - echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start" + echo "[entrypoint] ERROR: dubizzle_scraper.proxy_bridge failed to start" exit 1 fi @@ -65,13 +65,13 @@ start_self_heal_watchdog_if_worker() { # Удаляем его перед запуском worker-процесса. rm -f /tmp/celery-worker.pid - if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then + if [ "${DUBIZZLE_SELF_HEAL_ENABLED:-true}" = "false" ]; then echo "[entrypoint] Self-heal watchdog disabled" return 0 fi echo "[entrypoint] Starting self-heal watchdog for worker..." - python -m iaai_scraper.worker.self_heal & + python -m dubizzle_scraper.worker.self_heal & SELF_HEAL_PID=$! sleep 1 diff --git a/iaai_scraper/core/config.py b/iaai_scraper/core/config.py deleted file mode 100644 index 859079d..0000000 --- a/iaai_scraper/core/config.py +++ /dev/null @@ -1,308 +0,0 @@ -import json -import os -from dataclasses import dataclass, field -from pathlib import Path - -from dotenv import load_dotenv - -load_dotenv() - - -TRUE_VALUES = {"1", "true", "yes", "on"} - - -# Хелперы для чтения env-переменных с приведением типов - -def _env_str(name: str, default: str) -> str: - value = os.getenv(name) - return value if value is not None else default - - -def _env_optional_str(name: str) -> str | None: - value = os.getenv(name) - if value is None: - return None - value = value.strip() - return value or None - - -def _env_path_str(name: str) -> str | None: - value = _env_optional_str(name) - if value is None: - return None - return str(Path(value).expanduser()) - - -def _env_bool(name: str, default: bool) -> bool: - fallback = "true" if default else "false" - return _env_str(name, fallback).strip().lower() in TRUE_VALUES - - -def _env_int(name: str, default: int) -> int: - return int(_env_str(name, str(default)).strip()) - - -def _env_float(name: str, default: float) -> float: - return float(_env_str(name, str(default)).strip()) - - -# Конфиг браузерного отпечатка (User-Agent, viewport, timezone) - -@dataclass(slots=True) -class FingerprintConfig: - user_agent: str = ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/135.0.0.0 Safari/537.36" - ) - viewport_presets: tuple[dict[str, int], ...] = ( - {"width": 1920, "height": 1080}, - {"width": 1600, "height": 900}, - {"width": 1536, "height": 864}, - {"width": 1440, "height": 900}, - {"width": 1366, "height": 768}, - ) - timezone_candidates: tuple[str, ...] = ( - "America/New_York", - "America/Chicago", - "America/Los_Angeles", - ) - locale: str = "en-US" - sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"' - - -# Конфиг перехвата сетевых запросов (лимиты на кол-во) - -@dataclass(slots=True) -class CaptureConfig: - capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True) - max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40) - max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30) - - -# Конфиг пауз между действиями (имитация человека) - -@dataclass(slots=True) -class HumanPaceConfig: - enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True) - after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5) - after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2) - after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5) - after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2) - before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1) - before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3) - after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05) - after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15) - between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05) - between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15) - after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8) - after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8) - - -# Конфиг сбора листинга (URL, лимиты страниц и машин) - -@dataclass(slots=True) -class ListingConfig: - cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars") - max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999) - max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000) - page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500) - include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True) - collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False) - # Порог раннего останова: если доля уже известных машин на странице >= этого значения, - # прекращаем листать — все новые машины уже найдены. 0 = отключено. - early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8) - # Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин). - # JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка. - # Пустая строка = без сегментации (backward compatible). - listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "") - - -# Список брендов IAAI для автоматической сегментации. -# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким. -IAAI_DEFAULT_MAKES: tuple[str, ...] = ( - "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", - "KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU", - "VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI", - "RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN", - "MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE", - "MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI", - "SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI", - "OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART", - "RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI", - "BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS", - "MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE", -) - -# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов. -IAAI_PAGINATION_CEILING = 22_600 - -# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам. -_LARGE_MAKES: frozenset[str] = frozenset({ - "TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI", - "KIA", "DODGE", "JEEP", -}) -_YEAR_SPLITS: tuple[tuple[int, int], ...] = ( - (1900, 2012), - (2013, 2019), - (2020, 2027), -) - - -def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]: - """Парсит IAAI_LISTING_SEGMENTS в список сегментов. - - Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None). - Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES. - Крупные бренды автоматически разбиваются по диапазонам годов. - """ - raw = raw.strip() - if not raw: - return [] - if raw.lower() == "auto": - segments: list[dict[str, str | int | None]] = [] - for m in IAAI_DEFAULT_MAKES: - if m in _LARGE_MAKES: - for yr_min, yr_max in _YEAR_SPLITS: - segments.append({"make": m, "year_min": yr_min, "year_max": yr_max}) - else: - segments.append({"make": m, "year_min": None, "year_max": None}) - return segments - try: - data = json.loads(raw) - except (json.JSONDecodeError, ValueError): - return [] - if not isinstance(data, list): - return [] - segments = [] - for item in data: - if isinstance(item, str): - segments.append({"make": item.upper(), "year_min": None, "year_max": None}) - elif isinstance(item, dict): - segments.append({ - "make": str(item.get("make") or "").upper() or None, - "year_min": int(item["year_min"]) if item.get("year_min") is not None else None, - "year_max": int(item["year_max"]) if item.get("year_max") is not None else None, - }) - return segments - - -# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle) - -@dataclass(slots=True) -class DatabaseConfig: - url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper") - echo: bool = _env_bool("IAAI_DATABASE_ECHO", False) - pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5) - max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10) - pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800) - auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False) - - -# --- Конфиг Redis (URL для Celery broker) --- - -@dataclass(slots=True) -class RedisConfig: - url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0") - socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0) - socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0) - health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30) - - -# --- Конфиг Discovery (режим обнаружения, hourly batch) --- - -@dataclass(slots=True) -class DiscoveryConfig: - mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap") - hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh") - hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500) - always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", True) - - -# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) --- - -@dataclass(slots=True) -class CeleryConfig: - broker_url: str = _env_str("CELERY_BROKER_URL", "") - result_backend: str = _env_str("CELERY_RESULT_BACKEND", "") - task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300) - task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600) - task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600) - worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4) - worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5) - broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200) - beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60) - beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None - batch_size: int = _env_int("CELERY_BATCH_SIZE", 50) - parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8) - parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False) - block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True) - - -# --- Конфиг прокси (server, username, password) --- - -@dataclass(slots=True) -class ProxyConfig: - server: str | None = _env_optional_str("IAAI_PROXY_SERVER") - username: str | None = _env_optional_str("IAAI_PROXY_USERNAME") - password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD") - - @property - def enabled(self) -> bool: - return bool(self.server) - - def to_playwright_dict(self) -> dict[str, str] | None: - if not self.server: - return None - result: dict[str, str] = {"server": self.server} - if self.username: - result["username"] = self.username - if self.password: - result["password"] = self.password - return result - - -# Главный объект настроек: собирает все блоки конфигурации - -@dataclass(slots=True) -class Settings: - home_url: str = "https://www.iaai.com/" - default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000) - network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400) - fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000) - fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1) - fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000) - max_retries: int = _env_int("IAAI_MAX_RETRIES", 3) - retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5) - retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0) - retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25) - headless: bool = _env_bool("IAAI_HEADLESS", True) - browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto") - log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO") - log_file: str | None = _env_optional_str("IAAI_LOG_FILE") - enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True) - sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False) - raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON") - tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE") - runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE") - scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60) - fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig) - capture: CaptureConfig = field(default_factory=CaptureConfig) - pace: HumanPaceConfig = field(default_factory=HumanPaceConfig) - listing: ListingConfig = field(default_factory=ListingConfig) - database: DatabaseConfig = field(default_factory=DatabaseConfig) - redis: RedisConfig = field(default_factory=RedisConfig) - celery: CeleryConfig = field(default_factory=CeleryConfig) - proxy: ProxyConfig = field(default_factory=ProxyConfig) - discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig) - - @property - def parallel_tabs(self) -> int: - return self.celery.parallel_tabs - - @property - def block_resources(self) -> bool: - return self.celery.block_resources - -# Глобальный синглтон — используется по умолчанию во всех модулях. -settings = Settings() diff --git a/pyproject.toml b/pyproject.toml index 7b5357a..6583733 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,9 +3,9 @@ requires = ["setuptools>=68", "wheel"] build-backend = "setuptools.build_meta" [project] -name = "iaai-scraper" +name = "dubizzle-scraper" version = "0.1.0" -description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL" +description = "Dubizzle scraper service with FastAPI, Celery, Playwright and PostgreSQL" readme = "README.md" requires-python = ">=3.11" dependencies = [ @@ -29,13 +29,13 @@ dev = [ ] [project.scripts] -iaai = "iaai_scraper.cli:main" +dubizzle = "dubizzle_scraper.cli:main" [tool.setuptools] include-package-data = true [tool.setuptools.packages.find] -include = ["iaai_scraper*"] +include = ["dubizzle_scraper*"] [tool.pytest.ini_options] addopts = "-q --disable-warnings" diff --git a/runtime_config.json b/runtime_config.json index bb43bcb..a1678ec 100644 --- a/runtime_config.json +++ b/runtime_config.json @@ -5,7 +5,7 @@ "ids_next_size": null, "ids_max_pages": null, "condition_check_enabled": false, - "lane": "iaai_cars", + "lane": "dubizzle_cars", "only_new": false, "limit": null }, diff --git a/tests/test_db.py b/tests/test_db.py index 7720590..2033538 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -6,10 +6,10 @@ from pathlib import Path from sqlalchemy import select -from iaai_scraper.core.config import Settings -from iaai_scraper.storage.db import PersistenceService -from iaai_scraper.storage.models import Car, Image, SyncRun -from iaai_scraper.storage.schemas import CarRecord, ImageRecord +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.storage.db import PersistenceService +from dubizzle_scraper.storage.models import Car, Image, SyncRun +from dubizzle_scraper.storage.schemas import CarRecord, ImageRecord class TestPersistenceServiceIntegration(unittest.TestCase): @@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase): @staticmethod def _record(origin_id: str, *, price: int = 1000) -> CarRecord: return CarRecord( - parser_id=f"iaai:{origin_id}", + parser_id=f"dubizzle:{origin_id}", brand="Toyota", model="Camry", year=2014, price=price, - origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US", origin_id=origin_id, slug=f"toyota-camry-{origin_id}", images=[ ImageRecord( - fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", - preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300", + fullres_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633", + preview_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=400&height=300", order_index=0, ) ], @@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase): second = self._record("888") second.images = [ ImageRecord( - fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633", - preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + fullres_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=845&height=633", + preview_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300", order_index=0, ) ] @@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase): def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None: first = self._record("OLD-ID") - first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + first.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US" self.persistence.upsert_car(first) second = self._record("NEW-ID") - second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US" + second.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US" result = self.persistence.upsert_car(second) self.assertEqual(result["action"], "updated") diff --git a/tests/test_listing.py b/tests/test_listing.py index 7d8e991..99fd6b3 100644 --- a/tests/test_listing.py +++ b/tests/test_listing.py @@ -2,9 +2,9 @@ from __future__ import annotations import unittest -from iaai_scraper.browser.pace import HumanPacer -from iaai_scraper.core.config import Settings -from iaai_scraper.browser.listing import ListingCollector +from dubizzle_scraper.browser.pace import HumanPacer +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.browser.listing import ListingCollector class _FakePage: @@ -58,8 +58,8 @@ class TestListingUnit(unittest.TestCase): self.assertEqual( links, [ - ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"), - ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"), + ("https://www.dubizzle.com/VehicleDetail/45184893~US", "45184893"), + ("https://www.dubizzle.com/VehicleDetail/45171480~US", "45171480"), ], ) diff --git a/tests/test_mappers.py b/tests/test_mappers.py index 621a5fc..d59d610 100644 --- a/tests/test_mappers.py +++ b/tests/test_mappers.py @@ -2,22 +2,109 @@ from __future__ import annotations import unittest -from iaai_scraper.parsing.mapper import CarMapper +from dubizzle_scraper.parsing.mapper import CarMapper class TestCarMapper(unittest.TestCase): def setUp(self) -> None: self.mapper = CarMapper() + def test_extracts_make_model_from_algolia_details(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://dubizzle.com/s/DOBI7J3", + vehicle_summary={ + "id": 16810399, + "details_v2": { + "tertiary": [ + {"slug": "make", "value": {"en": "Toyota", "ar": "تويوتا"}}, + {"slug": "model", "value": {"en": "Land Cruiser", "ar": "لاند كروزر"}}, + ] + }, + "category_v2": { + "names_en": ["Motors", "Used Cars", "Toyota", "Land Cruiser"], + "slug_paths": [ + "motors", + "motors/used-cars", + "motors/used-cars/toyota", + "motors/used-cars/toyota/land-cruiser", + ], + }, + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "Toyota") + self.assertEqual(record.model, "Land Cruiser") + + def test_extracts_make_model_from_legacy_details(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://dubizzle.com/s/DOBIlegacy", + vehicle_summary={ + "objectID": "item:legacy:1", + "details": { + "Make": {"en": {"label": "Make", "value": "Honda"}}, + "Model": {"en": {"label": "Model", "value": "Civic"}}, + }, + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "Honda") + self.assertEqual(record.model, "Civic") + + def test_extracts_extended_algolia_fields(self) -> None: + record = self.mapper.map_to_car_record( + vehicle_url="https://dubizzle.com/s/DOBGaGs", + vehicle_summary={ + "id": 16345114, + "year": 2012, + "price": 59000.0, + "kilometers": 91800, + "seller_type": "DL", + "site": {"en": "Dubai"}, + "category_v2": { + "names_en": ["Motors", "Used Cars", "Porsche", "Cayenne"], + "slug_paths": [ + "motors", + "motors/used-cars", + "motors/used-cars/porsche", + "motors/used-cars/porsche/cayenne", + ], + }, + "details": { + "Body Type": {"en": {"value": "SUV"}}, + "Transmission Type": {"en": {"value": "Automatic Transmission"}}, + "Exterior Color": {"en": {"value": "Brown"}}, + "Steering Side": {"en": {"value": "Left Hand"}}, + "Engine Capacity (cc)": {"en": {"value": "4800 cc"}}, + "Make": {"en": {"value": "Porsche"}}, + "Model": {"en": {"value": "Cayenne"}}, + }, + "photo_mains": ["https://dbz-images.dubizzle.com/images/example.jpeg?impolicy=dpv"], + }, + payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, + ) + + self.assertEqual(record.brand, "Porsche") + self.assertEqual(record.model, "Cayenne") + self.assertEqual(record.price, 59000) + self.assertEqual(record.mileage, 91800) + self.assertEqual(record.color, "brown") + self.assertEqual(record.body_type, "SUV") + self.assertEqual(record.gearbox, "AT") + self.assertEqual(record.steering_wheel, "LEFT") + self.assertEqual(record.engine_volume, 4800) + self.assertEqual(record.country, "AE") + def test_deduplicates_images_by_image_key(self) -> None: urls = [ - "https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150", - "https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300", + "https://vis.dubizzle.com/resizer?imageKeys=1&width=200&height=150", + "https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300", ] record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US", vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, ) @@ -28,7 +115,7 @@ class TestCarMapper(unittest.TestCase): def test_no_damage_marker_is_not_damaged(self) -> None: record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/123~US", + vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US", vehicle_summary={"make": "Ford", "model": "Focus"}, payload_insights={ "vehicle_core": {}, @@ -43,7 +130,7 @@ class TestCarMapper(unittest.TestCase): def test_unknown_empty_values_and_normalization(self) -> None: record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/999~US", + vehicle_url="https://www.dubizzle.com/VehicleDetail/999~US", vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, ) @@ -54,7 +141,7 @@ class TestCarMapper(unittest.TestCase): # Нормализация регистра и пробелов. record2 = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/888~US", + vehicle_url="https://www.dubizzle.com/VehicleDetail/888~US", vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "}, payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}}, ) @@ -63,7 +150,7 @@ class TestCarMapper(unittest.TestCase): def test_price_and_currency_parsing(self) -> None: record = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/777~US", + vehicle_url="https://www.dubizzle.com/VehicleDetail/777~US", vehicle_summary={"make": "Toyota", "model": "Corolla"}, payload_insights={ "vehicle_core": {}, @@ -76,7 +163,7 @@ class TestCarMapper(unittest.TestCase): self.assertEqual(record.price, 5200) record2 = self.mapper.map_to_car_record( - vehicle_url="https://www.iaai.com/VehicleDetail/778~US", + vehicle_url="https://www.dubizzle.com/VehicleDetail/778~US", vehicle_summary={"make": "Toyota", "model": "Corolla"}, payload_insights={ "vehicle_core": {}, diff --git a/tests/test_parser.py b/tests/test_parser.py index 72a2b6c..abd9494 100644 --- a/tests/test_parser.py +++ b/tests/test_parser.py @@ -2,7 +2,7 @@ from __future__ import annotations import unittest -from iaai_scraper.parsing.parser import VehicleParser +from dubizzle_scraper.parsing.parser import VehicleParser class TestVehicleParserUnit(unittest.TestCase): @@ -29,11 +29,11 @@ class TestVehicleParserUnit(unittest.TestCase): self.assertEqual(parsed["model"], "CAMRY") def test_extract_image_urls_filters_and_deduplicates(self) -> None: - vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US" + vehicle_url = "https://www.dubizzle.com/VehicleDetail/45089484~US" payloads = [{"imageUrls": [ - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633", - "https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633", + "https://vis.dubizzle.com/resizer?imageKeys=99999999~SID2&width=845&height=633", ]}] urls = self.parser._extract_image_urls(payloads, "", vehicle_url) self.assertEqual(len(urls), 1) diff --git a/tests/test_resilience.py b/tests/test_resilience.py index 8b2bf81..32b6b2e 100644 --- a/tests/test_resilience.py +++ b/tests/test_resilience.py @@ -4,17 +4,17 @@ import unittest from types import SimpleNamespace from unittest.mock import MagicMock, patch -from iaai_scraper.scraper import IAAIScraper -from iaai_scraper.core.config import Settings -from iaai_scraper.worker import tasks +from dubizzle_scraper.scraper import DUBIZZLEScraper +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.worker import tasks class TestResilience(unittest.TestCase): - def _make_scraper(self) -> IAAIScraper: + def _make_scraper(self) -> DUBIZZLEScraper: s = Settings() s.log_level = "CRITICAL" s.database.url = "sqlite://" - return IAAIScraper(s) + return DUBIZZLEScraper(s) def test_update_task_progress_updates_global_marker(self) -> None: redis_client = MagicMock() @@ -60,11 +60,11 @@ class TestResilience(unittest.TestCase): result = scraper._sync_listing_streaming( make=None, model=None, - lane="iaai_cars", + lane="dubizzle_cars", limit=None, effective_only_new=False, started_at=0.0, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST", + listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST", ) self.assertEqual(result["total"], 0) diff --git a/tests/test_scraper.py b/tests/test_scraper.py index 7d6192a..9bba52d 100644 --- a/tests/test_scraper.py +++ b/tests/test_scraper.py @@ -5,29 +5,29 @@ from concurrent.futures import TimeoutError as FuturesTimeoutError from types import SimpleNamespace from unittest.mock import MagicMock, patch -from iaai_scraper.core.config import Settings -from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError -from iaai_scraper.scraper import IAAIScraper -from iaai_scraper.storage.schemas import CarRecord +from dubizzle_scraper.core.config import Settings +from dubizzle_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError +from dubizzle_scraper.scraper import DUBIZZLEScraper +from dubizzle_scraper.storage.schemas import CarRecord def make_db_record(origin_id: str) -> dict[str, object]: return CarRecord( - parser_id=f"iaai:{origin_id}", + parser_id=f"dubizzle:{origin_id}", brand="Toyota", model="Camry", - origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US", + origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US", origin_id=origin_id, slug=f"toyota-camry-{origin_id}", ).model_dump(mode="json") class TestScraperSync(unittest.TestCase): - def _make_scraper(self) -> IAAIScraper: + def _make_scraper(self) -> DUBIZZLEScraper: s = Settings() s.log_level = "CRITICAL" s.database.url = "sqlite://" - return IAAIScraper(s) + return DUBIZZLEScraper(s) def test_sync_vehicle_uses_db_record(self) -> None: scraper = self._make_scraper() @@ -37,7 +37,7 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0}) scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")}) - result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US") + result = scraper.sync_vehicle("https://www.dubizzle.com/VehicleDetail/111~US") self.assertEqual(result["status"], "success") self.assertIn("trace_id", result) @@ -50,13 +50,13 @@ class TestScraperSync(unittest.TestCase): scraper.persistence.start_sync_run = MagicMock(return_value=2) scraper.persistence.finish_sync_run = MagicMock() scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=( - {"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"}, + {"https://www.dubizzle.com/VehicleDetail/111~US"}, {"dubizzle:222"}, )) scraper.collect_listing = MagicMock(return_value={ "vehicle_urls": [ - "https://www.iaai.com/VehicleDetail/111~US", - "https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/VehicleDetail/333~US", + "https://www.dubizzle.com/VehicleDetail/111~US", + "https://www.dubizzle.com/VehicleDetail/222~US", + "https://www.dubizzle.com/VehicleDetail/333~US", ] }) scraper.sync_batch = MagicMock(return_value={ @@ -79,7 +79,7 @@ class TestScraperSync(unittest.TestCase): }) result2 = scraper2.sync_listing( only_new=True, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA", year_min=2020, year_max=2027, ) self.assertEqual(result2["cars_upserted"], 10) @@ -155,35 +155,35 @@ class TestScraperSync(unittest.TestCase): self.assertEqual(result["status"], "partial_success") def test_build_segment_listing_url(self) -> None: - base = "https://www.iaai.com/Vehiclelisting/Cars" + base = "https://www.dubizzle.com/Vehiclelisting/Cars" self.assertEqual( - IAAIScraper._build_segment_listing_url(base, "TOYOTA"), - "https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA", + DUBIZZLEScraper._build_segment_listing_url(base, "TOYOTA"), + "https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA", ) self.assertEqual( - IAAIScraper._build_segment_listing_url(base, "LAND ROVER"), - "https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER", + DUBIZZLEScraper._build_segment_listing_url(base, "LAND ROVER"), + "https://www.dubizzle.com/Vehiclelisting/Cars?Make=LAND%20ROVER", ) - self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base) - self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base) + self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, None), base) + self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, ""), base) def test_guard_and_protection_detection(self) -> None: with self.assertRaises(AntiBotDetectedError): - IAAIScraper._raise_if_blocked_or_incomplete( + DUBIZZLEScraper._raise_if_blocked_or_incomplete( {"dom_hints": {"has_captcha_text": True, "has_antibot_text": False}, "access_notes": {}, "vehicle_summary": {}}, - "https://www.iaai.com/VehicleDetail/999~US", + "https://www.dubizzle.com/VehicleDetail/999~US", ) with self.assertRaises(SiteStructureChangedError): - IAAIScraper._raise_if_blocked_or_incomplete( + DUBIZZLEScraper._raise_if_blocked_or_incomplete( {"dom_hints": {"has_captcha_text": False, "has_antibot_text": False}, "access_notes": {"possible_captcha": False, "possible_antibot": False}, "vehicle_summary": {}}, - "https://www.iaai.com/VehicleDetail/999~US", + "https://www.dubizzle.com/VehicleDetail/999~US", ) - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) - self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) - self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) + self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT"))) + self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("captcha challenge"))) + self.assertFalse(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("plain validation error"))) def test_close_resets_browser_state(self) -> None: scraper = self._make_scraper() @@ -201,7 +201,7 @@ class TestScraperSync(unittest.TestCase): scraper = self._make_scraper() page = MagicMock() page_result = SimpleNamespace( - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")], + vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/123~US")], ) scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) scraper.listing_collector.open_cars_listing = MagicMock() @@ -228,7 +228,7 @@ class TestScraperSync(unittest.TestCase): page = MagicMock() page_result = SimpleNamespace( page_number=1, - vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")], + vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/999~US", lot_number="999")], next_page_detected=False, ) @@ -245,7 +245,7 @@ class TestScraperSync(unittest.TestCase): "year_max": None, }) scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result) - scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"]) + scraper._extract_page_urls = MagicMock(return_value=["https://www.dubizzle.com/VehicleDetail/999~US"]) scraper.sync_batch = MagicMock(return_value={ "cars_upserted": 1, "cars_failed": 0, @@ -256,11 +256,11 @@ class TestScraperSync(unittest.TestCase): result = scraper._sync_listing_streaming( make=None, model=None, - lane="iaai_cars", + lane="dubizzle_cars", limit=None, effective_only_new=False, started_at=0.0, - listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE", + listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=EAGLE", ) scraper.listing_collector.open_cars_listing.assert_called_once() @@ -278,9 +278,9 @@ class TestScraperSync(unittest.TestCase): }) urls = [ - "https://www.iaai.com/VehicleDetail/111~US", - "https://www.iaai.com/VehicleDetail/222~US", - "https://www.iaai.com/VehicleDetail/333~US", + "https://www.dubizzle.com/VehicleDetail/111~US", + "https://www.dubizzle.com/VehicleDetail/222~US", + "https://www.dubizzle.com/VehicleDetail/333~US", ] first_record = CarRecord.model_validate(make_db_record("111")) @@ -298,8 +298,8 @@ class TestScraperSync(unittest.TestCase): yield 0, first_record raise FuturesTimeoutError() - with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ - patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ + with patch("dubizzle_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \ + patch("dubizzle_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \ patch.object(scraper, "_browser_fallback_parallel", return_value={ "records": [], "failures": [], diff --git a/tests/test_self_heal.py b/tests/test_self_heal.py index f5c74e1..a963766 100644 --- a/tests/test_self_heal.py +++ b/tests/test_self_heal.py @@ -3,7 +3,7 @@ from __future__ import annotations import unittest from unittest.mock import MagicMock, patch -from iaai_scraper.worker import self_heal +from dubizzle_scraper.worker import self_heal class TestSelfHeal(unittest.TestCase): @@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase): redis_client = MagicMock() redis_client.get.side_effect = lambda key: { self_heal.GLOBAL_PROGRESS_TS_KEY: None, - "iaai:state:task_progress:a": '{"ts": 100}', - "iaai:state:task_progress:b": '{"ts": 250}', - "iaai:state:task_progress:c": '{"ts": 150}', + "dubizzle:state:task_progress:a": '{"ts": 100}', + "dubizzle:state:task_progress:b": '{"ts": 250}', + "dubizzle:state:task_progress:c": '{"ts": 150}', }.get(key) redis_client.scan_iter.return_value = [ - "iaai:state:task_progress:a", - "iaai:state:task_progress:b", - "iaai:state:task_progress:c", + "dubizzle:state:task_progress:a", + "dubizzle:state:task_progress:b", + "dubizzle:state:task_progress:c", ] ts = self_heal._read_last_progress_ts(redis_client) @@ -38,20 +38,20 @@ class TestSelfHeal(unittest.TestCase): redis_client = MagicMock() redis_client.get.side_effect = lambda key: { self_heal.GLOBAL_PROGRESS_TS_KEY: None, - "iaai:state:task_progress:a": "{bad-json}", - "iaai:state:task_progress:b": '{"foo": "bar"}', + "dubizzle:state:task_progress:a": "{bad-json}", + "dubizzle:state:task_progress:b": '{"foo": "bar"}', }.get(key) redis_client.scan_iter.return_value = [ - "iaai:state:task_progress:a", - "iaai:state:task_progress:b", + "dubizzle:state:task_progress:a", + "dubizzle:state:task_progress:b", ] ts = self_heal._read_last_progress_ts(redis_client) self.assertIsNone(ts) - @patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) - @patch("iaai_scraper.worker.self_heal.os.kill") + @patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("dubizzle_scraper.worker.self_heal.os.kill") @patch("builtins.open") def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None: open_mock.return_value.__enter__.return_value.read.return_value = "123" @@ -64,8 +64,8 @@ class TestSelfHeal(unittest.TestCase): self.assertEqual(kill_mock.call_args_list[1].args, (123, 0)) self.assertEqual(kill_mock.call_args_list[2].args[0], 123) - @patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None) - @patch("iaai_scraper.worker.self_heal.os.kill") + @patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None) + @patch("dubizzle_scraper.worker.self_heal.os.kill") @patch("builtins.open") def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None: open_mock.return_value.__enter__.return_value.read.return_value = "123" diff --git a/tests/test_utils.py b/tests/test_utils.py index 2927c1e..bd61353 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -2,8 +2,8 @@ from __future__ import annotations import unittest -from iaai_scraper.core.utils import deep_find_key -from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS +from dubizzle_scraper.core.utils import deep_find_key +from dubizzle_scraper.core.config import parse_listing_segments, _AUTO_YEAR_SPLITS class TestDeepFindKey(unittest.TestCase): @@ -28,32 +28,19 @@ class TestParseListingSegments(unittest.TestCase): self.assertEqual(parse_listing_segments("invalid"), []) def test_auto_segments_complete_coverage(self) -> None: - """auto: все бренды покрыты, крупные разбиты по годам, годы без дыр.""" + """auto: все годовые диапазоны покрыты, без дыр и без make-фильтра.""" segs = parse_listing_segments("auto") - # Точное число сегментов. - expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES)) - self.assertEqual(len(segs), expected) - - # Все бренды из списка присутствуют. - makes_in_segments = {s["make"] for s in segs} - for make in IAAI_DEFAULT_MAKES: - self.assertIn(make, makes_in_segments) - - # Крупные бренды разбиты на 3 сегмента с годами. - toyota = [s for s in segs if s["make"] == "TOYOTA"] - self.assertEqual(len(toyota), 3) - for s in toyota: - self.assertIsNotNone(s["year_min"]) - - # Мелкие бренды без годов. - lexus = [s for s in segs if s["make"] == "LEXUS"] - self.assertEqual(len(lexus), 1) - self.assertIsNone(lexus[0]["year_min"]) + self.assertEqual(len(segs), len(_AUTO_YEAR_SPLITS)) + self.assertEqual( + [(s["year_min"], s["year_max"]) for s in segs], + list(_AUTO_YEAR_SPLITS), + ) + self.assertTrue(all(s["make"] is None for s in segs)) # Годовые диапазоны покрывают 1950-2027. years = set() - for yr_min, yr_max in _YEAR_SPLITS: + for yr_min, yr_max in _AUTO_YEAR_SPLITS: years.update(range(yr_min, yr_max + 1)) for year in range(1950, 2027): self.assertIn(year, years) diff --git a/tests/test_worker_tasks.py b/tests/test_worker_tasks.py index 32ac019..fa4d4dd 100644 --- a/tests/test_worker_tasks.py +++ b/tests/test_worker_tasks.py @@ -5,8 +5,8 @@ from dataclasses import replace import unittest from unittest.mock import MagicMock, patch -from iaai_scraper.worker import tasks -from iaai_scraper.core.config import settings as base_settings +from dubizzle_scraper.worker import tasks +from dubizzle_scraper.core.config import settings as base_settings class TestWorkerTaskLockHelpers(unittest.TestCase): @@ -187,13 +187,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): ] with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace( + base_settings.discovery, + always_full_scan=False, + mode="listing", + ), + celery=replace( + base_settings.celery, + parallel_segments=False, + ), + ), + ), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.side_effect = lambda key: ( @@ -211,7 +227,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__exit__.return_value = None - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-resume-seg") try: result = tasks.sync_listing_task.run() @@ -228,6 +244,22 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): segments = [{"make": "ACURA"}, {"make": "AUDI"}] with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace( + base_settings.discovery, + always_full_scan=False, + mode="listing", + ), + celery=replace( + base_settings.celery, + parallel_segments=False, + ), + ), + ), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=True), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ @@ -235,7 +267,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() # Оставшийся чекпоинт не должен использоваться. @@ -254,10 +286,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__exit__.return_value = None - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-792") try: - result = tasks.sync_listing_task.run() + result = tasks.sync_listing_task.run(limit=1) finally: tasks.sync_listing_task.pop_request() @@ -271,13 +303,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): segments = [{"make": "ACURA"}, {"make": "AUDI"}] with patch.object(tasks, "_get_persistence") as get_persistence, \ patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace( + base_settings.discovery, + always_full_scan=False, + mode="listing", + ), + celery=replace( + base_settings.celery, + parallel_segments=False, + ), + ), + ), \ patch.object(tasks, "_acquire_lock", return_value=True), \ patch.object(tasks, "_is_full_scan_done", return_value=False), \ patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.side_effect = lambda key: ( @@ -295,7 +343,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock scraper_ctx.__exit__.return_value = None - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-beyond") try: result = tasks.sync_listing_task.run() @@ -507,7 +555,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \ patch.object(tasks.sync_listing_task, "update_state"), \ patch.object(tasks, "_run_browser_job") as run_job, \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]): + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.return_value = None @@ -554,7 +602,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): patch.object(tasks, "_release_lock_if_owner") as release_lock, \ patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \ patch.object(tasks.sync_listing_task, "update_state"), \ - patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments): + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): get_persistence.return_value = MagicMock() redis_client = MagicMock() redis_client.get.side_effect = lambda key: ( @@ -579,7 +627,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): scraper_ctx.__enter__.return_value.sync_listing = MagicMock() scraper_ctx.__exit__.return_value = None - with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx): + with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx): tasks.sync_listing_task.push_request(id="task-always-full-resume") try: result = tasks.sync_listing_task.run() @@ -591,6 +639,85 @@ class TestWorkerTaskLockHelpers(unittest.TestCase): self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1) release_lock.assert_called_once() + def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + celery=replace(base_settings.celery, parallel_segments=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks, "_reset_segments_progress") as reset_progress, \ + patch.object(tasks, "_set_segmented_scan_active") as set_active, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + redis_client = MagicMock() + redis_client.smembers.return_value = {"0", "1"} + redis_client.get.return_value = None + get_redis.return_value = redis_client + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + tasks.sync_listing_task.push_request(id="task-always-hourly") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "success") + self.assertEqual(result["segments_dispatched"], 2) + reset_progress.assert_called_once_with(redis_client, len(segments)) + set_active.assert_called_once_with(redis_client, total=len(segments)) + self.assertEqual(task_app.send_task.call_count, 2) + release_lock.assert_called_once() + + def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None: + segments = [{"make": "TOYOTA"}, {"make": "FORD"}] + with patch.object(tasks, "_get_persistence") as get_persistence, \ + patch.object(tasks, "_get_redis") as get_redis, \ + patch.object( + tasks, + "Settings", + return_value=replace( + base_settings, + discovery=replace(base_settings.discovery, always_full_scan=True), + celery=replace(base_settings.celery, parallel_segments=True), + ), + ), \ + patch.object(tasks, "_acquire_lock", return_value=True), \ + patch.object(tasks, "_is_full_scan_done", return_value=True), \ + patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \ + patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \ + patch.object(tasks, "_release_lock_if_owner") as release_lock, \ + patch.object(tasks.sync_listing_task, "update_state"): + get_persistence.return_value = MagicMock() + get_redis.return_value = MagicMock() + start_heartbeat.return_value = (MagicMock(), MagicMock()) + + with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \ + patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments): + tasks.sync_listing_task.push_request(id="task-always-skip-duplicate") + try: + result = tasks.sync_listing_task.run() + finally: + tasks.sync_listing_task.pop_request() + + self.assertEqual(result["status"], "skipped") + self.assertEqual(result["reason"], "segmented_scan_in_progress") + task_app.send_task.assert_not_called() + release_lock.assert_called_once() + if __name__ == "__main__": unittest.main() \ No newline at end of file diff --git a/uv.lock b/uv.lock index e2820a9..4a4b8a1 100644 --- a/uv.lock +++ b/uv.lock @@ -342,7 +342,7 @@ wheels = [ ] [[package]] -name = "iaai-scraper" +name = "dubizzle-scraper" version = "0.1.0" source = { editable = "." } dependencies = [