qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update scraper package

Browse Source
This commit is contained in:
qananasikq
2026-04-24 20:46:58 +03:00
parent 6aba4f0dbd
commit d484088e18
59 changed files with 2548 additions and 1272 deletions
Download Patch File Download Diff File Expand all files Collapse all files

89
.env.example
View File

@@ -1,51 +1,51 @@
IAAI_HEADLESS=true
IAAI_LOG_LEVEL=INFO
DUBIZZLE_HEADLESS=true
DUBIZZLE_LOG_LEVEL=INFO
IAAI_CAPTURE_SAME_ORIGIN_ONLY=true
IAAI_MAX_CAPTURED_REQUESTS=40
IAAI_MAX_CAPTURED_JSON_RESPONSES=20
DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY=true
DUBIZZLE_MAX_CAPTURED_REQUESTS=40
DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES=20
IAAI_CARS_LISTING_URL=https://www.iaai.com/Vehiclelisting/Cars
IAAI_LISTING_SEGMENTS=auto
IAAI_MAX_PAGES_PER_RUN=999999
IAAI_MAX_VEHICLES_PER_RUN=999999
IAAI_PAGE_LINK_LIMIT=999999
IAAI_INCLUDE_PAGINATION=true
IAAI_COLLECT_CURRENT_PAGE_ONLY=false
DUBIZZLE_CARS_LISTING_URL=https://uae.dubizzle.com/motors/used-cars/
DUBIZZLE_LISTING_SEGMENTS=auto
DUBIZZLE_MAX_PAGES_PER_RUN=999999
DUBIZZLE_MAX_VEHICLES_PER_RUN=999999
DUBIZZLE_PAGE_LINK_LIMIT=999999
DUBIZZLE_INCLUDE_PAGINATION=true
DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY=false
IAAI_HUMAN_PACE_ENABLED=true
IAAI_PARALLEL_TABS=10
DUBIZZLE_HUMAN_PACE_ENABLED=true
DUBIZZLE_PARALLEL_TABS=10
CELERY_BATCH_SIZE=100
IAAI_AFTER_LISTING_OPEN_MIN_S=0.2
IAAI_AFTER_LISTING_OPEN_MAX_S=0.5
IAAI_AFTER_FILTER_ACTION_MIN_S=0.2
IAAI_AFTER_FILTER_ACTION_MAX_S=0.5
IAAI_BEFORE_VEHICLE_OPEN_MIN_S=0.02
IAAI_BEFORE_VEHICLE_OPEN_MAX_S=0.08
IAAI_AFTER_VEHICLE_OPEN_MIN_S=0.02
IAAI_AFTER_VEHICLE_OPEN_MAX_S=0.08
IAAI_BETWEEN_VEHICLES_MIN_S=0.02
IAAI_BETWEEN_VEHICLES_MAX_S=0.08
IAAI_AFTER_PAGE_CHANGE_MIN_S=0.2
IAAI_AFTER_PAGE_CHANGE_MAX_S=0.5
DUBIZZLE_AFTER_LISTING_OPEN_MIN_S=0.2
DUBIZZLE_AFTER_LISTING_OPEN_MAX_S=0.5
DUBIZZLE_AFTER_FILTER_ACTION_MIN_S=0.2
DUBIZZLE_AFTER_FILTER_ACTION_MAX_S=0.5
DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S=0.02
DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S=0.08
DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S=0.02
DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S=0.08
DUBIZZLE_BETWEEN_VEHICLES_MIN_S=0.02
DUBIZZLE_BETWEEN_VEHICLES_MAX_S=0.08
DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S=0.2
DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S=0.5
IAAI_SYNC_ONLY_NEW=false
IAAI_TOKENS_FILE=/data/tokens.json
IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
DUBIZZLE_SYNC_ONLY_NEW=false
DUBIZZLE_TOKENS_FILE=/data/tokens.json
DUBIZZLE_RUNTIME_CONFIG_FILE=/app/runtime_config.json
IAAI_MAX_RETRIES=5
IAAI_RETRY_DELAY_SECONDS=4
IAAI_RETRY_BACKOFF_MULTIPLIER=2.0
IAAI_RETRY_JITTER_SECONDS=0.5
IAAI_TIMEOUT_MS=90000
IAAI_FALLBACK_NAV_TIMEOUT_MS=30000
DUBIZZLE_MAX_RETRIES=5
DUBIZZLE_RETRY_DELAY_SECONDS=4
DUBIZZLE_RETRY_BACKOFF_MULTIPLIER=2.0
DUBIZZLE_RETRY_JITTER_SECONDS=0.5
DUBIZZLE_TIMEOUT_MS=90000
DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS=30000
IAAI_DATABASE_URL=postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
IAAI_DATABASE_ECHO=false
IAAI_DATABASE_POOL_SIZE=5
IAAI_DATABASE_MAX_OVERFLOW=5
DUBIZZLE_DATABASE_URL=postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
DUBIZZLE_DATABASE_ECHO=false
DUBIZZLE_DATABASE_POOL_SIZE=5
DUBIZZLE_DATABASE_MAX_OVERFLOW=5
IAAI_REDIS_URL=redis://redis:6379/0
DUBIZZLE_REDIS_URL=redis://redis:6379/0
CELERY_BROKER_URL=redis://redis:6379/0
CELERY_RESULT_BACKEND=redis://redis:6379/0
@@ -55,6 +55,13 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000
CELERY_WORKER_CONCURRENCY=1
CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
CELERY_BEAT_SYNC_LIMIT=0
IAAI_ALWAYS_FULL_SCAN=true
DUBIZZLE_ALWAYS_FULL_SCAN=true
CELERY_PARALLEL_SEGMENTS=true
DUBIZZLE_DISCOVERY_MODE=algolia
DUBIZZLE_ALGOLIA_APPLICATION_ID=WD0PTZ13ZS
DUBIZZLE_ALGOLIA_API_KEY=cef139620248f1bc328a00fddc7107a6
DUBIZZLE_ALGOLIA_INDEX_NAME=motors.com
DUBIZZLE_ALGOLIA_BASE_URL=https://WD0PTZ13ZS-dsn.algolia.net
DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY=false
SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT=6
SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS=21600

42
Dockerfile
View File

@@ -1,8 +1,43 @@
FROM mcr.microsoft.com/playwright/python:v1.58.0-noble
FROM python:3.12-slim-bookworm
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1
ENV PLAYWRIGHT_BROWSERS_PATH=/ms-playwright
RUN apt-get update \
&& apt-get install -y --no-install-recommends \
curl \
ca-certificates \
bash \
build-essential \
libasound2 \
libatk-bridge2.0-0 \
libatk1.0-0 \
libc6 \
libcairo2 \
libcups2 \
libdbus-1-3 \
libdrm2 \
libgbm1 \
libglib2.0-0 \
libgtk-3-0 \
libnspr4 \
libnss3 \
libpango-1.0-0 \
libx11-6 \
libx11-xcb1 \
libxcb1 \
libxcomposite1 \
libxdamage1 \
libxext6 \
libxfixes3 \
libxkbcommon0 \
libxrandr2 \
wget \
xdg-utils \
&& rm -rf /var/lib/apt/lists/*
RUN groupadd --system app && useradd --system --gid app --create-home app
WORKDIR /app
@@ -19,7 +54,8 @@ RUN python -m playwright install chromium firefox
COPY . .
RUN pip install --no-cache-dir -e .
RUN python -m compileall -q iaai_scraper
RUN python -m compileall -q dubizzle_scraper
RUN sed -i 's/\r$//' /app/entrypoint.sh
RUN chmod +x entrypoint.sh
RUN chown -R app:app /app
@@ -29,4 +65,4 @@ USER app
# По умолчанию запускаем API.
ENTRYPOINT ["./entrypoint.sh"]
CMD ["uvicorn", "iaai_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]
CMD ["uvicorn", "dubizzle_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

2
alembic.ini
View File

@@ -3,7 +3,7 @@
[alembic]
script_location = alembic
prepend_sys_path = .
sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
sqlalchemy.url = postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper
[loggers]
keys = root,sqlalchemy,alembic

4
alembic/env.py
View File

@@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool
# Добавляем корень проекта в sys.path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from iaai_scraper.core.config import Settings
from iaai_scraper.storage.models import Base
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.storage.models import Base
config = context.config

2
alembic/versions/001_initial.py
View File

@@ -11,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Таблица cars — аукционные машины с IAAI
# Таблица cars — аукционные машины с DUBIZZLE
op.create_table(
"cars",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),

8
alembic/versions/003_add_composite_indexes.py
View File

@@ -10,12 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Partial index для активных машин IAAI (is_sold = FALSE)
# Partial index для активных машин DUBIZZLE (is_sold = FALSE)
# Ускоряет поиск при mark_sold операциях
op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
"CREATE INDEX IF NOT EXISTS ix_cars_active_dubizzle "
"ON cars (origin_url) "
"WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
"WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%'"
)
# Composite index для проверки дубликатов изображений
@@ -35,4 +35,4 @@ def upgrade() -> None:
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
op.drop_index("ix_images_car_id_fullres", table_name="images")
op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")
op.execute("DROP INDEX IF EXISTS ix_cars_active_dubizzle")

101
docker-compose.yml
View File

@@ -1,35 +1,40 @@
x-app-env: &app-env
# Всегда используем Postgres.
IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
DUBIZZLE_DATABASE_URL: postgresql+psycopg2://dubizzle:dubizzle@postgres:5432/dubizzle_scraper
DUBIZZLE_REDIS_URL: ${DUBIZZLE_REDIS_URL:-redis://redis:6379/0}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS: ${DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
# 0 = без лимита.
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
CELERY_PARALLEL_SEGMENTS: ${CELERY_PARALLEL_SEGMENTS:-true}
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0.3}
IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-auto}
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
IAAI_BROWSER_ENGINE: chromium
DUBIZZLE_INTER_BATCH_DELAY_SECONDS: ${DUBIZZLE_INTER_BATCH_DELAY_SECONDS:-0.3}
DUBIZZLE_FAIL_RATE_THRESHOLD: ${DUBIZZLE_FAIL_RATE_THRESHOLD:-0.9}
DUBIZZLE_PARALLEL_TABS: ${DUBIZZLE_PARALLEL_TABS:-8}
DUBIZZLE_BLOCK_RESOURCES: ${DUBIZZLE_BLOCK_RESOURCES:-true}
DUBIZZLE_DISCOVERY_MODE: ${DUBIZZLE_DISCOVERY_MODE:-algolia}
DUBIZZLE_LISTING_SEGMENTS: ${DUBIZZLE_LISTING_SEGMENTS:-auto}
DUBIZZLE_CARS_LISTING_URL: ${DUBIZZLE_CARS_LISTING_URL:-https://dubai.dubizzle.com/motors/used-cars/}
DUBIZZLE_MAX_PAGES_PER_RUN: ${DUBIZZLE_MAX_PAGES_PER_RUN:-9999}
DUBIZZLE_MAX_VEHICLES_PER_RUN: ${DUBIZZLE_MAX_VEHICLES_PER_RUN:-50000}
DUBIZZLE_ALWAYS_FULL_SCAN: ${DUBIZZLE_ALWAYS_FULL_SCAN:-true}
# Работаем через Algolia; fallback в sitemap по умолчанию отключён.
DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY: ${DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY:-false}
DUBIZZLE_HUMAN_PACE_ENABLED: ${DUBIZZLE_HUMAN_PACE_ENABLED:-true}
DUBIZZLE_TOKENS_FILE: ${DUBIZZLE_TOKENS_FILE:-/data/tokens.json}
DUBIZZLE_RUNTIME_CONFIG_FILE: ${DUBIZZLE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
DUBIZZLE_BROWSER_ENGINE: chromium
# Self-heal для worker.
IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
DUBIZZLE_SELF_HEAL_ENABLED: ${DUBIZZLE_SELF_HEAL_ENABLED:-true}
DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
DUBIZZLE_SELF_HEAL_STALL_SECONDS: ${DUBIZZLE_SELF_HEAL_STALL_SECONDS:-900}
DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
TZ: ${TZ:-UTC}
x-env-file: &env-file
@@ -42,29 +47,32 @@ x-app-service: &app-service
environment: *app-env
volumes:
- ./runtime_config.json:/app/runtime_config.json:ro
- ./artifacts:/app/artifacts
x-worker-service: &worker-service
<<: *app-service
volumes:
- ./runtime_config.json:/app/runtime_config.json:ro
- ./artifacts:/app/artifacts
- tokens_data:/data
services:
# PostgreSQL.
postgres:
image: postgres:16-alpine
container_name: iaai-postgres
container_name: dubizzle-postgres
restart: unless-stopped
environment:
POSTGRES_USER: iaai
POSTGRES_PASSWORD: iaai
POSTGRES_DB: iaai_scraper
POSTGRES_USER: dubizzle
POSTGRES_PASSWORD: dubizzle
POSTGRES_DB: dubizzle_scraper
ports:
- "127.0.0.1:5432:5432"
# Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
- "127.0.0.1:${DUBIZZLE_POSTGRES_HOST_PORT:-15432}:5432"
volumes:
- pgdata:/var/lib/postgresql/data
healthcheck:
test: ["CMD-SHELL", "pg_isready -U iaai -d iaai_scraper"]
test: ["CMD-SHELL", "pg_isready -U dubizzle -d dubizzle_scraper"]
interval: 5s
timeout: 3s
retries: 5
@@ -77,10 +85,11 @@ services:
# Redis.
redis:
image: redis:7-alpine
container_name: iaai-redis
container_name: dubizzle-redis
restart: unless-stopped
ports:
- "127.0.0.1:6379:6379"
# Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
- "127.0.0.1:${DUBIZZLE_REDIS_HOST_PORT:-16379}:6379"
healthcheck:
test: ["CMD", "redis-cli", "ping"]
interval: 5s
@@ -101,7 +110,7 @@ services:
# Миграции.
migrate:
<<: *app-service
container_name: iaai-migrate
container_name: dubizzle-migrate
restart: "no"
depends_on:
postgres:
@@ -111,17 +120,18 @@ services:
# API.
api:
<<: *app-service
container_name: iaai-api
container_name: dubizzle-api
restart: unless-stopped
ports:
- "127.0.0.1:8000:8000"
# Хост-порт вынесен в env, чтобы избежать конфликтов с другими проектами.
- "127.0.0.1:${DUBIZZLE_API_HOST_PORT:-18000}:8000"
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
command: >
uvicorn iaai_scraper.api.app:app
uvicorn dubizzle_scraper.api.app:app
--host 0.0.0.0 --port 8000 --workers 1
healthcheck:
test: ["CMD", "python", "-c", "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8000/health', timeout=5)"]
@@ -148,13 +158,13 @@ services:
condition: service_healthy
stop_grace_period: 60s
command: >
celery -A iaai_scraper.worker.celery_app worker
celery -A dubizzle_scraper.worker.celery_app worker
--loglevel=info --concurrency=1 --pool=solo
--pidfile=/tmp/celery-worker.pid
-Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
healthcheck:
# Проверка worker.
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'iaai_scraper.worker.self_heal' >/dev/null"]
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'dubizzle_scraper.worker.self_heal' >/dev/null"]
interval: 60s
timeout: 10s
retries: 3
@@ -168,7 +178,7 @@ services:
# Beat.
beat:
<<: *worker-service
container_name: iaai-beat
container_name: dubizzle-beat
restart: unless-stopped
init: true
depends_on:
@@ -177,7 +187,7 @@ services:
redis:
condition: service_healthy
command: >
celery -A iaai_scraper.worker.celery_app beat
celery -A dubizzle_scraper.worker.celery_app beat
--loglevel=info
--pidfile=/tmp/celery-beat.pid
--schedule=/tmp/celerybeat-schedule
@@ -193,6 +203,25 @@ services:
max-size: "10m"
max-file: "5"
# Разовый прогон sync-listing через Docker CLI.
sync:
<<: *worker-service
container_name: dubizzle-sync
restart: "no"
profiles: ["manual"]
depends_on:
migrate:
condition: service_completed_successfully
redis:
condition: service_healthy
command: >
bash -lc "dubizzle sync-listing --limit ${DUBIZZLE_SYNC_LISTING_LIMIT:-100} --output /app/artifacts/json/docker_sync_listing.json"
logging:
driver: json-file
options:
max-size: "10m"
max-file: "5"
volumes:
pgdata:
redisdata:

0
iaai_scraper/__init__.py → dubizzle_scraper/__init__.py
View File

0
iaai_scraper/api/__init__.py → dubizzle_scraper/api/__init__.py
View File

4
iaai_scraper/api/app.py → dubizzle_scraper/api/app.py
View File

@@ -20,8 +20,8 @@ def create_app(settings: Settings | None = None) -> FastAPI:
_settings = settings or Settings()
app = FastAPI(
title="IAAI Scraper API",
description="REST API для управления задачами скрапинга IAAI и просмотра данных",
title="Dubizzle Scraper API",
description="REST API для управления задачами скрапинга Dubizzle и просмотра данных",
version="1.0.0",
lifespan=lifespan,
)

0
iaai_scraper/api/deps.py → dubizzle_scraper/api/deps.py
View File

0
iaai_scraper/api/routes/__init__.py → dubizzle_scraper/api/routes/__init__.py
View File

0
iaai_scraper/api/routes/cars.py → dubizzle_scraper/api/routes/cars.py
View File

4
iaai_scraper/api/routes/health.py → dubizzle_scraper/api/routes/health.py
View File

@@ -9,7 +9,7 @@ from ..deps import get_persistence
from ...storage.db import PersistenceService
router = APIRouter()
logger = logging.getLogger("iaai_scraper.api.health")
logger = logging.getLogger("dubizzle_scraper.api.health")
@router.get("/health")
@@ -25,6 +25,6 @@ def health_check(persistence: PersistenceService = Depends(get_persistence)):
return {
"status": "ok" if db_ok else "degraded",
"service": "iaai-scraper-api",
"service": "dubizzle-scraper-api",
"database": "connected" if db_ok else "unavailable",
}

4
iaai_scraper/api/routes/tasks.py → dubizzle_scraper/api/routes/tasks.py
View File

@@ -15,13 +15,13 @@ router = APIRouter()
class SyncVehicleRequest(BaseModel):
vehicle_url: str
lane: str = "iaai"
lane: str = "dubizzle"
class SyncListingRequest(BaseModel):
make: str | None = None
model: str | None = None
lane: str = "iaai_cars"
lane: str = "dubizzle_cars"
limit: int | None = None
only_new: bool | None = None

0
iaai_scraper/browser/__init__.py → dubizzle_scraper/browser/__init__.py
View File

6
iaai_scraper/browser/factory.py → dubizzle_scraper/browser/factory.py
View File

@@ -11,7 +11,7 @@ except ImportError:
from ..core.config import Settings
logger = logging.getLogger("iaai_scraper.browser")
logger = logging.getLogger("dubizzle_scraper.browser")
def _build_init_script() -> str:
@@ -72,11 +72,11 @@ class BrowserFactory:
# Выбор движка.
engine = self.settings.browser_engine.strip().lower()
if engine == "auto":
# Для IAAI стабильнее Chromium.
# Для DUBIZZLE стабильнее Chromium.
return "chromium"
if engine in ("firefox", "chromium"):
return engine
logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
logger.warning("Unknown DUBIZZLE_BROWSER_ENGINE=%r, falling back to auto", engine)
return "chromium"
def create_browser(self, playwright: Playwright) -> Browser:

31
iaai_scraper/browser/listing.py → dubizzle_scraper/browser/listing.py
View File

@@ -11,9 +11,15 @@ from .pace import HumanPacer
from ..core.config import Settings
from ..core.utils import first_non_empty
logger = logging.getLogger("iaai_scraper.listing")
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
logger = logging.getLogger("dubizzle_scraper.listing")
VEHICLE_HREF_RE = re.compile(
r'(?:/VehicleDetail/(?P<veh_id>\d+)(?:~[A-Z]{2})?)|(?:/motors/used-cars/[^"\s]+?(?:/ad-(?P<ad_id>\d+)/?|---(?P<slug_id>[a-f0-9]{32})/?))|(?:/s/(?P<short_id>[A-Za-z0-9]+))',
re.IGNORECASE,
)
VEHICLE_LINK_SELECTOR = (
"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail'], "
"a[href*='/motors/used-cars/'], a[href*='/s/']"
)
COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
"button:has-text('Accept All')",
"button:has-text('Accept all')",
@@ -108,7 +114,7 @@ class ListingCollector:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
const a = document.querySelector(\"{VEHICLE_LINK_SELECTOR}\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=12000,
@@ -331,7 +337,10 @@ class ListingCollector:
match = VEHICLE_HREF_RE.search(href)
if not match:
continue
lot_number = match.group(1)
lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or None
if lot_number is None:
short = match.group("short_id")
lot_number = short if short else None
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
@@ -341,7 +350,7 @@ class ListingCollector:
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
# Fallback: на DUBIZZLE ссылки иногда не рендерятся как <a>,
# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
if not links:
try:
@@ -381,7 +390,7 @@ class ListingCollector:
seen: set[str] = set()
for match in VEHICLE_HREF_RE.finditer(normalized):
lot_number = match.group(1)
lot_number = match.group("veh_id") or match.group("ad_id") or match.group("slug_id") or match.group("short_id") or ""
absolute = urljoin(self.settings.home_url, match.group(0))
if absolute in seen:
continue
@@ -424,7 +433,7 @@ class ListingCollector:
self.pacer.after_page_change()
return True
# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
# Fallback для DUBIZZLE: пагинация часто рендерится как набор номеров страниц
# + стрелка с иконкой, без явного текста Next.
try:
clicked = bool(page.evaluate(
@@ -573,7 +582,7 @@ class ListingCollector:
):
page_known = sum(
1 for item in page_result.vehicle_links
if item.lot_number and f"iaai:{item.lot_number}" in known_origin_ids
if item.lot_number and f"dubizzle:{item.lot_number}" in known_origin_ids
)
page_new = len(page_result.vehicle_links) - page_known
ratio = page_known / len(page_result.vehicle_links)
@@ -651,6 +660,10 @@ class ListingCollector:
count = locator.count()
if count == 0:
continue
# Тестовые/fake локаторы могут не поддерживать nth/get_attribute.
# В таком случае считаем наличие селектора достаточным признаком next.
if not hasattr(locator, "nth"):
return True
# Проверяем, что хотя бы один элемент не disabled.
# Disabled "Next" на последней странице не означает наличия следующей.
for i in range(min(count, 3)):

4
iaai_scraper/browser/network.py → dubizzle_scraper/browser/network.py
View File

@@ -8,7 +8,7 @@ from playwright.sync_api import Page, Request, Response
from ..core.config import Settings
logger = logging.getLogger("iaai_scraper.network")
logger = logging.getLogger("dubizzle_scraper.network")
@dataclass
@@ -44,7 +44,7 @@ class NetworkCapture:
if not self.settings.capture.capture_same_origin_only or not self._origin:
return True
netloc = urlparse(url).netloc.lower()
return netloc == self._origin or netloc.endswith(".iaai.com")
return netloc == self._origin or netloc.endswith(".dubizzle.com")
def _on_request(self, request: Request) -> None:
# Берём только xhr/fetch

0
iaai_scraper/browser/pace.py → dubizzle_scraper/browser/pace.py
View File

35
iaai_scraper/cli.py → dubizzle_scraper/cli.py
View File

@@ -1,13 +1,13 @@
import argparse
from pathlib import Path
from .core.config import Settings
from .core.config import Settings, parse_listing_segments
from .core.utils import save_to_json
from .scraper import IAAIScraper
from .scraper import DUBIZZLEScraper
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="IAAI scraper CLI")
parser = argparse.ArgumentParser(description="Dubizzle scraper CLI")
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
subparsers = parser.add_subparsers(dest="command", required=True)
@@ -19,24 +19,24 @@ def build_parser() -> argparse.ArgumentParser:
listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
listing_parser.add_argument("--make", default=None)
listing_parser.add_argument("--model", default=None)
listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_listing_links.json"))
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
scrape_parser.add_argument("vehicle_url")
scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
scrape_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_vehicle_detail.json"))
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
sync_vehicle_parser.add_argument("vehicle_url")
sync_vehicle_parser.add_argument("--lane", default="iaai")
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
sync_vehicle_parser.add_argument("--lane", default="dubizzle")
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_vehicle.json"))
sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
sync_listing_parser.add_argument("--make", default=None)
sync_listing_parser.add_argument("--model", default=None)
sync_listing_parser.add_argument("--lane", default="iaai_cars")
sync_listing_parser.add_argument("--lane", default="dubizzle_cars")
sync_listing_parser.add_argument("--limit", type=int, default=None)
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "dubizzle_sync_listing.json"))
return parser
@@ -53,7 +53,7 @@ def main() -> None:
if args.debug:
runtime_settings.log_level = "DEBUG"
with IAAIScraper(runtime_settings) as scraper:
with DUBIZZLEScraper(runtime_settings) as scraper:
if args.command == "init-db":
data = scraper.init_db()
print(f"DB initialized: {data}")
@@ -66,6 +66,21 @@ def main() -> None:
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
else:
only_new = None if args.only_new is None else args.only_new == "true"
segments = parse_listing_segments(scraper.settings.listing.listing_segments_json)
use_segmented = (
bool(segments)
and args.make is None
and args.model is None
and args.limit is None
and scraper.settings.discovery.mode in {"listing", "algolia"}
)
if use_segmented:
data = scraper.sync_listing_segmented(
segments=segments,
lane=args.lane,
only_new=only_new,
)
else:
data = scraper.sync_listing(
make=args.make,
model=args.model,

0
iaai_scraper/core/__init__.py → dubizzle_scraper/core/__init__.py
View File

315
dubizzle_scraper/core/config.py Normal file
View File

@@ -0,0 +1,315 @@
import json
import os
from dataclasses import dataclass, field
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
TRUE_VALUES = {"1", "true", "yes", "on"}
def _resolve_env_value(name: str) -> str | None:
"""Возвращает значение env с поддержкой legacy-префиксов DUBIZZLE_/IAAI_."""
value = os.getenv(name)
if value is not None:
return value
if name.startswith("DUBIZZLE_"):
return os.getenv("IAAI_" + name[len("DUBIZZLE_"):])
return None
# Хелперы для чтения env-переменных с приведением типов
def _env_str(name: str, default: str) -> str:
value = _resolve_env_value(name)
return value if value is not None else default
def _env_optional_str(name: str) -> str | None:
value = _resolve_env_value(name)
if value is None:
return None
value = value.strip()
return value or None
def _env_path_str(name: str) -> str | None:
value = _env_optional_str(name)
if value is None:
return None
return str(Path(value).expanduser())
def _env_bool(name: str, default: bool) -> bool:
fallback = "true" if default else "false"
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
def _env_int(name: str, default: int) -> int:
return int(_env_str(name, str(default)).strip())
def _env_float(name: str, default: float) -> float:
return float(_env_str(name, str(default)).strip())
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
@dataclass(slots=True)
class FingerprintConfig:
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/135.0.0.0 Safari/537.36"
)
viewport_presets: tuple[dict[str, int], ...] = (
{"width": 1920, "height": 1080},
{"width": 1600, "height": 900},
{"width": 1536, "height": 864},
{"width": 1440, "height": 900},
{"width": 1366, "height": 768},
)
timezone_candidates: tuple[str, ...] = (
"America/New_York",
"America/Chicago",
"America/Los_Angeles",
)
locale: str = "en-US"
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
@dataclass(slots=True)
class CaptureConfig:
capture_same_origin_only: bool = _env_bool("DUBIZZLE_CAPTURE_SAME_ORIGIN_ONLY", True)
max_requests: int = _env_int("DUBIZZLE_MAX_CAPTURED_REQUESTS", 40)
max_json_responses: int = _env_int("DUBIZZLE_MAX_CAPTURED_JSON_RESPONSES", 30)
# Конфиг пауз между действиями (имитация человека)
@dataclass(slots=True)
class HumanPaceConfig:
enabled: bool = _env_bool("DUBIZZLE_HUMAN_PACE_ENABLED", True)
after_listing_open_min_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MIN_S", 0.5)
after_listing_open_max_s: float = _env_float("DUBIZZLE_AFTER_LISTING_OPEN_MAX_S", 1.2)
after_filter_action_min_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MIN_S", 0.5)
after_filter_action_max_s: float = _env_float("DUBIZZLE_AFTER_FILTER_ACTION_MAX_S", 1.2)
before_vehicle_open_min_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
before_vehicle_open_max_s: float = _env_float("DUBIZZLE_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
after_vehicle_open_min_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
after_vehicle_open_max_s: float = _env_float("DUBIZZLE_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
between_vehicles_min_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MIN_S", 0.05)
between_vehicles_max_s: float = _env_float("DUBIZZLE_BETWEEN_VEHICLES_MAX_S", 0.15)
after_page_change_min_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MIN_S", 0.8)
after_page_change_max_s: float = _env_float("DUBIZZLE_AFTER_PAGE_CHANGE_MAX_S", 1.8)
# Конфиг сбора листинга (URL, лимиты страниц и машин)
@dataclass(slots=True)
class ListingConfig:
cars_url: str = _env_str("DUBIZZLE_CARS_LISTING_URL", "https://dubai.dubizzle.com/motors/used-cars/")
max_pages_per_run: int = _env_int("DUBIZZLE_MAX_PAGES_PER_RUN", 9999)
max_vehicles_per_run: int = _env_int("DUBIZZLE_MAX_VEHICLES_PER_RUN", 50000)
page_link_limit: int = _env_int("DUBIZZLE_PAGE_LINK_LIMIT", 500)
include_pagination: bool = _env_bool("DUBIZZLE_INCLUDE_PAGINATION", True)
collect_current_page_only: bool = _env_bool("DUBIZZLE_COLLECT_CURRENT_PAGE_ONLY", False)
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
early_stop_threshold: float = _env_float("DUBIZZLE_EARLY_STOP_THRESHOLD", 0.8)
# Сегментация листинга по брендам для обхода лимита пагинации DUBIZZLE (~22 600 машин).
# JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
# Пустая строка = без сегментации (backward compatible).
listing_segments_json: str = _env_str("DUBIZZLE_LISTING_SEGMENTS", "")
# Пагинационный потолок одного Algolia-запроса: ~100 страниц × 100 = 10 000 результатов.
DUBIZZLE_PAGINATION_CEILING = 10_000
# Авто-сегментация по году. Эти диапазоны подобраны так, чтобы каждый сегмент
# оставался ниже лимита Algolia и собирался быстрее, чем старая make/year схема.
_AUTO_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
(1900, 2012),
(2013, 2015),
(2016, 2017),
(2018, 2019),
(2020, 2021),
(2022, 2023),
(2024, 2025),
(2026, 2027),
)
def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
"""Парсит DUBIZZLE_LISTING_SEGMENTS в список сегментов.
Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
Специальное значение ``"auto"`` генерирует быстрые year-only сегменты,
которые гарантированно проходят через Algolia без упора в лимит ~10k.
"""
raw = raw.strip()
if not raw:
return []
if raw.lower() == "auto":
return [
{"make": None, "year_min": yr_min, "year_max": yr_max}
for yr_min, yr_max in _AUTO_YEAR_SPLITS
]
try:
data = json.loads(raw)
except (json.JSONDecodeError, ValueError):
return []
if not isinstance(data, list):
return []
segments = []
for item in data:
if isinstance(item, str):
segments.append({"make": item.upper(), "year_min": None, "year_max": None})
elif isinstance(item, dict):
segments.append({
"make": str(item.get("make") or "").upper() or None,
"year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
"year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
})
return segments
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
@dataclass(slots=True)
class DatabaseConfig:
url: str = _env_str("DUBIZZLE_DATABASE_URL", "postgresql+psycopg2://dubizzle:dubizzle@localhost:5432/dubizzle_scraper")
echo: bool = _env_bool("DUBIZZLE_DATABASE_ECHO", False)
pool_size: int = _env_int("DUBIZZLE_DATABASE_POOL_SIZE", 5)
max_overflow: int = _env_int("DUBIZZLE_DATABASE_MAX_OVERFLOW", 10)
pool_recycle_seconds: int = _env_int("DUBIZZLE_DATABASE_POOL_RECYCLE_SECONDS", 1800)
auto_create_tables: bool = _env_bool("DUBIZZLE_DATABASE_AUTO_CREATE_TABLES", False)
# --- Конфиг Redis (URL для Celery broker) ---
@dataclass(slots=True)
class RedisConfig:
url: str = _env_str("DUBIZZLE_REDIS_URL", "redis://localhost:6379/0")
socket_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
socket_connect_timeout_seconds: float = _env_float("DUBIZZLE_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
health_check_interval_seconds: int = _env_int("DUBIZZLE_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
@dataclass(slots=True)
class DiscoveryConfig:
mode: str = _env_str("DUBIZZLE_DISCOVERY_MODE", "algolia")
hourly_mode: str = _env_str("DUBIZZLE_HOURLY_MODE", "rolling_refresh")
hourly_refresh_batch_size: int = _env_int("DUBIZZLE_HOURLY_REFRESH_BATCH_SIZE", 500)
always_full_scan: bool = _env_bool("DUBIZZLE_ALWAYS_FULL_SCAN", False)
sitemap_fallback_on_empty: bool = _env_bool("DUBIZZLE_SITEMAP_FALLBACK_ON_EMPTY", False)
@dataclass(slots=True)
class AlgoliaConfig:
application_id: str = _env_str("DUBIZZLE_ALGOLIA_APPLICATION_ID", "WD0PTZ13ZS")
api_key: str = _env_str(
"DUBIZZLE_ALGOLIA_API_KEY",
"cef139620248f1bc328a00fddc7107a6",
)
index_name: str = _env_str("DUBIZZLE_ALGOLIA_INDEX_NAME", "motors.com")
category_slug: str = _env_str("DUBIZZLE_ALGOLIA_CATEGORY_SLUG", "motors/used-cars")
base_url: str = _env_str("DUBIZZLE_ALGOLIA_BASE_URL", "https://WD0PTZ13ZS-dsn.algolia.net")
hits_per_page: int = _env_int("DUBIZZLE_ALGOLIA_HITS_PER_PAGE", 100)
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@dataclass(slots=True)
class CeleryConfig:
broker_url: str = _env_str("CELERY_BROKER_URL", "")
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("DUBIZZLE_PARALLEL_TABS", 8)
parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
block_resources: bool = _env_bool("DUBIZZLE_BLOCK_RESOURCES", True)
# --- Конфиг прокси (server, username, password) ---
@dataclass(slots=True)
class ProxyConfig:
server: str | None = _env_optional_str("DUBIZZLE_PROXY_SERVER")
username: str | None = _env_optional_str("DUBIZZLE_PROXY_USERNAME")
password: str | None = _env_optional_str("DUBIZZLE_PROXY_PASSWORD")
@property
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
if self.username:
result["username"] = self.username
if self.password:
result["password"] = self.password
return result
# Главный объект настроек: собирает все блоки конфигурации
@dataclass(slots=True)
class Settings:
home_url: str = "https://www.dubizzle.com/"
default_timeout_ms: int = _env_int("DUBIZZLE_TIMEOUT_MS", 45000)
network_settle_ms: int = _env_int("DUBIZZLE_NETWORK_SETTLE_MS", 400)
fast_path_timeout_ms: int = _env_int("DUBIZZLE_FAST_PATH_TIMEOUT_MS", 5000)
fast_path_max_attempts: int = _env_int("DUBIZZLE_FAST_PATH_MAX_ATTEMPTS", 1)
fallback_navigation_timeout_ms: int = _env_int("DUBIZZLE_FALLBACK_NAV_TIMEOUT_MS", 15000)
max_retries: int = _env_int("DUBIZZLE_MAX_RETRIES", 3)
retry_delay_seconds: float = _env_float("DUBIZZLE_RETRY_DELAY_SECONDS", 2.5)
retry_backoff_multiplier: float = _env_float("DUBIZZLE_RETRY_BACKOFF_MULTIPLIER", 2.0)
retry_jitter_seconds: float = _env_float("DUBIZZLE_RETRY_JITTER_SECONDS", 0.25)
headless: bool = _env_bool("DUBIZZLE_HEADLESS", True)
browser_engine: str = _env_str("DUBIZZLE_BROWSER_ENGINE", "auto")
log_level: str = _env_str("DUBIZZLE_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("DUBIZZLE_LOG_FILE")
enable_trace_id_logs: bool = _env_bool("DUBIZZLE_ENABLE_TRACE_ID_LOGS", True)
sync_only_new: bool = _env_bool("DUBIZZLE_SYNC_ONLY_NEW", False)
raw_output_json: str | None = _env_optional_str("DUBIZZLE_RAW_OUTPUT_JSON")
tokens_file: str | None = _env_path_str("DUBIZZLE_TOKENS_FILE")
runtime_config_file: str | None = _env_path_str("DUBIZZLE_RUNTIME_CONFIG_FILE")
scheduler_interval_minutes: int = _env_int("DUBIZZLE_SCHEDULER_INTERVAL_MINUTES", 60)
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
capture: CaptureConfig = field(default_factory=CaptureConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
algolia: AlgoliaConfig = field(default_factory=AlgoliaConfig)
@property
def parallel_tabs(self) -> int:
return self.celery.parallel_tabs
@property
def block_resources(self) -> bool:
return self.celery.block_resources
# Глобальный синглтон — используется по умолчанию во всех модулях.
settings = Settings()

0
iaai_scraper/core/exceptions.py → dubizzle_scraper/core/exceptions.py
View File

0
iaai_scraper/core/logs.py → dubizzle_scraper/core/logs.py
View File

2
iaai_scraper/core/retry.py → dubizzle_scraper/core/retry.py
View File

@@ -9,7 +9,7 @@ from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
from .exceptions import AntiBotDetectedError
logger = logging.getLogger("iaai_scraper.retry")
logger = logging.getLogger("dubizzle_scraper.retry")
# Типы исключений, при которых retry имеет смысл.
RETRYABLE_EXCEPTIONS = (

2
iaai_scraper/core/runtime_config.py → dubizzle_scraper/core/runtime_config.py
View File

@@ -5,7 +5,7 @@ from pathlib import Path
from typing import Any
logger = logging.getLogger("iaai_scraper.runtime_config")
logger = logging.getLogger("dubizzle_scraper.runtime_config")
def _normalize_text(value: str) -> str:

0
iaai_scraper/core/utils.py → dubizzle_scraper/core/utils.py
View File

8
iaai_scraper/discovery/__init__.py → dubizzle_scraper/discovery/__init__.py
View File

@@ -5,6 +5,11 @@ from .sitemap import (
discover_vehicle_urls_from_sitemap,
discover_vehicle_urls_from_sitemap_with_stats,
)
from .algolia import (
AlgoliaDiscoveryError,
AlgoliaDiscoveryResult,
discover_vehicle_hits_from_algolia,
)
__all__ = [
"SitemapDiscoveryError",
@@ -12,4 +17,7 @@ __all__ = [
"SitemapDiscoveryStats",
"discover_vehicle_urls_from_sitemap",
"discover_vehicle_urls_from_sitemap_with_stats",
"AlgoliaDiscoveryError",
"AlgoliaDiscoveryResult",
"discover_vehicle_hits_from_algolia",
]

479
dubizzle_scraper/discovery/algolia.py Normal file
View File

@@ -0,0 +1,479 @@
from __future__ import annotations
import json
import logging
import time
from dataclasses import dataclass, field
from typing import Any
from urllib.parse import parse_qs, urlencode, urlparse
from urllib.request import Request, urlopen
logger = logging.getLogger("dubizzle_scraper.discovery.algolia")
ALGOLIA_HARD_PAGE_LIMIT = 100
ALGOLIA_MAX_HITS_PER_QUERY = ALGOLIA_HARD_PAGE_LIMIT * 100
ALGOLIA_ID_RANGE_START = 0
ALGOLIA_ID_RANGE_END = 20_000_000
ALGOLIA_ID_RANGE_MIN_WINDOW = 100
class AlgoliaDiscoveryError(RuntimeError):
pass
@dataclass(slots=True)
class AlgoliaDiscoveryPageStat:
page_number: int
hits_count: int
@dataclass(slots=True)
class AlgoliaDiscoveryResult:
vehicle_urls: list[str] = field(default_factory=list)
hit_records: dict[str, dict[str, Any]] = field(default_factory=dict)
origin_ids_by_url: dict[str, str] = field(default_factory=dict)
pages: list[dict[str, int]] = field(default_factory=list)
early_stopped: bool = False
truncated_by_time_budget: bool = False
total_hits: int = 0
@dataclass(slots=True)
class _AlgoliaShard:
category_slug: str
id_min: int | None = None
id_max: int | None = None
year_min: int | None = None
year_max: int | None = None
def filter_expr(self) -> str:
parts = [f"(category_v2.slug_paths:{self.category_slug})"]
if self.year_min is not None:
parts.append(f"year>={int(self.year_min)}")
if self.year_max is not None:
parts.append(f"year<={int(self.year_max)}")
if self.id_min is not None:
parts.append(f"id>={int(self.id_min)}")
if self.id_max is not None:
parts.append(f"id<={int(self.id_max)}")
return " AND ".join(parts)
def label(self) -> str:
label = self.category_slug
if self.year_min is not None or self.year_max is not None:
label += f"[year:{self.year_min}-{self.year_max}]"
if self.id_min is None and self.id_max is None:
return label
return f"{label}[id:{self.id_min}-{self.id_max}]"
@dataclass(slots=True)
class _AlgoliaHttpClient:
endpoint: str
app_id: str
api_key: str
user_agent: str
index_name: str
hits_per_page: int
def request(self, *, page: int, filters: str) -> dict[str, Any]:
params = urlencode(
{
"query": "",
"page": page,
"hitsPerPage": self.hits_per_page,
"filters": filters,
}
)
payload = {"requests": [{"indexName": self.index_name, "params": params}]}
request = Request(
self.endpoint,
data=json.dumps(payload).encode("utf-8"),
headers={
"content-type": "application/json",
"x-algolia-application-id": self.app_id,
"x-algolia-api-key": self.api_key,
"accept": "application/json",
"user-agent": self.user_agent,
},
method="POST",
)
with urlopen(request, timeout=30) as response:
body = response.read().decode("utf-8", errors="ignore")
parsed = json.loads(body)
results = parsed.get("results") or []
return results[0] if results and isinstance(results[0], dict) else {}
def _build_origin_id_from_hit(hit: dict[str, Any]) -> str | None:
for key in ("id", "objectID", "uuid"):
value = hit.get(key)
if value is None:
continue
text = str(value).strip()
if text:
return f"dubizzle:{text}"
permalink = str(hit.get("permalink") or "").strip()
if permalink:
tail = permalink.rstrip("/").split("/")[-1]
if tail:
return f"dubizzle:{tail}"
return None
def _build_vehicle_url_from_hit(hit: dict[str, Any]) -> str | None:
permalink = str(hit.get("permalink") or "").strip()
if permalink:
if permalink.startswith("http://") or permalink.startswith("https://"):
return permalink
if permalink.startswith("/"):
return f"https://www.dubizzle.com{permalink}"
return f"https://www.dubizzle.com/{permalink.lstrip('/')}"
short = str(hit.get("short_url") or "").strip()
if short:
if short.startswith("http://") or short.startswith("https://"):
return short
return f"https://dubizzle.com/s/{short.strip('/')}"
hit_id = hit.get("id") or hit.get("objectID")
if hit_id is not None:
return f"https://www.dubizzle.com/motors/used-cars/ad-{hit_id}/"
return None
def _extract_make_from_listing_url(listing_url: str | None) -> str | None:
if not listing_url:
return None
try:
parsed = urlparse(listing_url)
params = parse_qs(parsed.query)
candidate = (params.get("Make") or params.get("make") or [None])[0]
if candidate:
return str(candidate).strip()
parts = [p for p in parsed.path.split("/") if p]
if len(parts) >= 3 and parts[0].lower() == "motors" and parts[1].lower() == "used-cars":
slug = parts[2].strip().lower()
if slug and slug != "s":
return slug.replace("-", " ")
except Exception:
return None
return None
def _make_slug(make: str | None) -> str | None:
if not make:
return None
slug = make.strip().lower().replace(" ", "-")
return slug or None
def _hit_matches_filters(
hit: dict[str, Any],
*,
make: str | None,
model: str | None,
year_min: int | None,
year_max: int | None,
) -> bool:
if make:
hit_make = str(hit.get("make") or "").strip().lower()
if hit_make != make.strip().lower():
return False
if model:
hit_model = str(hit.get("model") or "").strip().lower()
if hit_model != model.strip().lower():
return False
hit_year_raw = hit.get("year")
hit_year: int | None = None
if hit_year_raw is not None:
try:
hit_year = int(hit_year_raw)
except Exception:
hit_year = None
if year_min is not None and (hit_year is None or hit_year < year_min):
return False
if year_max is not None and (hit_year is None or hit_year > year_max):
return False
return True
def _probe_total_hits(client: _AlgoliaHttpClient, shard: _AlgoliaShard) -> int:
first = client.request(page=0, filters=shard.filter_expr())
return int(first.get("nbHits") or 0)
def _split_id_range(shard: _AlgoliaShard) -> tuple[_AlgoliaShard, _AlgoliaShard] | None:
lo = shard.id_min if shard.id_min is not None else ALGOLIA_ID_RANGE_START
hi = shard.id_max if shard.id_max is not None else ALGOLIA_ID_RANGE_END
if hi - lo <= ALGOLIA_ID_RANGE_MIN_WINDOW:
return None
mid = (lo + hi) // 2
return (
_AlgoliaShard(
category_slug=shard.category_slug,
id_min=lo,
id_max=mid,
year_min=shard.year_min,
year_max=shard.year_max,
),
_AlgoliaShard(
category_slug=shard.category_slug,
id_min=mid + 1,
id_max=hi,
year_min=shard.year_min,
year_max=shard.year_max,
),
)
def _expand_shards(client: _AlgoliaHttpClient, initial_shard: _AlgoliaShard, max_duration_seconds: float | None, started_at: float) -> tuple[list[_AlgoliaShard], bool, int]:
queue: list[_AlgoliaShard] = [initial_shard]
ready: list[_AlgoliaShard] = []
truncated = False
total_hits = 0
while queue:
if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
truncated = True
break
shard = queue.pop(0)
shard_total = _probe_total_hits(client, shard)
if shard is initial_shard:
total_hits = shard_total
if shard_total == 0:
continue
if shard_total <= ALGOLIA_MAX_HITS_PER_QUERY:
ready.append(shard)
continue
split = _split_id_range(shard)
if split is None:
logger.warning(
"Shard %s still exceeds limit=%d but id-window is too small to split further (hits=%d)",
shard.label(),
ALGOLIA_MAX_HITS_PER_QUERY,
shard_total,
)
ready.append(shard)
continue
logger.warning(
"Splitting Algolia shard %s with hits=%d into %s and %s",
shard.label(),
shard_total,
split[0].label(),
split[1].label(),
)
queue.insert(0, split[1])
queue.insert(0, split[0])
return ready, truncated, total_hits
def _fetch_shard_hits(
*,
client: _AlgoliaHttpClient,
shard: _AlgoliaShard,
make: str | None,
model: str | None,
year_min: int | None,
year_max: int | None,
known_origin_ids: set[str] | None,
threshold: float,
max_duration_seconds: float | None,
started_at: float,
) -> tuple[list[str], dict[str, dict[str, Any]], dict[str, str], list[dict[str, int]], bool, bool]:
urls: list[str] = []
hit_records: dict[str, dict[str, Any]] = {}
origin_ids_by_url: dict[str, str] = {}
pages: list[dict[str, int]] = []
early_stopped = False
truncated_by_time_budget = False
page = 0
nb_pages = None
while True:
if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
truncated_by_time_budget = True
break
try:
first = client.request(page=page, filters=shard.filter_expr())
except Exception as exc:
raise AlgoliaDiscoveryError(
f"Algolia request failed for shard={shard.label()} page={page}: {exc}"
) from exc
hits = first.get("hits") or []
if not isinstance(hits, list):
hits = []
if page == 0:
nb_pages = min(int(first.get("nbPages") or 0), ALGOLIA_HARD_PAGE_LIMIT)
pages.append({"page_number": page + 1, "links_found": len(hits), "shard": shard.label()})
if not hits:
break
page_known = 0
page_new = 0
for raw_hit in hits:
if not isinstance(raw_hit, dict):
continue
if not _hit_matches_filters(
raw_hit,
make=make,
model=model,
year_min=year_min,
year_max=year_max,
):
continue
url = _build_vehicle_url_from_hit(raw_hit)
if not url:
continue
origin_id = _build_origin_id_from_hit(raw_hit)
if origin_id and known_origin_ids is not None and origin_id in known_origin_ids:
page_known += 1
else:
page_new += 1
if url in hit_records:
continue
urls.append(url)
hit_records[url] = raw_hit
if origin_id:
origin_ids_by_url[url] = origin_id
if known_origin_ids is not None and threshold > 0 and (page_known + page_new) > 0:
ratio = page_known / (page_known + page_new)
if ratio >= threshold and page_new == 0:
early_stopped = True
break
page += 1
if nb_pages is not None and page >= nb_pages:
break
return urls, hit_records, origin_ids_by_url, pages, early_stopped, truncated_by_time_budget
def discover_vehicle_hits_from_algolia(
*,
settings,
make: str | None = None,
model: str | None = None,
limit: int | None = None,
year_min: int | None = None,
year_max: int | None = None,
listing_url: str | None = None,
known_origin_ids: set[str] | None = None,
max_duration_seconds: float | None = None,
) -> AlgoliaDiscoveryResult:
app_id = settings.algolia.application_id.strip()
api_key = settings.algolia.api_key.strip()
index_name = settings.algolia.index_name.strip()
if not app_id or not api_key or not index_name:
raise AlgoliaDiscoveryError("Algolia credentials/index are not configured")
effective_make = make or _extract_make_from_listing_url(listing_url)
hits_per_page = max(1, min(100, int(settings.algolia.hits_per_page)))
base_url = settings.algolia.base_url.strip().rstrip("/")
if not base_url:
base_url = f"https://{app_id}-dsn.algolia.net"
category_slug = settings.algolia.category_slug.strip() or "motors/used-cars"
make_slug = _make_slug(effective_make)
if make_slug:
category_slug = f"{category_slug}/{make_slug}"
client = _AlgoliaHttpClient(
endpoint=f"{base_url}/1/indexes/*/queries",
app_id=app_id,
api_key=api_key,
user_agent=settings.fingerprint.user_agent,
index_name=index_name,
hits_per_page=hits_per_page,
)
threshold = float(settings.listing.early_stop_threshold)
started_at = time.perf_counter()
initial_shard = _AlgoliaShard(
category_slug=category_slug,
id_min=ALGOLIA_ID_RANGE_START,
id_max=ALGOLIA_ID_RANGE_END,
year_min=year_min,
year_max=year_max,
)
shards, shard_build_truncated, total_hits = _expand_shards(
client,
initial_shard,
max_duration_seconds,
started_at,
)
collected_urls: list[str] = []
hits_by_url: dict[str, dict[str, Any]] = {}
origin_ids_by_url: dict[str, str] = {}
pages: list[dict[str, int]] = []
early_stopped = False
truncated_by_time_budget = shard_build_truncated
logger.warning(
"Algolia shard plan ready: total_hits=%d shards=%d category=%s",
total_hits,
len(shards),
category_slug,
)
for shard in shards:
shard_urls, shard_hits, shard_origin_ids, shard_pages, shard_early_stop, shard_truncated = _fetch_shard_hits(
client=client,
shard=shard,
make=effective_make,
model=model,
year_min=year_min,
year_max=year_max,
known_origin_ids=known_origin_ids,
threshold=threshold,
max_duration_seconds=max_duration_seconds,
started_at=started_at,
)
pages.extend(shard_pages)
early_stopped = early_stopped or shard_early_stop
truncated_by_time_budget = truncated_by_time_budget or shard_truncated
for url in shard_urls:
if url in hits_by_url:
continue
collected_urls.append(url)
hits_by_url[url] = shard_hits[url]
if url in shard_origin_ids:
origin_ids_by_url[url] = shard_origin_ids[url]
if limit is not None and limit > 0 and len(collected_urls) >= limit:
break
if truncated_by_time_budget:
break
logger.info(
"Algolia discovery done: urls=%d total_hits=%d pages=%d shards=%d",
len(collected_urls),
total_hits,
len(pages),
len(shards),
)
if limit is not None and limit > 0 and len(collected_urls) > limit:
collected_urls = collected_urls[:limit]
return AlgoliaDiscoveryResult(
vehicle_urls=collected_urls,
hit_records={url: hits_by_url[url] for url in collected_urls if url in hits_by_url},
origin_ids_by_url={url: origin_ids_by_url[url] for url in collected_urls if url in origin_ids_by_url},
pages=pages,
early_stopped=early_stopped,
truncated_by_time_budget=truncated_by_time_budget,
total_hits=total_hits,
)

4
iaai_scraper/discovery/sitemap.py → dubizzle_scraper/discovery/sitemap.py
View File

@@ -10,9 +10,9 @@ from urllib.parse import urlsplit, urlunsplit
from urllib.request import Request, urlopen, ProxyHandler, build_opener
import xml.etree.ElementTree as ET
logger = logging.getLogger("iaai_scraper.discovery.sitemap")
logger = logging.getLogger("dubizzle_scraper.discovery.sitemap")
DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
DEFAULT_SITEMAP_INDEX_URL = "https://www.dubizzle.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
_SITEMAP_TIMEOUT_SECONDS = 30
_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)

0
iaai_scraper/parsing/__init__.py → dubizzle_scraper/parsing/__init__.py
View File

271
iaai_scraper/parsing/mapper.py → dubizzle_scraper/parsing/mapper.py
View File

@@ -20,7 +20,7 @@ from ..storage.schemas import CarRecord, ImageRecord
class CarMapper:
# Маппер IAAI в CarRecord.
# Маппер DUBIZZLE в CarRecord.
BODY_MAP = {
"sedan": "SEDAN", "coupe": "COUPE", "hatchback": "HATCHBACK", "sport utility": "SUV",
@@ -44,6 +44,7 @@ class CarMapper:
COUNTRY_MAP = {
"us": "US", "usa": "US", "united states": "US", "ca": "CA", "canada": "CA",
"jp": "JP", "japan": "JP", "kr": "KR", "korea": "KR", "south korea": "KR",
"ae": "AE", "uae": "AE", "united arab emirates": "AE", "dubai": "AE", "abu dhabi": "AE",
}
NO_DAMAGE_MARKERS = {"normal wear", "normal wear & tear", "normal wear and tear", "n/a", "na", "none", "no damage", "minor dents/scratches"}
@@ -59,9 +60,14 @@ class CarMapper:
origin_id = self._build_origin_id(vehicle_url, vehicle_summary, core)
parser_id = self._generate_parser_id(origin_id)
brand = self._as_str(first_non_empty([core.get("make"), vehicle_summary.get("make")])) or "UNKNOWN"
model = self._as_str(first_non_empty([core.get("model"), vehicle_summary.get("model")])) or "UNKNOWN"
year = self._to_year(first_non_empty([core.get("year"), vehicle_summary.get("year")]))
brand = self._resolve_make(core, vehicle_summary) or "UNKNOWN"
model = self._resolve_model(core, vehicle_summary) or "UNKNOWN"
year = self._to_year(first_non_empty([
core.get("year"),
vehicle_summary.get("year"),
self._extract_detail_v2_value(vehicle_summary, "year"),
self._extract_detail_value(vehicle_summary, "Year"),
]))
price = self._first_parsed_int(
[
pricing.get("buy_now"),
@@ -69,29 +75,90 @@ class CarMapper:
vehicle_summary.get("buy_now"),
vehicle_summary.get("current_bid"),
pricing.get("actual_cash_value"),
vehicle_summary.get("price"),
self._extract_detail_v2_value(vehicle_summary, "price"),
self._extract_detail_value(vehicle_summary, "Price"),
],
self._to_money_int,
)
mileage = self._parse_odometer(
first_non_empty([core.get("odometer"), vehicle_summary.get("odometer")])
first_non_empty([
core.get("odometer"),
core.get("kilometers"),
vehicle_summary.get("odometer"),
vehicle_summary.get("kilometers"),
self._extract_detail_v2_value(vehicle_summary, "kilometers"),
self._extract_detail_value(vehicle_summary, "Kilometers"),
])
)
color = self._normalize_color(first_non_empty([core.get("color"), vehicle_summary.get("color"), "other"]))
drive = self._normalize_drive(first_non_empty([core.get("drive"), vehicle_summary.get("drive")]))
color = self._normalize_color(first_non_empty([
core.get("color"),
vehicle_summary.get("color"),
vehicle_summary.get("exterior_color"),
self._extract_detail_v2_value(vehicle_summary, "exterior_color"),
self._extract_detail_value(vehicle_summary, "Exterior Color"),
"other",
]))
drive = self._normalize_drive(first_non_empty([
core.get("drive"),
vehicle_summary.get("drive"),
vehicle_summary.get("drive_type"),
self._extract_detail_v2_value(vehicle_summary, "drive_system"),
self._extract_detail_value(vehicle_summary, "Drive Type"),
self._extract_detail_value(vehicle_summary, "Drive System"),
]))
# Пробуем взять привод из двигателя.
if not drive or drive == "NA":
engine_text = self._as_str(first_non_empty([core.get("engine"), vehicle_summary.get("engine")]))
engine_text = self._as_str(first_non_empty([
core.get("engine"),
vehicle_summary.get("engine"),
self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
]))
if engine_text:
inferred_drive = self._normalize_drive(engine_text)
if inferred_drive and inferred_drive != "NA":
drive = inferred_drive
gearbox = self._normalize_gearbox(first_non_empty([core.get("gearbox"), vehicle_summary.get("gearbox")]))
steering = self._normalize_steering(first_non_empty([core.get("steering_wheel"), vehicle_summary.get("steering_wheel")])) or "LEFT"
body_type = self._normalize_body_type(first_non_empty([core.get("body_type"), vehicle_summary.get("body_type")]))
engine_volume = self._to_engine_cc(first_non_empty([core.get("engine"), core.get("engine_volume"), vehicle_summary.get("engine"), vehicle_summary.get("engine_volume")]))
gearbox = self._normalize_gearbox(first_non_empty([
core.get("gearbox"),
vehicle_summary.get("gearbox"),
vehicle_summary.get("transmission_type"),
vehicle_summary.get("transmission"),
self._extract_detail_v2_value(vehicle_summary, "transmission_type"),
self._extract_detail_value(vehicle_summary, "Transmission Type"),
]))
steering = self._normalize_steering(first_non_empty([
core.get("steering_wheel"),
vehicle_summary.get("steering_wheel"),
self._extract_detail_v2_value(vehicle_summary, "steering_side"),
self._extract_detail_value(vehicle_summary, "Steering Side"),
])) or "LEFT"
body_type = self._normalize_body_type(first_non_empty([
core.get("body_type"),
vehicle_summary.get("body_type"),
self._extract_detail_v2_value(vehicle_summary, "body_type"),
self._extract_detail_value(vehicle_summary, "Body Type"),
]))
engine_volume = self._to_engine_cc(first_non_empty([
core.get("engine"),
core.get("engine_volume"),
vehicle_summary.get("engine"),
vehicle_summary.get("engine_volume"),
self._extract_detail_v2_value(vehicle_summary, "engine_capacity_cc"),
self._extract_detail_value(vehicle_summary, "Engine Capacity (cc)"),
]))
title_text = self._as_str(first_non_empty([core.get("title"), vehicle_summary.get("title"), ""]))
seller = self._as_str(first_non_empty([core.get("seller"), vehicle_summary.get("seller"), ""]))
location = self._as_str(first_non_empty([core.get("location"), vehicle_summary.get("location"), auction.get("branch"), ""]))
country = self._normalize_country(first_non_empty([core.get("country"), location, "US"]))
location = self._as_str(first_non_empty([
core.get("location"),
vehicle_summary.get("location"),
vehicle_summary.get("location_name"),
self._extract_nested_text(vehicle_summary.get("neighbourhood"), "en"),
self._extract_location_country(vehicle_summary),
auction.get("branch"),
"",
]))
country = self._normalize_country(first_non_empty([core.get("country"), location, "AE"]))
is_damaged = self._bool_damage(damage, vehicle_summary)
is_sold = self._bool_sold(auction)
one_owner = self._boolish(first_non_empty([core.get("one_owner"), vehicle_summary.get("one_owner"), False]))
@@ -115,13 +182,13 @@ class CarMapper:
)
slug = self._slugify(" ".join(filter(None, [brand, model, str(year or "")])))
images_records = self._build_images(images.get("urls") or vehicle_summary.get("image_urls") or [])
origin = "IAAI"
origin = "DUBIZZLE"
return CarRecord(
parser_id=parser_id, brand=brand, model=model, year=year, price=price, currency=currency,
mileage=mileage, country=country, is_sold=is_sold, color=color, drive=drive, gearbox=gearbox,
steering_wheel=steering, body_type=body_type, engine_volume=engine_volume,
selling_type=self._normalize_selling_type("AUCTION"), one_owner=one_owner, new_car=new_car,
selling_type=self._normalize_selling_type(first_non_empty([core.get("selling_type"), "STOCK"])), one_owner=one_owner, new_car=new_car,
is_hidden=False, origin=origin, origin_url=vehicle_url, origin_id=origin_id, is_damaged=is_damaged,
evaluation=evaluation, non_smoking=non_smoking, rental=rental, repair_history=repair_history,
slug=slug, last_seen_at=datetime.now(timezone.utc), images=images_records,
@@ -131,6 +198,146 @@ class CarMapper:
def _as_str(value: Any) -> str:
return "" if value is None else str(value).strip()
def _resolve_make(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
category_make, _ = self._extract_category_make_model(vehicle_summary)
slug_make, _ = self._extract_slug_make_model(vehicle_summary)
return self._first_text(
[
core.get("make"),
vehicle_summary.get("make"),
self._extract_detail_v2_value(vehicle_summary, "make"),
self._extract_detail_value(vehicle_summary, "Make"),
category_make,
slug_make,
]
)
def _resolve_model(self, core: dict[str, Any], vehicle_summary: dict[str, Any]) -> str | None:
_, category_model = self._extract_category_make_model(vehicle_summary)
_, slug_model = self._extract_slug_make_model(vehicle_summary)
return self._first_text(
[
core.get("model"),
vehicle_summary.get("model"),
self._extract_detail_v2_value(vehicle_summary, "model"),
self._extract_detail_value(vehicle_summary, "Model"),
category_model,
slug_model,
]
)
def _first_text(self, values: list[Any]) -> str | None:
for value in values:
text = self._coerce_text(value)
if text:
return text
return None
def _coerce_text(self, value: Any) -> str | None:
if value is None:
return None
if isinstance(value, list):
for item in value:
text = self._coerce_text(item)
if text:
return text
return None
if isinstance(value, dict):
for key in ("en", "value", "name", "text", "label"):
if key in value:
text = self._coerce_text(value.get(key))
if text:
return text
for nested in value.values():
text = self._coerce_text(nested)
if text:
return text
return None
text = self._as_str(value)
return text or None
def _extract_nested_text(self, value: Any, preferred_key: str = "en") -> str | None:
if not isinstance(value, dict):
return self._coerce_text(value)
return self._coerce_text(value.get(preferred_key)) or self._coerce_text(value)
def _extract_detail_v2_value(self, vehicle_summary: dict[str, Any], slug: str) -> str | None:
details_v2 = vehicle_summary.get("details_v2")
if not isinstance(details_v2, dict):
return None
target = slug.strip().lower()
for section in details_v2.values():
if not isinstance(section, list):
continue
for item in section:
if not isinstance(item, dict):
continue
if self._as_str(item.get("slug")).lower() != target:
continue
text = self._coerce_text(item.get("value"))
if text:
return text
return None
def _extract_detail_value(self, vehicle_summary: dict[str, Any], detail_key: str) -> str | None:
details = vehicle_summary.get("details")
if not isinstance(details, dict):
return None
target = detail_key.strip().lower()
for key, value in details.items():
if self._as_str(key).lower() != target:
continue
text = self._coerce_text(value)
if text:
return text
return None
def _extract_category_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
category_v2 = vehicle_summary.get("category_v2")
if isinstance(category_v2, dict):
names_en = category_v2.get("names_en")
if isinstance(names_en, list) and len(names_en) >= 4:
return self._coerce_text(names_en[2]), self._coerce_text(names_en[3])
category = vehicle_summary.get("category")
if isinstance(category, dict):
names_en = category.get("en")
if isinstance(names_en, list) and len(names_en) >= 3:
return self._coerce_text(names_en[1]), self._coerce_text(names_en[2])
return None, None
def _extract_slug_make_model(self, vehicle_summary: dict[str, Any]) -> tuple[str | None, str | None]:
category_v2 = vehicle_summary.get("category_v2")
if not isinstance(category_v2, dict):
return None, None
slug_paths = category_v2.get("slug_paths")
if not isinstance(slug_paths, list) or len(slug_paths) < 3:
return None, None
make = self._humanize_slug_path_part(slug_paths[2])
model = self._humanize_slug_path_part(slug_paths[3]) if len(slug_paths) >= 4 else None
return make, model
def _humanize_slug_path_part(self, value: Any) -> str | None:
text = self._as_str(value)
if not text:
return None
slug = text.split("/")[-1].strip().strip("-")
if not slug:
return None
return slug.replace("-", " ").title()
def _extract_location_country(self, vehicle_summary: dict[str, Any]) -> str | None:
site = vehicle_summary.get("site")
if isinstance(site, dict):
return self._coerce_text(site.get("en"))
location_list = vehicle_summary.get("location_list")
if isinstance(location_list, dict):
en_values = location_list.get("en")
if isinstance(en_values, list) and en_values:
return self._coerce_text(en_values[0])
return None
@staticmethod
def _to_int(value: Any) -> int | None:
if value is None:
@@ -238,7 +445,7 @@ class CarMapper:
return None
if liters_match := re.search(r"(\d+(?:\.\d+)?)\s*l", text):
return int(float(liters_match.group(1)) * 1000)
if cubic_match := re.search(r"(\d{3,5})\s*(?:cc|cm3|cubic)", text):
if cubic_match := re.search(r"(\d{3,5})(?:\+)?\s*(?:cc|cm3|cubic)", text):
return int(cubic_match.group(1))
return int(text) if re.match(r"^\d+$", text) else None
@@ -278,12 +485,16 @@ class CarMapper:
return self._map_value(value, self.BODY_MAP, BODY_TYPE_ENUM_VALUES, empty_default="OTHER", fallback="OTHER") or "OTHER"
def _normalize_country(self, value: Any) -> str:
fallback = "US" if "US" in COUNTRY_ENUM_VALUES else "NA"
return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="US", fallback=fallback) or fallback
fallback = "AE" if "AE" in COUNTRY_ENUM_VALUES else "NA"
return self._map_value(value, self.COUNTRY_MAP, COUNTRY_ENUM_VALUES, empty_default="AE", fallback=fallback) or fallback
def _normalize_selling_type(self, value: Any) -> str:
text = self._as_str(value) or "AUCTION"
return text if text in SELLING_TYPE_ENUM_VALUES else "AUCTION"
text = (self._as_str(value) or "STOCK").upper()
if text in SELLING_TYPE_ENUM_VALUES:
return text
if text in {"DEALER", "PRIVATE", "CLASSIFIED"}:
return "STOCK"
return "STOCK"
def _map_value(
self,
@@ -357,13 +568,13 @@ class CarMapper:
@staticmethod
def _make_fullres_url(url: str) -> str:
if "vis.iaai.com" in url:
if "vis.dubizzle.com" in url:
return re.sub(r'height=\d+', 'height=633', re.sub(r'width=\d+', 'width=845', url))
return url
@staticmethod
def _make_preview_url(url: str) -> str:
if "vis.iaai.com" in url:
if "vis.dubizzle.com" in url:
preview = re.sub(r'height=\d+', 'height=300', re.sub(r'width=\d+', 'width=400', url))
if preview != url:
return preview
@@ -387,16 +598,22 @@ class CarMapper:
return "car-" + "".join(chars)
def _build_origin_id(self, vehicle_url: str, vehicle_summary: dict[str, Any], core: dict[str, Any]) -> str:
# Формат: iaai:{lot_number}.
for value in [core.get("lot_number"), vehicle_summary.get("lot_number")]:
# Формат: dubizzle:{lot_number}.
for value in [
core.get("lot_number"),
vehicle_summary.get("lot_number"),
vehicle_summary.get("id"),
vehicle_summary.get("objectID"),
vehicle_summary.get("uuid"),
]:
text = self._as_str(value)
if text:
return f"iaai:{text}"
return f"dubizzle:{text}"
tail = urlparse(vehicle_url).path.rstrip("/").split("/")[-1]
if "~" in tail:
tail = tail.split("~")[0]
raw = tail or self._slugify(vehicle_url)
return f"iaai:{raw}"
return f"dubizzle:{raw}"
@staticmethod
def _slugify(value: str) -> str:

12
iaai_scraper/parsing/parser.py → dubizzle_scraper/parsing/parser.py
View File

@@ -6,7 +6,7 @@ from typing import Any
from ..core.utils import LOT_RE, PRICE_RE, deep_find_all_keys, deep_find_key, first_non_empty
logger = logging.getLogger("iaai_scraper.parsers")
logger = logging.getLogger("dubizzle_scraper.parsers")
class VehicleParser:
@@ -350,7 +350,7 @@ class VehicleParser:
if isinstance(item, str) and item.startswith("http"):
cleaned = html_module.unescape(item)
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
continue
if cleaned not in seen_flat:
seen_flat.add(cleaned)
@@ -360,7 +360,7 @@ class VehicleParser:
if isinstance(child, str) and child.startswith("http"):
cleaned = html_module.unescape(child)
lowered = cleaned.lower()
if vehicle_key and "vis.iaai.com" in lowered and vehicle_key not in cleaned:
if vehicle_key and "vis.dubizzle.com" in lowered and vehicle_key not in cleaned:
continue
if cleaned not in seen_flat:
seen_flat.add(cleaned)
@@ -374,13 +374,13 @@ class VehicleParser:
if vehicle_key and vehicle_key in url:
seen_flat.add(url)
flat.append(url)
elif any(token in lowered for token in ["vis.iaai.com", "anvis", "vehicleimage"]):
elif any(token in lowered for token in ["vis.dubizzle.com", "anvis", "vehicleimage"]):
if vehicle_key and vehicle_key not in url:
continue
seen_flat.add(url)
flat.append(url)
if vehicle_key:
for url in re.findall(r'https?://vis\.iaai\.com[^\s"\'<>]+', html or ""):
for url in re.findall(r'https?://vis\.dubizzle\.com[^\s"\'<>]+', html or ""):
cleaned = html_module.unescape(url)
if cleaned not in seen_flat and vehicle_key in cleaned:
seen_flat.add(cleaned)
@@ -390,7 +390,7 @@ class VehicleParser:
lowered = url.lower()
if any(pat in lowered for pat in {"dimensions", "threesixty", "360view", ".js", ".css", ".svg", "/home/", "iframeview"}):
continue
if "vis.iaai.com" in lowered and "/resizer" not in lowered:
if "vis.dubizzle.com" in lowered and "/resizer" not in lowered:
continue
filtered.append(url)
return filtered

0
iaai_scraper/proxy_bridge.py → dubizzle_scraper/proxy_bridge.py
View File

1153
iaai_scraper/scraper.py → dubizzle_scraper/scraper.py
View File

File diff suppressed because it is too large Load Diff

0
iaai_scraper/storage/__init__.py → dubizzle_scraper/storage/__init__.py
View File

26
iaai_scraper/storage/db.py → dubizzle_scraper/storage/db.py
View File

@@ -11,7 +11,7 @@ from ..core.config import Settings
from .models import Base, Car, Image, SyncRun
from .schemas import CarRecord
logger = logging.getLogger("iaai_scraper.db")
logger = logging.getLogger("dubizzle_scraper.db")
CAR_DB_FIELDS = {
@@ -492,7 +492,7 @@ class PersistenceService:
logger.error("Individual upsert failed for %s: %s", record.origin_id, exc)
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "iaai") -> int:
def mark_sold_not_in_listing(self, active_origin_ids: set[str], lane: str = "dubizzle") -> int:
"""Помечает авто как проданные, если их нет в активном листинге (по origin_id)."""
if not active_origin_ids:
return 0
@@ -501,7 +501,7 @@ class PersistenceService:
update(Car)
.where(Car.origin_id.notin_(active_origin_ids))
.where(Car.is_sold == False) # noqa: E712
.where(Car.origin_id.like("iaai:%"))
.where(Car.origin_id.like("dubizzle:%"))
.values(is_sold=True)
)
result = session.execute(stmt)
@@ -510,7 +510,7 @@ class PersistenceService:
logger.info("Marked %d cars as sold (no longer in listing)", count)
return count
def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "iaai") -> int:
def mark_sold_not_in_listing_by_urls(self, active_origin_urls: set[str], lane: str = "dubizzle") -> int:
"""Помечает авто как проданные, если их URL нет в активном листинге.
Для PostgreSQL использует временную таблицу + LEFT JOIN вместо NOT IN,
@@ -532,7 +532,7 @@ class PersistenceService:
if is_postgres:
# Считаем кандидатов на sold.
total_active = session.execute(
text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'dubizzle:%%'")
).scalar() or 0
if total_active == 0:
@@ -560,7 +560,7 @@ class PersistenceService:
LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
WHERE a.url IS NULL
AND c.is_sold = FALSE
AND c.origin_id LIKE 'iaai:%%'
AND c.origin_id LIKE 'dubizzle:%%'
""")).scalar() or 0
# Защита от аномалии.
@@ -581,7 +581,7 @@ class PersistenceService:
LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
WHERE a.url IS NULL
AND c.is_sold = FALSE
AND c.origin_id LIKE 'iaai:%%'
AND c.origin_id LIKE 'dubizzle:%%'
) sub
WHERE cars.id = sub.id
"""))
@@ -591,13 +591,13 @@ class PersistenceService:
stmt = (
update(Car)
.where(Car.is_sold == False) # noqa: E712
.where(Car.origin_id.like("iaai:%"))
.where(Car.origin_id.like("dubizzle:%"))
.values(is_sold=True)
)
# Загружаем active URL.
all_active = session.execute(
select(Car.id, Car.origin_url).where(
Car.is_sold == False, Car.origin_id.like("iaai:%") # noqa: E712
Car.is_sold == False, Car.origin_id.like("dubizzle:%") # noqa: E712
)
).all()
mark_ids = [row[0] for row in all_active if _norm(row[1]) not in normalized_urls]
@@ -621,7 +621,7 @@ class PersistenceService:
logger.info("Marked %d cars as sold by URL (no longer in listing)", count)
return count
def get_all_origin_ids_for_lane(self, prefix: str = "iaai:") -> set[str]:
def get_all_origin_ids_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
"""Возвращает все известные origin_id для заданного префикса.
Использует yield_per для потоковой загрузки при большом количестве записей.
@@ -632,7 +632,7 @@ class PersistenceService:
)
return {str(row[0]) for row in result if row and row[0]}
def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
def get_all_active_origin_urls_for_lane(self, prefix: str = "dubizzle:") -> set[str]:
"""Возвращает origin_url всех активных (не проданных) авто для заданного lane-префикса."""
with self.session_scope() as session:
result = session.execute(
@@ -643,7 +643,7 @@ class PersistenceService:
)
return {str(row[0]) for row in result if row and row[0]}
def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
def count_active_cars_for_lane(self, prefix: str = "dubizzle:") -> int:
"""Возвращает количество активных (не проданных) авто для заданного lane-префикса."""
from sqlalchemy import func as sa_func
with self.session_scope() as session:
@@ -657,7 +657,7 @@ class PersistenceService:
def get_active_origin_urls_batch_for_refresh(
self,
prefix: str = "iaai:",
prefix: str = "dubizzle:",
offset: int = 0,
limit: int = 500,
) -> list[str]:

4
iaai_scraper/storage/enums.py → dubizzle_scraper/storage/enums.py
View File

@@ -16,9 +16,9 @@ BODY_TYPE_ENUM_VALUES = (
"OTHER",
"NA",
)
COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "NA")
COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "AE", "NA")
ORIGIN_ENUM_VALUES = (
"IAAI",
"DUBIZZLE",
"NA",
)
SELLING_TYPE_ENUM_VALUES = ("STOCK", "AUCTION", "TENDER", "NA")

0
iaai_scraper/storage/models.py → dubizzle_scraper/storage/models.py
View File

0
iaai_scraper/storage/schemas.py → dubizzle_scraper/storage/schemas.py
View File

0
iaai_scraper/worker/__init__.py → dubizzle_scraper/worker/__init__.py
View File

28
iaai_scraper/worker/celery_app.py → dubizzle_scraper/worker/celery_app.py
View File

@@ -9,8 +9,7 @@ from redis import Redis
from ..core.config import settings
from ..core.logs import setup_logging
logger = logging.getLogger("iaai_scraper.worker.celery_app")
STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
logger = logging.getLogger("dubizzle_scraper.worker.celery_app")
@celery_setup_logging.connect
@@ -37,7 +36,7 @@ def _result_backend() -> str:
celery_app = Celery(
"iaai_scraper",
"dubizzle_scraper",
broker=_broker_url(),
backend=_result_backend(),
)
@@ -79,10 +78,13 @@ celery_app.conf.update(
worker_hijack_root_logger=False,
beat_schedule={
"periodic-sync-listing": {
"task": "iaai_scraper.worker.tasks.sync_listing_task",
"task": "dubizzle_scraper.worker.tasks.sync_listing_task",
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
"args": (),
"kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
"kwargs": {
"limit": settings.celery.beat_sync_limit,
"only_new": False if settings.discovery.always_full_scan else True,
},
"options": {
"queue": "scraping",
"expires": settings.celery.beat_sync_interval_minutes * 60.0,
@@ -90,11 +92,11 @@ celery_app.conf.update(
}
},
task_routes={
"iaai_scraper.worker.tasks.*": {"queue": "scraping"}
"dubizzle_scraper.worker.tasks.*": {"queue": "scraping"}
},
)
celery_app.autodiscover_tasks(["iaai_scraper.worker"])
celery_app.autodiscover_tasks(["dubizzle_scraper.worker"])
@worker_ready.connect
@@ -111,7 +113,7 @@ def _on_worker_ready(**kwargs):
retry_on_timeout=True,
)
for stale_key in ("iaai:locks:sync_listing",):
for stale_key in ("dubizzle:locks:sync_listing",):
try:
ttl = redis_client.ttl(stale_key)
if ttl is not None and ttl != -2:
@@ -127,10 +129,8 @@ def _on_worker_ready(**kwargs):
if queue_len > 0:
logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len)
return
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
except Exception:
logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
logger.warning("Worker ready startup sync check failed; skipping immediate dispatch", exc_info=True)
return
finally:
if redis_client is not None:
@@ -139,13 +139,9 @@ def _on_worker_ready(**kwargs):
except Exception:
pass
if not should_dispatch:
logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
return
logger.info("Worker ready — dispatching initial sync_listing task")
celery_app.send_task(
"iaai_scraper.worker.tasks.sync_listing_task",
"dubizzle_scraper.worker.tasks.sync_listing_task",
kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
queue="scraping",
expires=settings.celery.beat_sync_interval_minutes * 60.0,

40
iaai_scraper/worker/self_heal.py → dubizzle_scraper/worker/self_heal.py
View File

@@ -8,21 +8,28 @@ import time
from redis import Redis
logger = logging.getLogger("iaai_scraper.worker.self_heal")
logger = logging.getLogger("dubizzle_scraper.worker.self_heal")
GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress"
GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts"
SELF_HEAL_RESTART_LOCK_KEY = "dubizzle:state:self_heal_restart_in_progress"
def _env_str(name: str, default: str) -> str:
value = os.getenv(name)
if value is None and name.startswith("DUBIZZLE_"):
value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):])
return value if value is not None else default
def _env_bool(name: str, default: bool) -> bool:
value = os.getenv(name)
value = _env_str(name, "true" if default else "false")
if value is None:
return default
return value.strip().lower() in {"1", "true", "yes", "on"}
def _env_int(name: str, default: int) -> int:
value = os.getenv(name)
value = _env_str(name, str(default))
if value is None:
return default
try:
@@ -32,7 +39,7 @@ def _env_int(name: str, default: int) -> int:
def _get_redis() -> Redis:
url = os.getenv("IAAI_REDIS_URL", "redis://redis:6379/0")
url = _env_str("DUBIZZLE_REDIS_URL", "redis://redis:6379/0")
return Redis.from_url(
url,
decode_responses=True,
@@ -62,7 +69,7 @@ def _read_last_progress_ts(redis_client: Redis) -> int | None:
# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
# Это дороже, но выполняется только при отсутствии основного маркера.
max_ts = 0
for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
for key in redis_client.scan_iter(match="dubizzle:state:task_progress:*"):
try:
payload = redis_client.get(key)
if not payload:
@@ -101,7 +108,8 @@ def _kill_worker_process() -> None:
# Если процесс ещё жив — принудительно убиваем.
os.kill(pid, 0)
logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
os.kill(pid, signal.SIGKILL)
kill_signal = getattr(signal, "SIGKILL", signal.SIGTERM)
os.kill(pid, kill_signal)
except ProcessLookupError:
pass
except Exception:
@@ -109,15 +117,15 @@ def _kill_worker_process() -> None:
def main() -> None:
if not _env_bool("IAAI_SELF_HEAL_ENABLED", True):
logger.info("Self-heal watchdog disabled via IAAI_SELF_HEAL_ENABLED")
if not _env_bool("DUBIZZLE_SELF_HEAL_ENABLED", True):
logger.info("Self-heal watchdog disabled via DUBIZZLE_SELF_HEAL_ENABLED/DUBIZZLE_SELF_HEAL_ENABLED")
return
queue_name = os.getenv("IAAI_CELERY_QUEUE", "scraping")
check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720))
startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
queue_name = _env_str("DUBIZZLE_CELERY_QUEUE", "scraping")
check_interval = max(5, _env_int("DUBIZZLE_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
stall_seconds = max(180, _env_int("DUBIZZLE_SELF_HEAL_STALL_SECONDS", 720))
startup_grace = max(30, _env_int("DUBIZZLE_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
restart_cooldown = max(60, _env_int("DUBIZZLE_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
logger.warning(
"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
@@ -195,7 +203,7 @@ def main() -> None:
if __name__ == "__main__":
logging.basicConfig(
level=os.getenv("IAAI_LOG_LEVEL", "INFO"),
level=_env_str("DUBIZZLE_LOG_LEVEL", "INFO"),
format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
)
main()

297
iaai_scraper/worker/tasks.py → dubizzle_scraper/worker/tasks.py
View File

@@ -1,4 +1,4 @@
# Задачи Celery для синхронизации автомобилей и листинга IAAI.
# Задачи Celery для синхронизации автомобилей и листинга Dubizzle.
import json
import logging
@@ -13,25 +13,39 @@ from celery import shared_task
from redis import Redis
from ..core.config import Settings, parse_listing_segments
from ..scraper import IAAIScraper
from ..scraper import DUBIZZLEScraper
from ..storage.db import PersistenceService
from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats
logger = logging.getLogger("iaai_scraper.worker.tasks")
logger = logging.getLogger("dubizzle_scraper.worker.tasks")
# Минимальная пауза между батчами (секунды) — не давит IAAI.
_INTER_BATCH_DELAY = max(float(os.getenv("IAAI_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0)
# Если доля failed в батче превышает порог — прерываем (IAAI блокирует).
_FAIL_RATE_THRESHOLD = min(max(float(os.getenv("IAAI_FAIL_RATE_THRESHOLD", "0.9")), 0.0), 1.0)
SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
def _env_str(name: str, default: str) -> str:
value = os.getenv(name)
if value is None and name.startswith("DUBIZZLE_"):
value = os.getenv("DUBIZZLE_" + name[len("DUBIZZLE_"):])
return value if value is not None else default
def _env_float(name: str, default: float) -> float:
try:
return float(_env_str(name, str(default)).strip())
except Exception:
return default
# Минимальная пауза между батчами (секунды).
_INTER_BATCH_DELAY = max(_env_float("DUBIZZLE_INTER_BATCH_DELAY_SECONDS", 0.3), 0.0)
# Если доля failed в батче превышает порог — прерываем.
_FAIL_RATE_THRESHOLD = min(max(_env_float("DUBIZZLE_FAIL_RATE_THRESHOLD", 0.9), 0.0), 1.0)
SYNC_LISTING_LOCK_KEY = "dubizzle:locks:sync_listing"
SYNC_FULL_SCAN_DONE_KEY = "dubizzle:state:sync_full_scan_done"
SYNC_LISTING_CHECKPOINT_KEY = "dubizzle:state:sync_listing_checkpoint"
SYNC_LISTING_CHECKPOINT_TTL_SECONDS = 7 * 24 * 60 * 60
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "iaai:state:sync_listing_bootstrap_failure_streak"
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_failure_streak"
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60
SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "iaai:state:sync_listing_bootstrap_continuation_streak"
SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = "dubizzle:state:sync_listing_bootstrap_continuation_streak"
SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT = max(
1,
int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_LIMIT", "6")),
@@ -40,19 +54,45 @@ SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS = max(
60,
int(os.getenv("SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_TTL_SECONDS", str(6 * 60 * 60))),
)
HOURLY_FAILURE_STREAK_KEY = "iaai:state:hourly_failure_streak"
HOURLY_FAILURE_STREAK_KEY = "dubizzle:state:hourly_failure_streak"
HOURLY_FAILURE_STREAK_LIMIT = 3
HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # сброс через 6 часов
SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task"
SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}"
SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress"
SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total"
SYNC_LISTING_FOLLOWUP_PENDING_KEY = "dubizzle:state:sync_listing_followup_pending"
SYNC_LISTING_TASK_NAME = "dubizzle_scraper.worker.tasks.sync_listing_task"
SYNC_SEGMENT_LOCK_KEY_FMT = "dubizzle:locks:sync_segment:{idx}"
SYNC_SEGMENTS_PROGRESS_KEY = "dubizzle:state:sync_segments_progress"
SYNC_SEGMENTS_TOTAL_KEY = "dubizzle:state:sync_segments_total"
SYNC_SEGMENTED_SCAN_ACTIVE_KEY = "dubizzle:state:sync_segmented_scan_active"
SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60
TASK_PROGRESS_KEY_FMT = "iaai:state:task_progress:{task_id}"
GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
SITEMAP_HOURLY_LAST_COUNT_KEY = "iaai:state:sitemap_hourly_last_count"
SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "iaai:state:sitemap_hourly_refresh_offset"
TASK_PROGRESS_KEY_FMT = "dubizzle:state:task_progress:{task_id}"
GLOBAL_PROGRESS_TS_KEY = "dubizzle:state:last_progress_ts"
SITEMAP_HOURLY_LAST_COUNT_KEY = "dubizzle:state:sitemap_hourly_last_count"
SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "dubizzle:state:sitemap_hourly_refresh_offset"
def _runtime_key(key: str) -> str:
return key.replace("dubizzle:", "dubizzle:")
def _origin_prefix() -> str:
return _env_str("DUBIZZLE_ORIGIN_PREFIX", "dubizzle:")
SYNC_LISTING_LOCK_KEY = _runtime_key(SYNC_LISTING_LOCK_KEY)
SYNC_FULL_SCAN_DONE_KEY = _runtime_key(SYNC_FULL_SCAN_DONE_KEY)
SYNC_LISTING_CHECKPOINT_KEY = _runtime_key(SYNC_LISTING_CHECKPOINT_KEY)
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_KEY)
SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY = _runtime_key(SYNC_LISTING_BOOTSTRAP_CONTINUATION_STREAK_KEY)
HOURLY_FAILURE_STREAK_KEY = _runtime_key(HOURLY_FAILURE_STREAK_KEY)
SYNC_LISTING_FOLLOWUP_PENDING_KEY = _runtime_key(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
SYNC_SEGMENT_LOCK_KEY_FMT = _runtime_key(SYNC_SEGMENT_LOCK_KEY_FMT)
SYNC_SEGMENTS_PROGRESS_KEY = _runtime_key(SYNC_SEGMENTS_PROGRESS_KEY)
SYNC_SEGMENTS_TOTAL_KEY = _runtime_key(SYNC_SEGMENTS_TOTAL_KEY)
SYNC_SEGMENTED_SCAN_ACTIVE_KEY = _runtime_key(SYNC_SEGMENTED_SCAN_ACTIVE_KEY)
TASK_PROGRESS_KEY_FMT = _runtime_key(TASK_PROGRESS_KEY_FMT)
GLOBAL_PROGRESS_TS_KEY = _runtime_key(GLOBAL_PROGRESS_TS_KEY)
SITEMAP_HOURLY_LAST_COUNT_KEY = _runtime_key(SITEMAP_HOURLY_LAST_COUNT_KEY)
SITEMAP_HOURLY_REFRESH_OFFSET_KEY = _runtime_key(SITEMAP_HOURLY_REFRESH_OFFSET_KEY)
STALL_WATCHDOG_NAVIGATION_STAGES = {
"listing_next_page_started",
"listing_resume_progress",
@@ -150,7 +190,16 @@ def _clear_task_progress(redis_client: Redis, task_id: str) -> None:
logger.warning("Failed to clear task progress for %s", task_id, exc_info=True)
def _stall_timeout_for_progress(stage: str | None, default_timeout: int) -> int:
def _stall_timeout_for_progress(
stage: str | None,
default_timeout: int,
*,
heartbeat_only: bool = False,
) -> int:
if heartbeat_only:
# Heartbeat-пульсы не считаем полноценным прогрессом:
# при долгом зависании даём только ограниченный grace-период.
return min(int(default_timeout), 120)
if stage in STALL_WATCHDOG_NAVIGATION_STAGES:
return max(int(default_timeout), STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS)
return int(default_timeout)
@@ -165,12 +214,12 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool |
discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings)
discovered_urls = discovery_result.vehicle_urls
active_urls = set(discovered_urls)
existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:")
existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix())
new_urls = [url for url in discovered_urls if url not in existing_urls]
sold_count = 0
if active_urls:
sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai")
sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":"))
cars_upserted = 0
cars_failed = 0
@@ -179,7 +228,7 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool |
failures: list[dict[str, str]] = []
if new_urls:
with IAAIScraper(settings) as scraper:
with DUBIZZLEScraper(settings) as scraper:
if progress_callback:
scraper.set_progress_callback(progress_callback)
batch_size = settings.celery.batch_size
@@ -194,7 +243,7 @@ def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool |
protection_events += batch_protection
images_upserted += int(batch_result.get("images_upserted", 0))
failures.extend(batch_result.get("failures", []))
# Fail-fast: если слишком много ошибок — IAAI блокирует, не тратим ресурсы.
# Fail-fast: если слишком много ошибок — DUBIZZLE блокирует, не тратим ресурсы.
total_in_batch = batch_ok + batch_fail
if total_in_batch > 0 and batch_fail / total_in_batch >= _FAIL_RATE_THRESHOLD:
logger.warning("Fail-fast: %d/%d failed in batch, stopping", batch_fail, total_in_batch)
@@ -233,7 +282,7 @@ def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new:
active_urls = set(discovered_urls)
sold_count = 0
if active_urls:
sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai")
sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":"))
cars_upserted = 0
cars_failed = 0
@@ -241,7 +290,7 @@ def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new:
images_upserted = 0
failures: list[dict[str, str]] = []
with IAAIScraper(settings) as scraper:
with DUBIZZLEScraper(settings) as scraper:
if progress_callback:
scraper.set_progress_callback(progress_callback)
batch_size = settings.celery.batch_size
@@ -301,12 +350,12 @@ def _hourly_sitemap_rolling_refresh_sync(
active_urls = set(discovered_urls)
sold_count = 0
if active_urls:
sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai")
sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane=_origin_prefix().rstrip(":"))
existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:")
existing_urls = persistence.get_all_active_origin_urls_for_lane(_origin_prefix())
new_urls = [url for url in discovered_urls if url not in existing_urls]
total_active = persistence.count_active_cars_for_lane("iaai:")
total_active = persistence.count_active_cars_for_lane(_origin_prefix())
batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size))
try:
offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0)
@@ -314,14 +363,14 @@ def _hourly_sitemap_rolling_refresh_sync(
offset = 0
refresh_urls = persistence.get_active_origin_urls_batch_for_refresh(
prefix="iaai:",
prefix=_origin_prefix(),
offset=offset,
limit=batch_size,
)
if not refresh_urls and total_active > 0:
offset = 0
refresh_urls = persistence.get_active_origin_urls_batch_for_refresh(
prefix="iaai:",
prefix=_origin_prefix(),
offset=0,
limit=batch_size,
)
@@ -352,7 +401,7 @@ def _hourly_sitemap_rolling_refresh_sync(
failures: list[dict[str, str]] = []
if target_urls:
with IAAIScraper(settings) as scraper:
with DUBIZZLEScraper(settings) as scraper:
if progress_callback:
scraper.set_progress_callback(progress_callback)
worker_batch_size = settings.celery.batch_size
@@ -438,12 +487,19 @@ def _start_stall_watchdog(
no_data_count = 0
data = json.loads(raw)
stage = data.get("stage")
heartbeat_only = bool(data.get("heartbeat_only"))
last_ts = int(data.get("ts") or 0)
if not last_ts:
continue
effective_stall_timeout = _stall_timeout_for_progress(stage, stall_timeout_seconds)
effective_stall_timeout = _stall_timeout_for_progress(
stage,
stall_timeout_seconds,
heartbeat_only=heartbeat_only,
)
age = int(time.time()) - last_ts
if age < effective_stall_timeout:
# Heartbeat-пульс не должен бесконечно продлевать дедлайн.
if not heartbeat_only:
watchdog_born = time.monotonic() # reset absolute deadline on real progress
continue
logger.error(
@@ -620,6 +676,15 @@ def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> N
def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None = None) -> bool:
return _has_running_task_named(celery_app, SYNC_LISTING_TASK_NAME, exclude_task_id=exclude_task_id)
def _has_running_task_named(
celery_app,
task_name: str,
*,
exclude_task_id: str | None = None,
) -> bool:
try:
inspector = celery_app.control.inspect(timeout=1.0)
snapshots = [
@@ -634,8 +699,8 @@ def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None =
for snapshot in snapshots:
for entries in snapshot.values():
for entry in entries or []:
task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
if task_name != SYNC_LISTING_TASK_NAME:
entry_task_name = str(entry.get("name") or entry.get("request", {}).get("name") or "")
if entry_task_name != task_name:
continue
# Исключаем текущую задачу — она не считается "другой запущенной"
entry_id = str(entry.get("id") or entry.get("request", {}).get("id") or "")
@@ -645,6 +710,59 @@ def _has_running_sync_listing_tasks(celery_app, *, exclude_task_id: str | None =
return False
def _has_running_sync_segment_tasks(celery_app) -> bool:
return _has_running_task_named(celery_app, "dubizzle_scraper.worker.tasks.sync_segment_task")
def _set_segmented_scan_active(redis_client: Redis, *, total: int) -> None:
try:
payload = {
"ts": int(time.time()),
"total": int(total),
}
redis_client.set(
SYNC_SEGMENTED_SCAN_ACTIVE_KEY,
json.dumps(payload, ensure_ascii=False),
ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS,
)
except Exception:
logger.warning("Failed to persist segmented scan active state", exc_info=True)
def _clear_segmented_scan_active(redis_client: Redis) -> None:
try:
redis_client.delete(SYNC_SEGMENTED_SCAN_ACTIVE_KEY)
except Exception:
logger.warning("Failed to clear segmented scan active state", exc_info=True)
def _is_segmented_scan_in_progress(redis_client: Redis, celery_app) -> bool:
try:
marker = redis_client.get(SYNC_SEGMENTED_SCAN_ACTIVE_KEY)
except Exception:
logger.warning("Failed to read segmented scan active state", exc_info=True)
return False
if not marker:
return False
if _has_running_sync_segment_tasks(celery_app):
return True
try:
queue_len = int(redis_client.llen("scraping") or 0)
except Exception:
logger.warning("Failed to inspect scraping queue length", exc_info=True)
queue_len = 0
if queue_len > 0:
return True
logger.warning("Clearing stale segmented scan active state: no running segment tasks detected")
_clear_segmented_scan_active(redis_client)
return False
def _clear_orphan_sync_listing_lock(redis_client: Redis, celery_app, *, current_task_id: str | None = None) -> bool:
try:
owner_token = redis_client.get(SYNC_LISTING_LOCK_KEY)
@@ -902,7 +1020,7 @@ def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[in
@shared_task(
name="iaai_scraper.worker.tasks.sync_segment_task",
name="dubizzle_scraper.worker.tasks.sync_segment_task",
bind=True,
max_retries=2,
default_retry_delay=60,
@@ -912,7 +1030,7 @@ def sync_segment_task(
self,
segment_index: int,
segment: dict,
lane: str = "iaai_cars",
lane: str = "dubizzle_cars",
only_new: bool | None = None,
is_bootstrap: bool = False,
):
@@ -965,7 +1083,7 @@ def sync_segment_task(
try:
def _job():
with IAAIScraper() as scraper:
with DUBIZZLEScraper() as scraper:
scraper.set_progress_callback(
lambda stage, meta: _update_task_progress(
redis_client,
@@ -1011,9 +1129,15 @@ def sync_segment_task(
segment_index, seg_label, completed, total,
)
if total > 0 and completed >= total:
_set_full_scan_done(redis_client, True)
always_full_scan = bool(Settings().discovery.always_full_scan)
_set_full_scan_done(redis_client, False if always_full_scan else True)
_clear_sync_checkpoint(redis_client)
_clear_segmented_scan_active(redis_client)
_clear_bootstrap_failure_streak(redis_client)
_clear_bootstrap_continuation_streak(redis_client)
if always_full_scan:
logger.warning("All %d segments completed; next hourly run will restart from segment 0", total)
else:
logger.warning("All %d segments completed; bootstrap full scan done", total)
return {
@@ -1075,20 +1199,20 @@ def sync_segment_task(
@shared_task(
name="iaai_scraper.worker.tasks.sync_vehicle_task",
name="dubizzle_scraper.worker.tasks.sync_vehicle_task",
bind=True,
max_retries=2,
default_retry_delay=30,
acks_late=True,
)
def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
def sync_vehicle_task(self, vehicle_url: str, lane: str = "dubizzle"):
# Скрапинг и upsert одного автомобиля.
persistence = _get_persistence()
persistence.create_tables()
try:
def _job():
with IAAIScraper() as scraper:
with DUBIZZLEScraper() as scraper:
return scraper.sync_vehicle(vehicle_url, lane=lane)
result = _run_browser_job(_job)
@@ -1106,7 +1230,7 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
@shared_task(
name="iaai_scraper.worker.tasks.sync_listing_task",
name="dubizzle_scraper.worker.tasks.sync_listing_task",
bind=True,
max_retries=3,
default_retry_delay=120,
@@ -1116,7 +1240,7 @@ def sync_listing_task(
self,
make: str | None = None,
model: str | None = None,
lane: str = "iaai_cars",
lane: str = "dubizzle_cars",
limit: int | None = None,
only_new: bool | None = None,
):
@@ -1159,11 +1283,12 @@ def sync_listing_task(
continuation_streak, should_continue = _bump_bootstrap_continuation_streak(redis_client)
if not should_continue:
_clear_followup_pending(redis_client)
# Переключаемся на часовой beat-режим и останавливаем
# немедленные bootstrap continuation, чтобы не зациклиться.
if not always_full_scan:
# Останавливаем немедленные bootstrap continuation, чтобы не зациклиться.
# Фиксируем done + чистим checkpoint даже при always_full_scan:
# это безопасно и предотвращает повторный старт со stale-сегмента.
_set_full_scan_done(redis_client, True)
_clear_sync_checkpoint(redis_client)
if not always_full_scan:
logger.error(
"Bootstrap continuation stopped after %d immediate runs; switching to hourly schedule",
continuation_streak,
@@ -1178,7 +1303,7 @@ def sync_listing_task(
try:
followup_expires = max(int(lock_ttl), int(delay_seconds) + 300)
self.app.send_task(
"iaai_scraper.worker.tasks.sync_listing_task",
"dubizzle_scraper.worker.tasks.sync_listing_task",
kwargs={
"make": make,
"model": model,
@@ -1247,17 +1372,15 @@ def sync_listing_task(
)
full_scan_done_before_run = _is_full_scan_done(redis_client)
hourly_mode = settings.discovery.hourly_mode.strip().lower()
discovery_mode = settings.discovery.mode.strip().lower()
always_full_scan = bool(settings.discovery.always_full_scan)
force_bootstrap_full_scan = always_full_scan or (not full_scan_done_before_run)
effective_limit = None if force_bootstrap_full_scan else limit
effective_only_new = False if force_bootstrap_full_scan else only_new
hourly_mode = settings.discovery.hourly_mode.strip().lower()
discovery_mode = settings.discovery.mode.strip().lower()
if always_full_scan:
# Для режима "полный прогон каждый запуск" приоритет — устойчивый resume,
# поэтому принудительно уходим в listing/segmented path вместо sitemap-mainline.
discovery_mode = "listing"
prefer_sitemap_mainline = (
discovery_mode == "sitemap"
and
not force_bootstrap_full_scan
and make is None
and model is None
@@ -1268,22 +1391,23 @@ def sync_listing_task(
use_hourly_sitemap_sync = (not always_full_scan) and full_scan_done_before_run and prefer_sitemap_mainline
# Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
# Используется только во время bootstrap для пропуска уже обработанных сегментов.
# Никаких page-level resume — внутри сегмента всегда стартуем с page 1.
# Используется во время bootstrap/full-scan resume.
last_completed_segment: int | None = None
if force_bootstrap_full_scan:
resume_from_checkpoint = force_bootstrap_full_scan and (
always_full_scan or (not full_scan_done_before_run)
)
if resume_from_checkpoint:
last_completed_segment = _load_last_completed_segment(redis_client)
else:
# После завершения bootstrap чекпоинт не нужен никогда.
_clear_sync_checkpoint(redis_client)
if force_bootstrap_full_scan:
logger.info(
"Bootstrap mode: forcing full scan (only_new=False, limit=None) until first complete run",
)
if always_full_scan:
if discovery_mode == "algolia":
logger.info(
"Always full scan mode enabled (IAAI_ALWAYS_FULL_SCAN=true): using listing resume path",
"Algolia full scan enabled: using segmented Algolia traversal when configured",
)
logger.info(
@@ -1413,28 +1537,31 @@ def sync_listing_task(
self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
# Определяем сегменты из конфига.
segments = parse_listing_segments(settings.listing.listing_segments_json)
use_segmented = (
bool(segments)
and make is None
and model is None
and not prefer_sitemap_mainline
and discovery_mode != "sitemap"
and discovery_mode in {"listing", "algolia"}
)
if prefer_sitemap_mainline:
if discovery_mode != "sitemap":
logger.warning(
"Unfiltered full scan forcing sitemap discovery despite IAAI_DISCOVERY_MODE=%s",
discovery_mode or "unset",
)
if segments:
if prefer_sitemap_mainline and segments:
logger.info("Ignoring configured listing segments for unfiltered sitemap full scan")
# --- Параллельный диспатч сегментов: dispatch & exit ---
if use_segmented and settings.celery.parallel_segments:
# Сегменты уже завершённые (для bootstrap resume) пропускаем по Redis SET.
segmented_scan_in_progress = False
if force_bootstrap_full_scan:
segmented_scan_in_progress = _is_segmented_scan_in_progress(redis_client, self.app)
if segmented_scan_in_progress:
logger.warning("Parallel segments: scan already in progress, skipping duplicate dispatch")
return {
"status": "skipped",
"reason": "segmented_scan_in_progress",
"task_id": task_id,
"mode": "parallel_segments",
}
already_completed: set[int] = set()
if force_bootstrap_full_scan:
try:
@@ -1443,17 +1570,20 @@ def sync_listing_task(
except Exception:
already_completed = set()
if always_full_scan:
already_completed = set()
pending = [
(idx, seg) for idx, seg in enumerate(segments)
if idx not in already_completed
]
if not pending:
# Всё уже сделано — фиксируем bootstrap done.
if force_bootstrap_full_scan:
_set_full_scan_done(redis_client, True)
_clear_sync_checkpoint(redis_client)
_clear_bootstrap_failure_streak(redis_client)
_clear_segmented_scan_active(redis_client)
logger.warning("Parallel segments: nothing to dispatch (all completed)")
return {
"status": "success",
@@ -1464,15 +1594,18 @@ def sync_listing_task(
"segments_already_completed": len(already_completed),
}
# При первом запуске bootstrap фиксируем total, чтобы знать когда остановиться.
if force_bootstrap_full_scan and not already_completed:
if force_bootstrap_full_scan and (always_full_scan or not already_completed):
_reset_segments_progress(redis_client, len(segments))
already_completed = set()
if force_bootstrap_full_scan:
_set_segmented_scan_active(redis_client, total=len(segments))
dispatched = 0
for idx, seg in pending:
try:
self.app.send_task(
"iaai_scraper.worker.tasks.sync_segment_task",
"dubizzle_scraper.worker.tasks.sync_segment_task",
kwargs={
"segment_index": idx,
"segment": seg,
@@ -1498,13 +1631,11 @@ def sync_listing_task(
"segments_dispatched": dispatched,
"segments_already_completed": len(already_completed),
}
# --- конец параллельной ветки ---
resume_from_segment = 0
if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None:
resume_from_segment = max(0, last_completed_segment + 1)
if resume_from_segment >= len(segments):
# Все сегменты уже пройдены — чекпоинт устарел, начинаем заново.
logger.info(
"Stored checkpoint segment=%d is beyond configured segments (%d); restarting bootstrap from segment 0",
last_completed_segment, len(segments),
@@ -1527,7 +1658,7 @@ def sync_listing_task(
)
def _job():
with IAAIScraper() as scraper:
with DUBIZZLEScraper() as scraper:
scraper.set_progress_callback(_progress_cb_main)
if use_segmented:
return scraper.sync_listing_segmented(
@@ -1538,7 +1669,7 @@ def sync_listing_task(
start_page=1,
progress_callback=(
(lambda seg_idx: _save_last_completed_segment(redis_client, seg_idx))
if force_bootstrap_full_scan else None
if resume_from_checkpoint else None
),
)
return scraper.sync_listing(
@@ -1654,7 +1785,7 @@ def sync_listing_task(
if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl):
try:
self.app.send_task(
"iaai_scraper.worker.tasks.sync_listing_task",
"dubizzle_scraper.worker.tasks.sync_listing_task",
kwargs={
"make": make,
"model": model,

20
entrypoint.sh
View File

@@ -4,7 +4,7 @@ set -euo pipefail
is_worker_command() {
local joined="$*"
case "$joined" in
*"celery -A iaai_scraper.worker.celery_app worker"*|*" celery -A iaai_scraper.worker.celery_app worker"*)
*"celery -A dubizzle_scraper.worker.celery_app worker"*|*" celery -A dubizzle_scraper.worker.celery_app worker"*)
return 0
;;
esac
@@ -36,19 +36,19 @@ start_proxy_bridge_if_needed() {
echo "[entrypoint] Starting proxy bridge (HTTP :8899 → SOCKS5 ${SOCKS5_PROXY_HOST}:${SOCKS5_PROXY_PORT:-1002})..."
if [ -z "${IAAI_PROXY_SERVER:-}" ]; then
export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
elif [ "${IAAI_PROXY_SERVER}" = "http://localhost:8899" ]; then
export IAAI_PROXY_SERVER="http://127.0.0.1:8899"
if [ -z "${DUBIZZLE_PROXY_SERVER:-}" ]; then
export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899"
elif [ "${DUBIZZLE_PROXY_SERVER}" = "http://localhost:8899" ]; then
export DUBIZZLE_PROXY_SERVER="http://127.0.0.1:8899"
fi
echo "[entrypoint] Using browser proxy: ${IAAI_PROXY_SERVER}"
python -m iaai_scraper.proxy_bridge &
echo "[entrypoint] Using browser proxy: ${DUBIZZLE_PROXY_SERVER}"
python -m dubizzle_scraper.proxy_bridge &
BRIDGE_PID=$!
sleep 1
if ! kill -0 "${BRIDGE_PID}" 2>/dev/null; then
echo "[entrypoint] ERROR: iaai_scraper.proxy_bridge failed to start"
echo "[entrypoint] ERROR: dubizzle_scraper.proxy_bridge failed to start"
exit 1
fi
@@ -65,13 +65,13 @@ start_self_heal_watchdog_if_worker() {
# Удаляем его перед запуском worker-процесса.
rm -f /tmp/celery-worker.pid
if [ "${IAAI_SELF_HEAL_ENABLED:-true}" = "false" ]; then
if [ "${DUBIZZLE_SELF_HEAL_ENABLED:-true}" = "false" ]; then
echo "[entrypoint] Self-heal watchdog disabled"
return 0
fi
echo "[entrypoint] Starting self-heal watchdog for worker..."
python -m iaai_scraper.worker.self_heal &
python -m dubizzle_scraper.worker.self_heal &
SELF_HEAL_PID=$!
sleep 1

308
iaai_scraper/core/config.py
View File

@@ -1,308 +0,0 @@
import json
import os
from dataclasses import dataclass, field
from pathlib import Path
from dotenv import load_dotenv
load_dotenv()
TRUE_VALUES = {"1", "true", "yes", "on"}
# Хелперы для чтения env-переменных с приведением типов
def _env_str(name: str, default: str) -> str:
value = os.getenv(name)
return value if value is not None else default
def _env_optional_str(name: str) -> str | None:
value = os.getenv(name)
if value is None:
return None
value = value.strip()
return value or None
def _env_path_str(name: str) -> str | None:
value = _env_optional_str(name)
if value is None:
return None
return str(Path(value).expanduser())
def _env_bool(name: str, default: bool) -> bool:
fallback = "true" if default else "false"
return _env_str(name, fallback).strip().lower() in TRUE_VALUES
def _env_int(name: str, default: int) -> int:
return int(_env_str(name, str(default)).strip())
def _env_float(name: str, default: float) -> float:
return float(_env_str(name, str(default)).strip())
# Конфиг браузерного отпечатка (User-Agent, viewport, timezone)
@dataclass(slots=True)
class FingerprintConfig:
user_agent: str = (
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
"AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/135.0.0.0 Safari/537.36"
)
viewport_presets: tuple[dict[str, int], ...] = (
{"width": 1920, "height": 1080},
{"width": 1600, "height": 900},
{"width": 1536, "height": 864},
{"width": 1440, "height": 900},
{"width": 1366, "height": 768},
)
timezone_candidates: tuple[str, ...] = (
"America/New_York",
"America/Chicago",
"America/Los_Angeles",
)
locale: str = "en-US"
sec_ch_ua: str = '"Google Chrome";v="135", "Chromium";v="135", "Not.A/Brand";v="24"'
# Конфиг перехвата сетевых запросов (лимиты на кол-во)
@dataclass(slots=True)
class CaptureConfig:
capture_same_origin_only: bool = _env_bool("IAAI_CAPTURE_SAME_ORIGIN_ONLY", True)
max_requests: int = _env_int("IAAI_MAX_CAPTURED_REQUESTS", 40)
max_json_responses: int = _env_int("IAAI_MAX_CAPTURED_JSON_RESPONSES", 30)
# Конфиг пауз между действиями (имитация человека)
@dataclass(slots=True)
class HumanPaceConfig:
enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
# Конфиг сбора листинга (URL, лимиты страниц и машин)
@dataclass(slots=True)
class ListingConfig:
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
# Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
# JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
# Пустая строка = без сегментации (backward compatible).
listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
# Список брендов IAAI для автоматической сегментации.
# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
IAAI_DEFAULT_MAKES: tuple[str, ...] = (
"TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
"KIA", "DODGE", "JEEP", "BMW", "MERCEDES-BENZ", "SUBARU",
"VOLKSWAGEN", "GMC", "MAZDA", "LEXUS", "CHRYSLER", "AUDI",
"RAM", "BUICK", "CADILLAC", "ACURA", "INFINITI", "LINCOLN",
"MITSUBISHI", "VOLVO", "JAGUAR", "LAND ROVER", "PORSCHE",
"MINI", "TESLA", "GENESIS", "FIAT", "ALFA ROMEO", "MASERATI",
"SCION", "PONTIAC", "SATURN", "MERCURY", "SAAB", "SUZUKI",
"OLDSMOBILE", "ISUZU", "HUMMER", "PLYMOUTH", "SMART",
"RIVIAN", "LUCID", "POLESTAR", "FERRARI", "LAMBORGHINI",
"BENTLEY", "ROLLS-ROYCE", "ASTON MARTIN", "MCLAREN", "LOTUS",
"MAYBACH", "FISKER", "GEO", "DAEWOO", "EAGLE",
)
# Пагинационный потолок IAAI: ~226 страниц × 100 = 22 600 результатов.
IAAI_PAGINATION_CEILING = 22_600
# Бренды, потенциально превышающие потолок пагинации — разбиваем по годам.
_LARGE_MAKES: frozenset[str] = frozenset({
"TOYOTA", "FORD", "CHEVROLET", "HONDA", "NISSAN", "HYUNDAI",
"KIA", "DODGE", "JEEP",
})
_YEAR_SPLITS: tuple[tuple[int, int], ...] = (
(1900, 2012),
(2013, 2019),
(2020, 2027),
)
def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
"""Парсит IAAI_LISTING_SEGMENTS в список сегментов.
Каждый сегмент — dict с ключами: make (str), year_min/year_max (int|None).
Специальное значение ``"auto"`` генерирует сегменты из IAAI_DEFAULT_MAKES.
Крупные бренды автоматически разбиваются по диапазонам годов.
"""
raw = raw.strip()
if not raw:
return []
if raw.lower() == "auto":
segments: list[dict[str, str | int | None]] = []
for m in IAAI_DEFAULT_MAKES:
if m in _LARGE_MAKES:
for yr_min, yr_max in _YEAR_SPLITS:
segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
else:
segments.append({"make": m, "year_min": None, "year_max": None})
return segments
try:
data = json.loads(raw)
except (json.JSONDecodeError, ValueError):
return []
if not isinstance(data, list):
return []
segments = []
for item in data:
if isinstance(item, str):
segments.append({"make": item.upper(), "year_min": None, "year_max": None})
elif isinstance(item, dict):
segments.append({
"make": str(item.get("make") or "").upper() or None,
"year_min": int(item["year_min"]) if item.get("year_min") is not None else None,
"year_max": int(item["year_max"]) if item.get("year_max") is not None else None,
})
return segments
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
@dataclass(slots=True)
class DatabaseConfig:
url: str = _env_str("IAAI_DATABASE_URL", "postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper")
echo: bool = _env_bool("IAAI_DATABASE_ECHO", False)
pool_size: int = _env_int("IAAI_DATABASE_POOL_SIZE", 5)
max_overflow: int = _env_int("IAAI_DATABASE_MAX_OVERFLOW", 10)
pool_recycle_seconds: int = _env_int("IAAI_DATABASE_POOL_RECYCLE_SECONDS", 1800)
auto_create_tables: bool = _env_bool("IAAI_DATABASE_AUTO_CREATE_TABLES", False)
# --- Конфиг Redis (URL для Celery broker) ---
@dataclass(slots=True)
class RedisConfig:
url: str = _env_str("IAAI_REDIS_URL", "redis://localhost:6379/0")
socket_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_TIMEOUT_SECONDS", 10.0)
socket_connect_timeout_seconds: float = _env_float("IAAI_REDIS_SOCKET_CONNECT_TIMEOUT_SECONDS", 5.0)
health_check_interval_seconds: int = _env_int("IAAI_REDIS_HEALTH_CHECK_INTERVAL_SECONDS", 30)
# --- Конфиг Discovery (режим обнаружения, hourly batch) ---
@dataclass(slots=True)
class DiscoveryConfig:
mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", True)
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@dataclass(slots=True)
class CeleryConfig:
broker_url: str = _env_str("CELERY_BROKER_URL", "")
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
# --- Конфиг прокси (server, username, password) ---
@dataclass(slots=True)
class ProxyConfig:
server: str | None = _env_optional_str("IAAI_PROXY_SERVER")
username: str | None = _env_optional_str("IAAI_PROXY_USERNAME")
password: str | None = _env_optional_str("IAAI_PROXY_PASSWORD")
@property
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
if self.username:
result["username"] = self.username
if self.password:
result["password"] = self.password
return result
# Главный объект настроек: собирает все блоки конфигурации
@dataclass(slots=True)
class Settings:
home_url: str = "https://www.iaai.com/"
default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
headless: bool = _env_bool("IAAI_HEADLESS", True)
browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
sync_only_new: bool = _env_bool("IAAI_SYNC_ONLY_NEW", False)
raw_output_json: str | None = _env_optional_str("IAAI_RAW_OUTPUT_JSON")
tokens_file: str | None = _env_path_str("IAAI_TOKENS_FILE")
runtime_config_file: str | None = _env_path_str("IAAI_RUNTIME_CONFIG_FILE")
scheduler_interval_minutes: int = _env_int("IAAI_SCHEDULER_INTERVAL_MINUTES", 60)
fingerprint: FingerprintConfig = field(default_factory=FingerprintConfig)
capture: CaptureConfig = field(default_factory=CaptureConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
@property
def parallel_tabs(self) -> int:
return self.celery.parallel_tabs
@property
def block_resources(self) -> bool:
return self.celery.block_resources
# Глобальный синглтон — используется по умолчанию во всех модулях.
settings = Settings()

8
pyproject.toml
View File

@@ -3,9 +3,9 @@ requires = ["setuptools>=68", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "iaai-scraper"
name = "dubizzle-scraper"
version = "0.1.0"
description = "IAAI scraper service with FastAPI, Celery, Playwright and PostgreSQL"
description = "Dubizzle scraper service with FastAPI, Celery, Playwright and PostgreSQL"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
@@ -29,13 +29,13 @@ dev = [
]
[project.scripts]
iaai = "iaai_scraper.cli:main"
dubizzle = "dubizzle_scraper.cli:main"
[tool.setuptools]
include-package-data = true
[tool.setuptools.packages.find]
include = ["iaai_scraper*"]
include = ["dubizzle_scraper*"]
[tool.pytest.ini_options]
addopts = "-q --disable-warnings"

2
runtime_config.json
View File

@@ -5,7 +5,7 @@
"ids_next_size": null,
"ids_max_pages": null,
"condition_check_enabled": false,
"lane": "iaai_cars",
"lane": "dubizzle_cars",
"only_new": false,
"limit": null
},

24
tests/test_db.py
View File

@@ -6,10 +6,10 @@ from pathlib import Path
from sqlalchemy import select
from iaai_scraper.core.config import Settings
from iaai_scraper.storage.db import PersistenceService
from iaai_scraper.storage.models import Car, Image, SyncRun
from iaai_scraper.storage.schemas import CarRecord, ImageRecord
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.storage.db import PersistenceService
from dubizzle_scraper.storage.models import Car, Image, SyncRun
from dubizzle_scraper.storage.schemas import CarRecord, ImageRecord
class TestPersistenceServiceIntegration(unittest.TestCase):
@@ -31,18 +31,18 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
@staticmethod
def _record(origin_id: str, *, price: int = 1000) -> CarRecord:
return CarRecord(
parser_id=f"iaai:{origin_id}",
parser_id=f"dubizzle:{origin_id}",
brand="Toyota",
model="Camry",
year=2014,
price=price,
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id,
slug=f"toyota-camry-{origin_id}",
images=[
ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=1&width=400&height=300",
fullres_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
preview_image="https://vis.dubizzle.com/resizer?imageKeys=1&width=400&height=300",
order_index=0,
)
],
@@ -79,8 +79,8 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
second = self._record("888")
second.images = [
ImageRecord(
fullres_image="https://vis.iaai.com/resizer?imageKeys=2&width=845&height=633",
preview_image="https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
fullres_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=845&height=633",
preview_image="https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
order_index=0,
)
]
@@ -108,11 +108,11 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
def test_upsert_falls_back_to_origin_url_to_prevent_duplicates(self) -> None:
first = self._record("OLD-ID")
first.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
first.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
self.persistence.upsert_car(first)
second = self._record("NEW-ID")
second.origin_url = "https://www.iaai.com/VehicleDetail/45089484~US"
second.origin_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
result = self.persistence.upsert_car(second)
self.assertEqual(result["action"], "updated")

10
tests/test_listing.py
View File

@@ -2,9 +2,9 @@ from __future__ import annotations
import unittest
from iaai_scraper.browser.pace import HumanPacer
from iaai_scraper.core.config import Settings
from iaai_scraper.browser.listing import ListingCollector
from dubizzle_scraper.browser.pace import HumanPacer
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.browser.listing import ListingCollector
class _FakePage:
@@ -58,8 +58,8 @@ class TestListingUnit(unittest.TestCase):
self.assertEqual(
links,
[
("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
("https://www.dubizzle.com/VehicleDetail/45184893~US", "45184893"),
("https://www.dubizzle.com/VehicleDetail/45171480~US", "45171480"),
],
)

107
tests/test_mappers.py
View File

@@ -2,22 +2,109 @@ from __future__ import annotations
import unittest
from iaai_scraper.parsing.mapper import CarMapper
from dubizzle_scraper.parsing.mapper import CarMapper
class TestCarMapper(unittest.TestCase):
def setUp(self) -> None:
self.mapper = CarMapper()
def test_extracts_make_model_from_algolia_details(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://dubizzle.com/s/DOBI7J3",
vehicle_summary={
"id": 16810399,
"details_v2": {
"tertiary": [
{"slug": "make", "value": {"en": "Toyota", "ar": "تويوتا"}},
{"slug": "model", "value": {"en": "Land Cruiser", "ar": "لاند كروزر"}},
]
},
"category_v2": {
"names_en": ["Motors", "Used Cars", "Toyota", "Land Cruiser"],
"slug_paths": [
"motors",
"motors/used-cars",
"motors/used-cars/toyota",
"motors/used-cars/toyota/land-cruiser",
],
},
},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.brand, "Toyota")
self.assertEqual(record.model, "Land Cruiser")
def test_extracts_make_model_from_legacy_details(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://dubizzle.com/s/DOBIlegacy",
vehicle_summary={
"objectID": "item:legacy:1",
"details": {
"Make": {"en": {"label": "Make", "value": "Honda"}},
"Model": {"en": {"label": "Model", "value": "Civic"}},
},
},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.brand, "Honda")
self.assertEqual(record.model, "Civic")
def test_extracts_extended_algolia_fields(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://dubizzle.com/s/DOBGaGs",
vehicle_summary={
"id": 16345114,
"year": 2012,
"price": 59000.0,
"kilometers": 91800,
"seller_type": "DL",
"site": {"en": "Dubai"},
"category_v2": {
"names_en": ["Motors", "Used Cars", "Porsche", "Cayenne"],
"slug_paths": [
"motors",
"motors/used-cars",
"motors/used-cars/porsche",
"motors/used-cars/porsche/cayenne",
],
},
"details": {
"Body Type": {"en": {"value": "SUV"}},
"Transmission Type": {"en": {"value": "Automatic Transmission"}},
"Exterior Color": {"en": {"value": "Brown"}},
"Steering Side": {"en": {"value": "Left Hand"}},
"Engine Capacity (cc)": {"en": {"value": "4800 cc"}},
"Make": {"en": {"value": "Porsche"}},
"Model": {"en": {"value": "Cayenne"}},
},
"photo_mains": ["https://dbz-images.dubizzle.com/images/example.jpeg?impolicy=dpv"],
},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
self.assertEqual(record.brand, "Porsche")
self.assertEqual(record.model, "Cayenne")
self.assertEqual(record.price, 59000)
self.assertEqual(record.mileage, 91800)
self.assertEqual(record.color, "brown")
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.gearbox, "AT")
self.assertEqual(record.steering_wheel, "LEFT")
self.assertEqual(record.engine_volume, 4800)
self.assertEqual(record.country, "AE")
def test_deduplicates_images_by_image_key(self) -> None:
urls = [
"https://vis.iaai.com/resizer?imageKeys=1&width=200&height=150",
"https://vis.iaai.com/resizer?imageKeys=1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=2&width=400&height=300",
"https://vis.dubizzle.com/resizer?imageKeys=1&width=200&height=150",
"https://vis.dubizzle.com/resizer?imageKeys=1&width=845&height=633",
"https://vis.dubizzle.com/resizer?imageKeys=2&width=400&height=300",
]
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
vehicle_summary={"make": "Honda", "model": "Civic", "image_urls": urls},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
@@ -28,7 +115,7 @@ class TestCarMapper(unittest.TestCase):
def test_no_damage_marker_is_not_damaged(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/123~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/123~US",
vehicle_summary={"make": "Ford", "model": "Focus"},
payload_insights={
"vehicle_core": {},
@@ -43,7 +130,7 @@ class TestCarMapper(unittest.TestCase):
def test_unknown_empty_values_and_normalization(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/999~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/999~US",
vehicle_summary={"make": " ", "model": None, "drive": "???", "gearbox": "unknown"},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
@@ -54,7 +141,7 @@ class TestCarMapper(unittest.TestCase):
# Нормализация регистра и пробелов.
record2 = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/888~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/888~US",
vehicle_summary={"make": "Honda", "model": "Civic", "drive": " Front Wheel Drive ", "gearbox": " AUTOMATIC "},
payload_insights={"vehicle_core": {}, "pricing": {}, "damage": {}, "auction": {}, "images": {}},
)
@@ -63,7 +150,7 @@ class TestCarMapper(unittest.TestCase):
def test_price_and_currency_parsing(self) -> None:
record = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/777~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/777~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={
"vehicle_core": {},
@@ -76,7 +163,7 @@ class TestCarMapper(unittest.TestCase):
self.assertEqual(record.price, 5200)
record2 = self.mapper.map_to_car_record(
vehicle_url="https://www.iaai.com/VehicleDetail/778~US",
vehicle_url="https://www.dubizzle.com/VehicleDetail/778~US",
vehicle_summary={"make": "Toyota", "model": "Corolla"},
payload_insights={
"vehicle_core": {},

10
tests/test_parser.py
View File

@@ -2,7 +2,7 @@ from __future__ import annotations
import unittest
from iaai_scraper.parsing.parser import VehicleParser
from dubizzle_scraper.parsing.parser import VehicleParser
class TestVehicleParserUnit(unittest.TestCase):
@@ -29,11 +29,11 @@ class TestVehicleParserUnit(unittest.TestCase):
self.assertEqual(parsed["model"], "CAMRY")
def test_extract_image_urls_filters_and_deduplicates(self) -> None:
vehicle_url = "https://www.iaai.com/VehicleDetail/45089484~US"
vehicle_url = "https://www.dubizzle.com/VehicleDetail/45089484~US"
payloads = [{"imageUrls": [
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.iaai.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
"https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.dubizzle.com/resizer?imageKeys=45089484~SID1&width=845&height=633",
"https://vis.dubizzle.com/resizer?imageKeys=99999999~SID2&width=845&height=633",
]}]
urls = self.parser._extract_image_urls(payloads, "", vehicle_url)
self.assertEqual(len(urls), 1)

14
tests/test_resilience.py
View File

@@ -4,17 +4,17 @@ import unittest
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.core.config import Settings
from iaai_scraper.worker import tasks
from dubizzle_scraper.scraper import DUBIZZLEScraper
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.worker import tasks
class TestResilience(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper:
def _make_scraper(self) -> DUBIZZLEScraper:
s = Settings()
s.log_level = "CRITICAL"
s.database.url = "sqlite://"
return IAAIScraper(s)
return DUBIZZLEScraper(s)
def test_update_task_progress_updates_global_marker(self) -> None:
redis_client = MagicMock()
@@ -60,11 +60,11 @@ class TestResilience(unittest.TestCase):
result = scraper._sync_listing_streaming(
make=None,
model=None,
lane="iaai_cars",
lane="dubizzle_cars",
limit=None,
effective_only_new=False,
started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TEST",
listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TEST",
)
self.assertEqual(result["total"], 0)

76
tests/test_scraper.py
View File

@@ -5,29 +5,29 @@ from concurrent.futures import TimeoutError as FuturesTimeoutError
from types import SimpleNamespace
from unittest.mock import MagicMock, patch
from iaai_scraper.core.config import Settings
from iaai_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from iaai_scraper.scraper import IAAIScraper
from iaai_scraper.storage.schemas import CarRecord
from dubizzle_scraper.core.config import Settings
from dubizzle_scraper.core.exceptions import AntiBotDetectedError, SiteStructureChangedError
from dubizzle_scraper.scraper import DUBIZZLEScraper
from dubizzle_scraper.storage.schemas import CarRecord
def make_db_record(origin_id: str) -> dict[str, object]:
return CarRecord(
parser_id=f"iaai:{origin_id}",
parser_id=f"dubizzle:{origin_id}",
brand="Toyota",
model="Camry",
origin_url=f"https://www.iaai.com/VehicleDetail/{origin_id}~US",
origin_url=f"https://www.dubizzle.com/VehicleDetail/{origin_id}~US",
origin_id=origin_id,
slug=f"toyota-camry-{origin_id}",
).model_dump(mode="json")
class TestScraperSync(unittest.TestCase):
def _make_scraper(self) -> IAAIScraper:
def _make_scraper(self) -> DUBIZZLEScraper:
s = Settings()
s.log_level = "CRITICAL"
s.database.url = "sqlite://"
return IAAIScraper(s)
return DUBIZZLEScraper(s)
def test_sync_vehicle_uses_db_record(self) -> None:
scraper = self._make_scraper()
@@ -37,7 +37,7 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.upsert_car = MagicMock(return_value={"action": "inserted", "images_upserted": 0})
scraper.scrape_vehicle_detail = MagicMock(return_value={"db_record": make_db_record("111")})
result = scraper.sync_vehicle("https://www.iaai.com/VehicleDetail/111~US")
result = scraper.sync_vehicle("https://www.dubizzle.com/VehicleDetail/111~US")
self.assertEqual(result["status"], "success")
self.assertIn("trace_id", result)
@@ -50,13 +50,13 @@ class TestScraperSync(unittest.TestCase):
scraper.persistence.start_sync_run = MagicMock(return_value=2)
scraper.persistence.finish_sync_run = MagicMock()
scraper.persistence.get_existing_urls_and_ids = MagicMock(return_value=(
{"https://www.iaai.com/VehicleDetail/111~US"}, {"iaai:222"},
{"https://www.dubizzle.com/VehicleDetail/111~US"}, {"dubizzle:222"},
))
scraper.collect_listing = MagicMock(return_value={
"vehicle_urls": [
"https://www.iaai.com/VehicleDetail/111~US",
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
"https://www.dubizzle.com/VehicleDetail/111~US",
"https://www.dubizzle.com/VehicleDetail/222~US",
"https://www.dubizzle.com/VehicleDetail/333~US",
]
})
scraper.sync_batch = MagicMock(return_value={
@@ -79,7 +79,7 @@ class TestScraperSync(unittest.TestCase):
})
result2 = scraper2.sync_listing(
only_new=True,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
year_min=2020, year_max=2027,
)
self.assertEqual(result2["cars_upserted"], 10)
@@ -155,35 +155,35 @@ class TestScraperSync(unittest.TestCase):
self.assertEqual(result["status"], "partial_success")
def test_build_segment_listing_url(self) -> None:
base = "https://www.iaai.com/Vehiclelisting/Cars"
base = "https://www.dubizzle.com/Vehiclelisting/Cars"
self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "TOYOTA"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=TOYOTA",
DUBIZZLEScraper._build_segment_listing_url(base, "TOYOTA"),
"https://www.dubizzle.com/Vehiclelisting/Cars?Make=TOYOTA",
)
self.assertEqual(
IAAIScraper._build_segment_listing_url(base, "LAND ROVER"),
"https://www.iaai.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
DUBIZZLEScraper._build_segment_listing_url(base, "LAND ROVER"),
"https://www.dubizzle.com/Vehiclelisting/Cars?Make=LAND%20ROVER",
)
self.assertEqual(IAAIScraper._build_segment_listing_url(base, None), base)
self.assertEqual(IAAIScraper._build_segment_listing_url(base, ""), base)
self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, None), base)
self.assertEqual(DUBIZZLEScraper._build_segment_listing_url(base, ""), base)
def test_guard_and_protection_detection(self) -> None:
with self.assertRaises(AntiBotDetectedError):
IAAIScraper._raise_if_blocked_or_incomplete(
DUBIZZLEScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": True, "has_antibot_text": False},
"access_notes": {}, "vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US",
"https://www.dubizzle.com/VehicleDetail/999~US",
)
with self.assertRaises(SiteStructureChangedError):
IAAIScraper._raise_if_blocked_or_incomplete(
DUBIZZLEScraper._raise_if_blocked_or_incomplete(
{"dom_hints": {"has_captcha_text": False, "has_antibot_text": False},
"access_notes": {"possible_captcha": False, "possible_antibot": False},
"vehicle_summary": {}},
"https://www.iaai.com/VehicleDetail/999~US",
"https://www.dubizzle.com/VehicleDetail/999~US",
)
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
self.assertTrue(IAAIScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
self.assertFalse(IAAIScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("NS_ERROR_NET_INTERRUPT")))
self.assertTrue(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("captcha challenge")))
self.assertFalse(DUBIZZLEScraper._is_protection_or_network_error(RuntimeError("plain validation error")))
def test_close_resets_browser_state(self) -> None:
scraper = self._make_scraper()
@@ -201,7 +201,7 @@ class TestScraperSync(unittest.TestCase):
scraper = self._make_scraper()
page = MagicMock()
page_result = SimpleNamespace(
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/123~US")],
vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/123~US")],
)
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper.listing_collector.open_cars_listing = MagicMock()
@@ -228,7 +228,7 @@ class TestScraperSync(unittest.TestCase):
page = MagicMock()
page_result = SimpleNamespace(
page_number=1,
vehicle_links=[SimpleNamespace(href="https://www.iaai.com/VehicleDetail/999~US", lot_number="999")],
vehicle_links=[SimpleNamespace(href="https://www.dubizzle.com/VehicleDetail/999~US", lot_number="999")],
next_page_detected=False,
)
@@ -245,7 +245,7 @@ class TestScraperSync(unittest.TestCase):
"year_max": None,
})
scraper.listing_collector.collect_current_page = MagicMock(return_value=page_result)
scraper._extract_page_urls = MagicMock(return_value=["https://www.iaai.com/VehicleDetail/999~US"])
scraper._extract_page_urls = MagicMock(return_value=["https://www.dubizzle.com/VehicleDetail/999~US"])
scraper.sync_batch = MagicMock(return_value={
"cars_upserted": 1,
"cars_failed": 0,
@@ -256,11 +256,11 @@ class TestScraperSync(unittest.TestCase):
result = scraper._sync_listing_streaming(
make=None,
model=None,
lane="iaai_cars",
lane="dubizzle_cars",
limit=None,
effective_only_new=False,
started_at=0.0,
listing_url="https://www.iaai.com/Vehiclelisting/Cars?Make=EAGLE",
listing_url="https://www.dubizzle.com/Vehiclelisting/Cars?Make=EAGLE",
)
scraper.listing_collector.open_cars_listing.assert_called_once()
@@ -278,9 +278,9 @@ class TestScraperSync(unittest.TestCase):
})
urls = [
"https://www.iaai.com/VehicleDetail/111~US",
"https://www.iaai.com/VehicleDetail/222~US",
"https://www.iaai.com/VehicleDetail/333~US",
"https://www.dubizzle.com/VehicleDetail/111~US",
"https://www.dubizzle.com/VehicleDetail/222~US",
"https://www.dubizzle.com/VehicleDetail/333~US",
]
first_record = CarRecord.model_validate(make_db_record("111"))
@@ -298,8 +298,8 @@ class TestScraperSync(unittest.TestCase):
yield 0, first_record
raise FuturesTimeoutError()
with patch("iaai_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
patch("iaai_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
with patch("dubizzle_scraper.core.runtime_config.RuntimeFiltersConfig.is_empty", return_value=True), \
patch("dubizzle_scraper.scraper.ThreadPoolExecutor", _FakeExecutor), \
patch.object(scraper, "_browser_fallback_parallel", return_value={
"records": [],
"failures": [],

30
tests/test_self_heal.py
View File

@@ -3,7 +3,7 @@ from __future__ import annotations
import unittest
from unittest.mock import MagicMock, patch
from iaai_scraper.worker import self_heal
from dubizzle_scraper.worker import self_heal
class TestSelfHeal(unittest.TestCase):
@@ -20,14 +20,14 @@ class TestSelfHeal(unittest.TestCase):
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: {
self_heal.GLOBAL_PROGRESS_TS_KEY: None,
"iaai:state:task_progress:a": '{"ts": 100}',
"iaai:state:task_progress:b": '{"ts": 250}',
"iaai:state:task_progress:c": '{"ts": 150}',
"dubizzle:state:task_progress:a": '{"ts": 100}',
"dubizzle:state:task_progress:b": '{"ts": 250}',
"dubizzle:state:task_progress:c": '{"ts": 150}',
}.get(key)
redis_client.scan_iter.return_value = [
"iaai:state:task_progress:a",
"iaai:state:task_progress:b",
"iaai:state:task_progress:c",
"dubizzle:state:task_progress:a",
"dubizzle:state:task_progress:b",
"dubizzle:state:task_progress:c",
]
ts = self_heal._read_last_progress_ts(redis_client)
@@ -38,20 +38,20 @@ class TestSelfHeal(unittest.TestCase):
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: {
self_heal.GLOBAL_PROGRESS_TS_KEY: None,
"iaai:state:task_progress:a": "{bad-json}",
"iaai:state:task_progress:b": '{"foo": "bar"}',
"dubizzle:state:task_progress:a": "{bad-json}",
"dubizzle:state:task_progress:b": '{"foo": "bar"}',
}.get(key)
redis_client.scan_iter.return_value = [
"iaai:state:task_progress:a",
"iaai:state:task_progress:b",
"dubizzle:state:task_progress:a",
"dubizzle:state:task_progress:b",
]
ts = self_heal._read_last_progress_ts(redis_client)
self.assertIsNone(ts)
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("iaai_scraper.worker.self_heal.os.kill")
@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("dubizzle_scraper.worker.self_heal.os.kill")
@patch("builtins.open")
def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
open_mock.return_value.__enter__.return_value.read.return_value = "123"
@@ -64,8 +64,8 @@ class TestSelfHeal(unittest.TestCase):
self.assertEqual(kill_mock.call_args_list[1].args, (123, 0))
self.assertEqual(kill_mock.call_args_list[2].args[0], 123)
@patch("iaai_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("iaai_scraper.worker.self_heal.os.kill")
@patch("dubizzle_scraper.worker.self_heal.time.sleep", return_value=None)
@patch("dubizzle_scraper.worker.self_heal.os.kill")
@patch("builtins.open")
def test_kill_worker_process_skips_sigkill_when_already_exited(self, open_mock, kill_mock, _sleep_mock) -> None:
open_mock.return_value.__enter__.return_value.read.return_value = "123"

33
tests/test_utils.py
View File

@@ -2,8 +2,8 @@ from __future__ import annotations
import unittest
from iaai_scraper.core.utils import deep_find_key
from iaai_scraper.core.config import parse_listing_segments, IAAI_DEFAULT_MAKES, _LARGE_MAKES, _YEAR_SPLITS
from dubizzle_scraper.core.utils import deep_find_key
from dubizzle_scraper.core.config import parse_listing_segments, _AUTO_YEAR_SPLITS
class TestDeepFindKey(unittest.TestCase):
@@ -28,32 +28,19 @@ class TestParseListingSegments(unittest.TestCase):
self.assertEqual(parse_listing_segments("invalid"), [])
def test_auto_segments_complete_coverage(self) -> None:
"""auto: все бренды покрыты, крупные разбиты по годам, годы без дыр."""
"""auto: все годовые диапазоны покрыты, без дыр и без make-фильтра."""
segs = parse_listing_segments("auto")
# Точное число сегментов.
expected = len(_LARGE_MAKES) * len(_YEAR_SPLITS) + (len(IAAI_DEFAULT_MAKES) - len(_LARGE_MAKES))
self.assertEqual(len(segs), expected)
# Все бренды из списка присутствуют.
makes_in_segments = {s["make"] for s in segs}
for make in IAAI_DEFAULT_MAKES:
self.assertIn(make, makes_in_segments)
# Крупные бренды разбиты на 3 сегмента с годами.
toyota = [s for s in segs if s["make"] == "TOYOTA"]
self.assertEqual(len(toyota), 3)
for s in toyota:
self.assertIsNotNone(s["year_min"])
# Мелкие бренды без годов.
lexus = [s for s in segs if s["make"] == "LEXUS"]
self.assertEqual(len(lexus), 1)
self.assertIsNone(lexus[0]["year_min"])
self.assertEqual(len(segs), len(_AUTO_YEAR_SPLITS))
self.assertEqual(
[(s["year_min"], s["year_max"]) for s in segs],
list(_AUTO_YEAR_SPLITS),
)
self.assertTrue(all(s["make"] is None for s in segs))
# Годовые диапазоны покрывают 1950-2027.
years = set()
for yr_min, yr_max in _YEAR_SPLITS:
for yr_min, yr_max in _AUTO_YEAR_SPLITS:
years.update(range(yr_min, yr_max + 1))
for year in range(1950, 2027):
self.assertIn(year, years)

151
tests/test_worker_tasks.py
View File

@@ -5,8 +5,8 @@ from dataclasses import replace
import unittest
from unittest.mock import MagicMock, patch
from iaai_scraper.worker import tasks
from iaai_scraper.core.config import settings as base_settings
from dubizzle_scraper.worker import tasks
from dubizzle_scraper.core.config import settings as base_settings
class TestWorkerTaskLockHelpers(unittest.TestCase):
@@ -187,13 +187,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(
base_settings.discovery,
always_full_scan=False,
mode="listing",
),
celery=replace(
base_settings.celery,
parallel_segments=False,
),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
@@ -211,7 +227,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-resume-seg")
try:
result = tasks.sync_listing_task.run()
@@ -228,6 +244,22 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(
base_settings.discovery,
always_full_scan=False,
mode="listing",
),
celery=replace(
base_settings.celery,
parallel_segments=False,
),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
@@ -235,7 +267,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_clear_sync_checkpoint") as clear_checkpoint, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
# Оставшийся чекпоинт не должен использоваться.
@@ -254,10 +286,10 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-792")
try:
result = tasks.sync_listing_task.run()
result = tasks.sync_listing_task.run(limit=1)
finally:
tasks.sync_listing_task.pop_request()
@@ -271,13 +303,29 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
segments = [{"make": "ACURA"}, {"make": "AUDI"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(
base_settings.discovery,
always_full_scan=False,
mode="listing",
),
celery=replace(
base_settings.celery,
parallel_segments=False,
),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
@@ -295,7 +343,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing_segmented = sync_segmented_mock
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-beyond")
try:
result = tasks.sync_listing_task.run()
@@ -507,7 +555,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_set_full_scan_done") as set_full_scan_done, \
patch.object(tasks.sync_listing_task, "update_state"), \
patch.object(tasks, "_run_browser_job") as run_job, \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=[]):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=[]):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.return_value = None
@@ -554,7 +602,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_run_browser_job", side_effect=lambda fn: fn()), \
patch.object(tasks.sync_listing_task, "update_state"), \
patch("iaai_scraper.worker.tasks.parse_listing_segments", return_value=segments):
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: (
@@ -579,7 +627,7 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
scraper_ctx.__enter__.return_value.sync_listing = MagicMock()
scraper_ctx.__exit__.return_value = None
with patch.object(tasks, "IAAIScraper", return_value=scraper_ctx):
with patch.object(tasks, "DUBIZZLEScraper", return_value=scraper_ctx):
tasks.sync_listing_task.push_request(id="task-always-full-resume")
try:
result = tasks.sync_listing_task.run()
@@ -591,6 +639,85 @@ class TestWorkerTaskLockHelpers(unittest.TestCase):
self.assertEqual(sync_segmented_mock.call_args.kwargs["start_page"], 1)
release_lock.assert_called_once()
def test_sync_listing_task_always_full_scan_resets_segment_progress_and_dispatches_again(self) -> None:
segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(base_settings.discovery, always_full_scan=True),
celery=replace(base_settings.celery, parallel_segments=True),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_is_segmented_scan_in_progress", return_value=False), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks, "_reset_segments_progress") as reset_progress, \
patch.object(tasks, "_set_segmented_scan_active") as set_active, \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
redis_client = MagicMock()
redis_client.smembers.return_value = {"0", "1"}
redis_client.get.return_value = None
get_redis.return_value = redis_client
start_heartbeat.return_value = (MagicMock(), MagicMock())
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
tasks.sync_listing_task.push_request(id="task-always-hourly")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "success")
self.assertEqual(result["segments_dispatched"], 2)
reset_progress.assert_called_once_with(redis_client, len(segments))
set_active.assert_called_once_with(redis_client, total=len(segments))
self.assertEqual(task_app.send_task.call_count, 2)
release_lock.assert_called_once()
def test_sync_listing_task_skips_duplicate_parallel_dispatch_while_segments_active(self) -> None:
segments = [{"make": "TOYOTA"}, {"make": "FORD"}]
with patch.object(tasks, "_get_persistence") as get_persistence, \
patch.object(tasks, "_get_redis") as get_redis, \
patch.object(
tasks,
"Settings",
return_value=replace(
base_settings,
discovery=replace(base_settings.discovery, always_full_scan=True),
celery=replace(base_settings.celery, parallel_segments=True),
),
), \
patch.object(tasks, "_acquire_lock", return_value=True), \
patch.object(tasks, "_is_full_scan_done", return_value=True), \
patch.object(tasks, "_is_segmented_scan_in_progress", return_value=True), \
patch.object(tasks, "_start_lock_heartbeat") as start_heartbeat, \
patch.object(tasks, "_release_lock_if_owner") as release_lock, \
patch.object(tasks.sync_listing_task, "update_state"):
get_persistence.return_value = MagicMock()
get_redis.return_value = MagicMock()
start_heartbeat.return_value = (MagicMock(), MagicMock())
with patch.object(tasks.sync_listing_task, "app", new=MagicMock()) as task_app, \
patch("dubizzle_scraper.worker.tasks.parse_listing_segments", return_value=segments):
tasks.sync_listing_task.push_request(id="task-always-skip-duplicate")
try:
result = tasks.sync_listing_task.run()
finally:
tasks.sync_listing_task.pop_request()
self.assertEqual(result["status"], "skipped")
self.assertEqual(result["reason"], "segmented_scan_in_progress")
task_app.send_task.assert_not_called()
release_lock.assert_called_once()
if __name__ == "__main__":
unittest.main()

2
uv.lock generated
View File

@@ -342,7 +342,7 @@ wheels = [
]
[[package]]
name = "iaai-scraper"
name = "dubizzle-scraper"
version = "0.1.0"
source = { editable = "." }
dependencies = [