Compare commits

...

8 Commits

Author SHA1 Message Date
qananasikq
2cebb78316 refresh api and tests 2026-05-11 12:46:07 +03:00
qananasikq
d9d073641b improve runtime sync flow 2026-05-11 12:45:51 +03:00
qananasikq
124dfc514f update infra and schema 2026-05-11 12:45:37 +03:00
qananasikq
5049a29ef7 Update setup 2026-05-06 21:07:24 +03:00
qananasikq
519686c0b7 Track sold cars 2026-05-06 21:07:24 +03:00
qananasikq
81e99a41e8 Refactor worker 2026-05-06 21:07:24 +03:00
qananasikq
47ee4311c7 Remove flaresolverr and optimize refresh upserts 2026-05-04 19:02:49 +03:00
qananasikq
6dc08c15f0 refactor mobile.de parser, fix country mapping, update README 2026-05-04 19:02:49 +03:00
45 changed files with 6538 additions and 3080 deletions

View File

@@ -1,69 +1,93 @@
# Параметры mobile.de для Docker.
# Docker environment for mobile.de
# Настройки PostgreSQL.
# Database
POSTGRES_USER=mobilede
POSTGRES_PASSWORD=mobilede
POSTGRES_DB=mobilede_scraper
MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
MOBILEDE_DATABASE_ECHO=false
MOBILEDE_DATABASE_POOL_SIZE=20
MOBILEDE_DATABASE_MAX_OVERFLOW=40
MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
# Порты сервисов.
# Ports
MOBILEDE_HOST_API_PORT=8000
MOBILEDE_HOST_POSTGRES_PORT=5432
MOBILEDE_HOST_REDIS_PORT=6379
# Временные legacy-переменные.
MOBILEDE_DATABASE_URL=postgresql+psycopg2://mobilede:mobilede@postgres:5432/mobilede_scraper
MOBILEDE_DATABASE_ECHO=false
MOBILEDE_DATABASE_POOL_SIZE=5
MOBILEDE_DATABASE_MAX_OVERFLOW=5
MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS=1800
# Настройки Redis и Celery.
# Redis and Celery
MOBILEDE_REDIS_URL=redis://redis:6379/0
CELERY_BROKER_URL=redis://redis:6379/0
CELERY_RESULT_BACKEND=redis://redis:6379/0
CELERY_TASK_SOFT_TIME_LIMIT=86400
CELERY_TASK_TIME_LIMIT=86520
CELERY_BROKER_VISIBILITY_TIMEOUT=90000
CELERY_WORKER_CONCURRENCY=1
CELERY_BEAT_SYNC_INTERVAL_MINUTES=60
IAAI_STARTUP_SYNC_ENABLED=true
MOBILEDE_STARTUP_MAX_PAGES=5
MOBILEDE_BEAT_MAX_PAGES=5
CELERY_WORKER_CONCURRENCY=3
CELERY_WORKER_POOL=prefork
CELERY_WORKER_MAX_TASKS_PER_CHILD=5
# Одноразовые CLI-сервисы.
MOBILEDE_SEARCH_START_PAGE=1
MOBILEDE_SEARCH_MAX_PAGES=1
MOBILEDE_REQUEST_DELAY_SECONDS=0.7
MOBILEDE_SYNC_LANE=mobile_de_cars
MOBILEDE_LISTING_ID=449929602
MOBILEDE_CONTINUOUS_SYNC_ENABLED=true
MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=0
# Runtime mode
MOBILEDE_STARTUP_SYNC_ENABLED=false
MOBILEDE_BEAT_SYNC_ENABLED=false
MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED=true
MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS=3600
MOBILEDE_CONTINUOUS_SYNC_ENABLED=false
MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS=3600
MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
# Search settings
MOBILEDE_FILTERED_SEARCH_URL=
MOBILEDE_FILTERED_SEARCH_URLS=
MOBILEDE_REQUEST_DELAY_SECONDS=0.30
MOBILEDE_CONCURRENT_PAGES=1
MOBILEDE_CURSOR_ENABLED=true
MOBILEDE_PROGRESS_LOG_EVERY_PAGES=10
MOBILEDE_ROTATE_RUNTIME_SEGMENTS=true
MOBILEDE_RUNTIME_INITIAL_TASKS=3
MOBILEDE_SEGMENT_PAGE_WINDOW=10
MOBILEDE_RESULTS_PER_PAGE=20
MOBILEDE_MAX_PAGE_NUMBER=50
MOBILEDE_SEGMENT_TARGET_RESULTS=1000
MOBILEDE_SKIP_EMPTY_WINDOW=true
MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS=true
MOBILEDE_COMPACT_SEGMENTS=true
MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE=false
# Прокси для HTTP-запросов.
# Пример:
# MOBILEDE_PROXY_SERVER=http://host:port
# MOBILEDE_PROXY_USERNAME=username
# MOBILEDE_PROXY_PASSWORD=password
# Planner and overflow
MOBILEDE_PREPLAN_SEGMENT_PROBES=true
MOBILEDE_PREPLAN_MAX_SEGMENTS=700
MOBILEDE_PREPLAN_MAX_PROBES=80
MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH=2
MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO=2.5
MOBILEDE_DYNAMIC_SEGMENT_PROBES=false
MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN=true
MOBILEDE_REFINE_ADAPTIVE_SEGMENTS=true
MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS=120
MOBILEDE_OVERFLOW_SPLIT_ENABLED=true
MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH=4
MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS=3
MOBILEDE_OVERFLOW_MIN_PRICE_SPLIT_SPAN=1000
# Existing car refresh
MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP=false
MOBILEDE_INCREMENTAL_PAGE_WINDOW=10
MOBILEDE_LIGHT_REFRESH_EXISTING=true
MOBILEDE_SKIP_IMAGES_FOR_UPDATED=1
MOBILEDE_FETCH_DETAIL_FOR_ALL_IMAGES=false
MOBILEDE_DETAIL_ENRICH_CONCURRENCY=1
MOBILEDE_DETAIL_ENRICH_COOLDOWN_SECONDS=1800
MOBILEDE_DETAIL_ENRICH_DELAY_SECONDS=0.45
MOBILEDE_DETAIL_ENRICH_JITTER_SECONDS=0.2
MOBILEDE_DETAIL_ENRICH_MAX_FAILURES_PER_PAGE=2
MOBILEDE_DETAIL_ENRICH_DISABLE_ON_403=true
# Proxy
MOBILEDE_PROXY_SERVER=
MOBILEDE_PROXY_USERNAME=
MOBILEDE_PROXY_PASSWORD=
# Опциональный SOCKS5 bridge.
# При SOCKS5_PROXY_HOST мост поднимается на http://127.0.0.1:8899.
# Тогда MOBILEDE_PROXY_SERVER можно не задавать.
SOCKS5_PROXY_HOST=
SOCKS5_PROXY_PORT=1002
SOCKS5_PROXY_USER=
SOCKS5_PROXY_PASS=
# Логи и runtime.
IAAI_LOG_LEVEL=INFO
IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
TZ=UTC
# Старые browser-переменные.
IAAI_HEADLESS=true
IAAI_BROWSER_ENGINE=chromium
IAAI_TOKENS_FILE=/data/tokens.json
IAAI_SELF_HEAL_ENABLED=true

8
.gitignore vendored
View File

@@ -6,6 +6,8 @@ coverage.xml
!.env.example
.venv/
venv/
*.tgz
*.tar.gz
*.pyc
*.pyo
*.pyd
@@ -24,3 +26,9 @@ tokens.json
.tmp_db_check.sql
deploy-*.tar.gz
tmp_worker_log.txt
*.bak.*
_snapshot_info.txt
.DS_Store
Thumbs.db
.idea/
.ruff_cache/

View File

@@ -27,7 +27,7 @@ STOPSIGNAL SIGINT
USER app
# По умолчанию запускаем API.
# По умолчанию запускаем API.
ENTRYPOINT ["./entrypoint.sh"]
CMD ["uvicorn", "mobilede_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

View File

@@ -16,7 +16,7 @@ from mobilede_scraper.storage.models import Base
config = context.config
VERSION_TABLE = "MOBILEDE_parser_alembic_version"
# Logging
# Логирование
if config.config_file_name is not None:
fileConfig(config.config_file_name)
@@ -28,7 +28,7 @@ target_metadata = Base.metadata
def run_migrations_offline() -> None:
# Run migrations in 'offline' mode.
# Запуск миграций в офлайн-режиме.
url = config.get_main_option("sqlalchemy.url")
context.configure(
url=url,
@@ -42,7 +42,7 @@ def run_migrations_offline() -> None:
def run_migrations_online() -> None:
# Run migrations in 'online' mode.
# Запуск миграций в онлайн-режиме.
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",

View File

@@ -99,5 +99,5 @@ def upgrade() -> None:
def downgrade() -> None:
# Compatibility-only migration for shared production databases.
# Миграция оставлена только для совместимости с общей production-базой.
pass

View File

@@ -20,5 +20,5 @@ def upgrade() -> None:
def downgrade() -> None:
# Compatibility-only migration for shared production databases.
# Миграция оставлена только для совместимости с общей production-базой.
pass

View File

@@ -10,21 +10,21 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Partial index для активных машин MOBILEDE (is_sold = FALSE)
# Ускоряет поиск при mark_sold операциях
# Частичный индекс для активных машин MOBILEDE (is_sold = FALSE).
# Ускоряет поиск при операциях mark_sold.
op.execute(
'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_MOBILEDE" '
'ON "MOBILEDE_cars" (origin_url) '
"WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%'"
)
# Composite index для проверки дубликатов изображений
# Составной индекс для проверки дубликатов изображений.
op.execute(
'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id_fullres" '
'ON "MOBILEDE_images" (car_id, fullres_image)'
)
# Index для быстрого поиска existing машин по origin_url
# Индекс для быстрого поиска существующих машин по origin_url.
op.execute(
'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_origin_url_id" '
'ON "MOBILEDE_cars" (origin_url, origin_id)'
@@ -32,5 +32,5 @@ def upgrade() -> None:
def downgrade() -> None:
# Compatibility-only migration for shared production databases.
# Миграция оставлена только для совместимости с общей production-базой.
pass

View File

@@ -1,4 +1,4 @@
# Placeholder migration (ingestion tables not yet needed)
# Заглушка миграции (таблицы ingestion пока не требуются)
from typing import Sequence, Union
from alembic import op

View File

@@ -0,0 +1,31 @@
from typing import Sequence, Union
import sqlalchemy as sa
from alembic import op
revision: str = "005_add_car_seen_sold_timestamps"
down_revision: Union[str, None] = "004_add_ingestion_tables"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.add_column(
"MOBILEDE_cars",
sa.Column("first_seen_at", sa.DateTime(timezone=True), nullable=True),
)
op.add_column(
"MOBILEDE_cars",
sa.Column("sold_at", sa.DateTime(timezone=True), nullable=True),
)
op.execute('UPDATE "MOBILEDE_cars" SET first_seen_at = last_seen_at WHERE first_seen_at IS NULL')
op.alter_column("MOBILEDE_cars", "first_seen_at", nullable=False)
op.create_index("ix_MOBILEDE_cars_first_seen_at", "MOBILEDE_cars", ["first_seen_at"])
op.create_index("ix_MOBILEDE_cars_sold_at", "MOBILEDE_cars", ["sold_at"])
def downgrade() -> None:
op.drop_index("ix_MOBILEDE_cars_sold_at", table_name="MOBILEDE_cars")
op.drop_index("ix_MOBILEDE_cars_first_seen_at", table_name="MOBILEDE_cars")
op.drop_column("MOBILEDE_cars", "sold_at")
op.drop_column("MOBILEDE_cars", "first_seen_at")

View File

@@ -4,31 +4,17 @@
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS: ${MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-20}
MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-40}
MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-40}
MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-80}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
IAAI_PROFILE: ${IAAI_PROFILE:-fast}
IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}}
IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true}
IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false}
IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false}
IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1}
IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-2}
IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0}
IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1}
IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
# 0 = без лимита.
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-6}
CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000}
MOBILEDE_STARTUP_MAX_PAGES: ${MOBILEDE_STARTUP_MAX_PAGES:-100}
MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100}
MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0}
MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.45}
MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-1}
MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800"
MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-}
MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-}
MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
@@ -37,51 +23,48 @@
MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10}
MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2}
MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-5}
MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true}
MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10}
MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20}
MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50}
MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1000}
MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-700}
MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-false}
MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-1000}
MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40}
# Первый проход должен идти по заранее нарезанным leaf-сегментам.
# Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально.
MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true}
MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400}
MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-320}
MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0}
MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-3}
MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN: ${MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN:-true}
MOBILEDE_FILTERED_URL_FAST_START: ${MOBILEDE_FILTERED_URL_FAST_START:-false}
MOBILEDE_REFINE_ADAPTIVE_SEGMENTS: ${MOBILEDE_REFINE_ADAPTIVE_SEGMENTS:-true}
MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS: ${MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS:-240}
MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES: ${MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES:-3}
MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8}
MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-false}
MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true}
MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.70}
MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.90}
MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.35}
MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true}
MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1}
MOBILEDE_FLARESOLVERR_ENABLED: ${MOBILEDE_FLARESOLVERR_ENABLED:-false}
MOBILEDE_FLARESOLVERR_URL: ${MOBILEDE_FLARESOLVERR_URL:-http://flaresolverr:8191/v1}
MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS: ${MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS:-120}
MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS: ${MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS:-60000}
MOBILEDE_FLARESOLVERR_SESSION: ${MOBILEDE_FLARESOLVERR_SESSION:-}
MOBILEDE_FLARESOLVERR_STATUSES: ${MOBILEDE_FLARESOLVERR_STATUSES:-403,429,503}
IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-32}
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
IAAI_FILTERED_SEARCH_URL: ${IAAI_FILTERED_SEARCH_URL:-}
IAAI_FILTERED_SEARCH_URLS: ${IAAI_FILTERED_SEARCH_URLS:-}
IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-runtime}
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json}
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
IAAI_BROWSER_ENGINE: chromium
MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false}
MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-10}
MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED: ${MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED:-false}
MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false}
MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10}
MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true}
MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
# Автовосстановление worker.
IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
MOBILEDE_SELF_HEAL_ENABLED: ${MOBILEDE_SELF_HEAL_ENABLED:-true}
MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
MOBILEDE_SELF_HEAL_STALL_SECONDS: ${MOBILEDE_SELF_HEAL_STALL_SECONDS:-900}
MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
# При простое БД перезапускаем worker.
IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
MOBILEDE_DB_IDLE_RESTART_SECONDS: ${MOBILEDE_DB_IDLE_RESTART_SECONDS:-3600}
TZ: ${TZ:-UTC}
x-env-file: &env-file
@@ -105,7 +88,6 @@ services:
# База данных.
postgres:
image: postgres:16-alpine
container_name: mobilede-postgres
restart: unless-stopped
environment:
POSTGRES_USER: ${POSTGRES_USER:-mobilede}
@@ -129,7 +111,6 @@ services:
# Очередь Redis.
redis:
image: redis:7-alpine
container_name: mobilede-redis
restart: unless-stopped
ports:
- "127.0.0.1:${MOBILEDE_HOST_REDIS_PORT:-6379}:6379"
@@ -150,27 +131,9 @@ services:
max-size: "10m"
max-file: "5"
flaresolverr:
image: ghcr.io/flaresolverr/flaresolverr:latest
container_name: mobilede-flaresolverr
restart: unless-stopped
environment:
LOG_LEVEL: ${FLARESOLVERR_LOG_LEVEL:-info}
LOG_HTML: ${FLARESOLVERR_LOG_HTML:-false}
CAPTCHA_SOLVER: ${FLARESOLVERR_CAPTCHA_SOLVER:-none}
TZ: ${TZ:-UTC}
ports:
- "127.0.0.1:${MOBILEDE_HOST_FLARESOLVERR_PORT:-8191}:8191"
logging:
driver: json-file
options:
max-size: "10m"
max-file: "5"
# Миграции.
migrate:
<<: *app-service
container_name: mobilede-migrate
restart: "no"
depends_on:
postgres:
@@ -180,7 +143,6 @@ services:
# API сервис.
api:
<<: *app-service
container_name: mobilede-api
restart: unless-stopped
ports:
- "127.0.0.1:${MOBILEDE_HOST_API_PORT:-8000}:8000"
@@ -208,9 +170,7 @@ services:
# Worker сервис.
worker:
<<: *worker-service
container_name: mobilede-worker
restart: unless-stopped
init: true
depends_on:
migrate:
condition: service_completed_successfully
@@ -219,9 +179,9 @@ services:
stop_grace_period: 60s
command: >
celery -A mobilede_scraper.worker.celery_app worker
--loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork}
--loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-8} --pool=${CELERY_WORKER_POOL:-prefork}
--pidfile=/tmp/celery-worker.pid
-Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-Q mobilede_sync --max-tasks-per-child=5
healthcheck:
# Проверка worker.
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"]
@@ -238,7 +198,6 @@ services:
# Beat сервис.
beat:
<<: *worker-service
container_name: mobilede-beat
restart: unless-stopped
init: true
depends_on:
@@ -266,7 +225,6 @@ services:
# Одноразовый сбор страниц поиска.
mobilede-search:
<<: *app-service
container_name: mobilede-search
profiles: ["cli"]
restart: "no"
depends_on:
@@ -285,7 +243,6 @@ services:
# Одноразовый sync поиска в БД.
mobilede-sync-search:
<<: *app-service
container_name: mobilede-sync-search
profiles: ["cli"]
restart: "no"
depends_on:
@@ -305,7 +262,6 @@ services:
# Одноразовый сбор карточки по ID.
mobilede-detail:
<<: *app-service
container_name: mobilede-detail
profiles: ["cli"]
restart: "no"
depends_on:

View File

@@ -1,4 +1,4 @@
# Создание FastAPI-приложения и настройка его жизненного цикла.
# FastAPI-приложение и его жизненный цикл.
from contextlib import asynccontextmanager
@@ -11,8 +11,7 @@ from .routes import cars, health, tasks
@asynccontextmanager
async def lifespan(app: FastAPI):
# Жизненный цикл.
# Таблицы создаются через Alembic-миграции (сервис migrate).
# Таблицы создаются миграциями Alembic.
yield
@@ -36,5 +35,5 @@ def create_app(settings: Settings | None = None) -> FastAPI:
return app
# Экземпляр приложения для запуска через uvicorn.
# Точка входа для uvicorn.
app = create_app()

View File

@@ -1,4 +1,4 @@
# Вспомогательные зависимости для FastAPI-роутов.
# Зависимости FastAPI.
from fastapi import Request

View File

@@ -1,7 +1,3 @@
"""API route package.
Routers are defined in dedicated modules (`cars`, `health`, `tasks`). This
package intentionally does not create routes to avoid duplicate task endpoints
and accidental imports of legacy parser channels during FastAPI startup.
"""
# Пакет роутов API.
# Здесь только импорт модулей роутов без побочных эффектов.

View File

@@ -1,4 +1,4 @@
# Роуты для просмотра автомобилей и агрегированной статистики.
# Роуты для авто и статистики.
from fastapi import APIRouter, Depends, HTTPException, Query
from sqlalchemy import func, select
@@ -22,7 +22,7 @@ def list_cars(
is_sold: bool | None = None,
persistence: PersistenceService = Depends(get_persistence),
):
# Список автомобилей с пагинацией и фильтрами
# Список авто с фильтрами.
with persistence.session_scope() as session:
query = select(Car)
@@ -59,7 +59,7 @@ def get_car(
car_id: int,
persistence: PersistenceService = Depends(get_persistence),
):
# Детальная информация об автомобиле с изображениями
# Карточка авто с фото.
with persistence.session_scope() as session:
car = session.get(Car, car_id)
if car is None:
@@ -72,7 +72,7 @@ def get_car_by_origin(
origin_id: str,
persistence: PersistenceService = Depends(get_persistence),
):
# Поиск автомобиля по origin_id
# Поиск по origin_id.
with persistence.session_scope() as session:
car = session.execute(
select(Car).where(Car.origin_id == origin_id)
@@ -84,7 +84,7 @@ def get_car_by_origin(
@router.get("/stats")
def get_stats(persistence: PersistenceService = Depends(get_persistence)):
# Общая статистика по БД
# Общая статистика.
with persistence.session_scope() as session:
total_cars = session.execute(select(func.count(Car.id))).scalar() or 0
total_images = session.execute(select(func.count(Image.id))).scalar() or 0

View File

@@ -1,4 +1,4 @@
# Роут проверки доступности сервиса и соединения с БД.
# Проверка API и БД.
import logging
@@ -14,7 +14,7 @@ logger = logging.getLogger("MOBILEDE_scraper.api.health")
@router.get("/health")
def health_check(persistence: PersistenceService = Depends(get_persistence)):
# Проверка API и БД
# Проверяем доступность БД.
db_ok = False
try:
with persistence.session_scope() as session:

View File

@@ -1,4 +1,4 @@
# Роуты запуска задач синхронизации и просмотра истории sync-runs.
# Роуты запуска задач и истории sync-run.
import json
@@ -11,8 +11,14 @@ from ...core.config import Settings
from ..deps import get_persistence
from ...storage.db import PersistenceService
from ...storage.models import SyncRun
from ...worker.celery_app import MOBILEDE_SYNC_QUEUE, celery_app
from ...worker.tasks import mobilede_sync_detail_task, mobilede_sync_runtime_segments_task, mobilede_sync_search_task
from ...worker.celery_app import celery_app
from ...worker.constants import MOBILEDE_SYNC_QUEUE
from ...worker.tasks import (
mobilede_enrich_images_batch_task,
mobilede_sync_detail_task,
mobilede_sync_runtime_segments_task,
mobilede_sync_search_task,
)
router = APIRouter()
@@ -38,6 +44,13 @@ class MobileDeSyncDetailRequest(BaseModel):
lane: str = "mobile_de_cars"
class MobileDeEnrichImagesRequest(BaseModel):
limit: int = 50
lane: str = "mobile_de_cars"
max_existing_images: int = 1
delay_seconds: float = 1.0
class MobileDeRuntimeSegmentsRequest(BaseModel):
lane: str = "mobile_de_cars"
delay_seconds: float = 0.7
@@ -72,6 +85,19 @@ def start_mobilede_sync_detail(body: MobileDeSyncDetailRequest):
}
@router.post("/mobilede/tasks/enrich-images")
def start_mobilede_enrich_images(body: MobileDeEnrichImagesRequest):
result = mobilede_enrich_images_batch_task.apply_async(
kwargs=body.model_dump(),
queue=MOBILEDE_SYNC_QUEUE,
)
return {
"task_id": result.id,
"status": "queued",
"queue": MOBILEDE_SYNC_QUEUE,
}
@router.post("/mobilede/tasks/sync-runtime-segments")
def start_mobilede_runtime_segments(body: MobileDeRuntimeSegmentsRequest):
result = mobilede_sync_runtime_segments_task.apply_async(
@@ -141,7 +167,7 @@ def list_sync_runs(
per_page: int = Query(20, ge=1, le=100),
persistence: PersistenceService = Depends(get_persistence),
):
# История запусков синхронизации
# История запусков.
with persistence.session_scope() as session:
total = session.execute(select(func.count(SyncRun.id))).scalar() or 0
offset = (page - 1) * per_page

View File

@@ -51,7 +51,6 @@ def _env_float(name: str, default: float) -> float:
@dataclass(slots=True)
class ListingConfig:
cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.mobile.de/Vehiclelisting/Cars")
filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
@@ -112,51 +111,6 @@ class CeleryConfig:
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("MOBILEDE_PARALLEL_TABS", 8)
fetch_concurrency: int = _env_int("MOBILEDE_FETCH_CONCURRENCY", _env_int("MOBILEDE_PARALLEL_TABS", 8))
parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
block_resources: bool = _env_bool("MOBILEDE_BLOCK_RESOURCES", True)
@dataclass(slots=True)
class ScrapingProfileConfig:
name: str = _env_str("MOBILEDE_SCRAPING_PROFILE", _env_str("MOBILEDE_PROFILE", "stable")).strip().lower()
http_first: bool = _env_bool("MOBILEDE_HTTP_FIRST", True)
browser_fallback_enabled: bool = _env_bool("MOBILEDE_BROWSER_FALLBACK_ENABLED", True)
anonymous_bootstrap_enabled: bool = _env_bool("MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED", True)
verbose_http_logs: bool = _env_bool("MOBILEDE_VERBOSE_HTTP_LOGS", False)
verbose_progress_logs: bool = _env_bool("MOBILEDE_VERBOSE_PROGRESS_LOGS", False)
challenge_refresh_enabled: bool = _env_bool("MOBILEDE_CHALLENGE_REFRESH_ENABLED", True)
challenge_refresh_attempts: int = _env_int("MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS", 3)
listing_post_attempts: int = _env_int("MOBILEDE_LISTING_POST_ATTEMPTS", 4)
request_jitter_max_s: float = _env_float("MOBILEDE_REQUEST_JITTER_MAX_S", 0.15)
detail_retries: int | None = _env_int("MOBILEDE_DETAIL_RETRIES", -1)
listing_retries: int | None = _env_int("MOBILEDE_LISTING_RETRIES", -1)
def __post_init__(self) -> None:
if self.name == "fast":
if "MOBILEDE_BROWSER_FALLBACK_ENABLED" not in os.environ:
self.browser_fallback_enabled = False
if "MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
self.anonymous_bootstrap_enabled = False
if "MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
self.challenge_refresh_attempts = 1
if "MOBILEDE_LISTING_POST_ATTEMPTS" not in os.environ:
self.listing_post_attempts = 2
if "MOBILEDE_REQUEST_JITTER_MAX_S" not in os.environ:
self.request_jitter_max_s = 0.0
if "MOBILEDE_DETAIL_RETRIES" not in os.environ:
self.detail_retries = 1
if "MOBILEDE_LISTING_RETRIES" not in os.environ:
self.listing_retries = 1
else:
if self.detail_retries is not None and self.detail_retries < 0:
self.detail_retries = None
if self.listing_retries is not None and self.listing_retries < 0:
self.listing_retries = None
# Конфиг прокси.
@@ -170,16 +124,6 @@ class ProxyConfig:
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
if self.username:
result["username"] = self.username
if self.password:
result["password"] = self.password
return result
def to_requests_proxy_url(self) -> str | None:
if not self.server:
return None
@@ -211,44 +155,13 @@ class ProxyConfig:
@dataclass(slots=True)
class Settings:
home_url: str = "https://www.mobile.de/"
default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000)
network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400)
fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000)
fast_path_max_attempts: int = _env_int("MOBILEDE_FAST_PATH_MAX_ATTEMPTS", 1)
fallback_navigation_timeout_ms: int = _env_int("MOBILEDE_FALLBACK_NAV_TIMEOUT_MS", 15000)
max_retries: int = _env_int("MOBILEDE_MAX_RETRIES", 3)
retry_delay_seconds: float = _env_float("MOBILEDE_RETRY_DELAY_SECONDS", 2.5)
retry_backoff_multiplier: float = _env_float("MOBILEDE_RETRY_BACKOFF_MULTIPLIER", 2.0)
retry_jitter_seconds: float = _env_float("MOBILEDE_RETRY_JITTER_SECONDS", 0.25)
headless: bool = _env_bool("MOBILEDE_HEADLESS", True)
browser_engine: str = _env_str("MOBILEDE_BROWSER_ENGINE", "auto")
log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("MOBILEDE_LOG_FILE")
enable_trace_id_logs: bool = _env_bool("MOBILEDE_ENABLE_TRACE_ID_LOGS", True)
sync_only_new: bool = _env_bool("MOBILEDE_SYNC_ONLY_NEW", False)
raw_output_json: str | None = _env_optional_str("MOBILEDE_RAW_OUTPUT_JSON")
tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE")
runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60)
listing: ListingConfig = field(default_factory=ListingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
@property
def parallel_tabs(self) -> int:
return self.celery.parallel_tabs
@property
def fetch_concurrency(self) -> int:
return self.celery.fetch_concurrency
@property
def block_resources(self) -> bool:
return self.celery.block_resources
# Глобальные настройки.
settings = Settings()

View File

@@ -1,14 +0,0 @@
class ScraperError(Exception):
"""Базовое исключение скрапера."""
class AntiBotDetectedError(ScraperError):
"""Вызывается, когда сайт блокирует автоматизацию."""
class SiteStructureChangedError(ScraperError):
"""Вызывается, когда структура страницы изменилась и данных не хватает."""
class ListingResumeError(ScraperError):
"""Вызывается, когда resume по checkpoint больше недостижим."""

View File

@@ -2,12 +2,12 @@ import logging
import sys
from contextvars import ContextVar
# Храним trace_id текущего потока/корутины.
# Trace ID текущего потока или корутины.
TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
class TraceIdFilter(logging.Filter):
# Добавляет trace_id в каждую запись лога для сквозной трассировки.
# Добавляет trace_id в запись лога.
def filter(self, record: logging.LogRecord) -> bool:
record.trace_id = TRACE_ID.get()
return True
@@ -18,7 +18,7 @@ def set_trace_id(trace_id: str) -> None:
def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
# stderr Docker и Celery prefork корректно его подхватывают.
# Пишем в stderr для Docker и Celery.
handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
if log_file:
handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
@@ -28,7 +28,7 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
handler.setLevel(getattr(logging, level.upper(), logging.INFO))
root = logging.getLogger()
root.setLevel(getattr(logging, level.upper(), logging.INFO))
# Убираем старые хендлеры, чтобы не дублировать после fork.
# Убираем старые хендлеры после fork.
for old_handler in list(root.handlers):
try:
old_handler.close()

View File

@@ -1,53 +0,0 @@
import logging
import random
import time
from collections.abc import Callable
from functools import wraps
from typing import Any
from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
from .exceptions import AntiBotDetectedError
logger = logging.getLogger("MOBILEDE_scraper.retry")
# Типы исключений, при которых retry имеет смысл.
RETRYABLE_EXCEPTIONS = (
PlaywrightTimeoutError,
Error,
ConnectionError,
OSError,
TimeoutError,
AntiBotDetectedError,
)
def retryable(
max_attempts: int,
delay_seconds: float = 2.5,
backoff_multiplier: float = 2.0,
jitter_seconds: float = 0.0,
) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
@wraps(func)
def wrapper(*args: Any, **kwargs: Any) -> Any:
last_error: Exception | None = None
for attempt in range(1, max_attempts + 1):
try:
return func(*args, **kwargs)
except RETRYABLE_EXCEPTIONS as exc:
last_error = exc
logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
if attempt < max_attempts:
sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
if jitter_seconds > 0:
sleep_for += random.uniform(0, jitter_seconds)
logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
time.sleep(sleep_for)
if last_error is not None:
raise last_error
raise RuntimeError("Retry wrapper failed without a captured exception")
return wrapper
return decorator

View File

@@ -1,7 +1,6 @@
import json
import re
import json
from pathlib import Path
from typing import Any, Callable, Iterable
from typing import Any
def save_to_json(data: Any, filename: str | Path) -> None:
@@ -10,21 +9,7 @@ def save_to_json(data: Any, filename: str | Path) -> None:
path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
def first_non_empty(values: Iterable[Any]) -> Any | None:
for value in values:
if value not in (None, "", [], {}, ()):
return value
return None
# Регулярные выражения для VIN, lot и price.
VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
LOT_RE = re.compile(r"\b(\d{7,10})\b")
PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
# Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
found = []
if _depth >= max_depth:
return found
@@ -37,36 +22,3 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
for item in obj:
found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
return found
def deep_find_all_keys(
payloads: list,
field_map: dict[str, set[str]],
max_depth: int = 64,
) -> dict[str, list]:
"""Извлекает все нужные поля за один проход по JSON."""
# Готовим обратную карту: нормализованный ключ -> имя поля.
reverse: dict[str, str] = {}
for field_name, keys in field_map.items():
for k in keys:
reverse[k.lower()] = field_name
result: dict[str, list] = {f: [] for f in field_map}
def _recurse(obj: Any, depth: int) -> None:
if depth >= max_depth:
return
if isinstance(obj, dict):
for k, v in obj.items():
field = reverse.get(k.lower())
if field is not None:
result[field].append(v)
_recurse(v, depth + 1)
elif isinstance(obj, list):
for item in obj:
_recurse(item, depth + 1)
for payload in payloads:
_recurse(payload, 0)
return result

View File

@@ -1 +0,0 @@
__all__: list[str] = []

View File

@@ -3,6 +3,7 @@ from __future__ import annotations
import logging
import os
import random
import threading
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from collections.abc import Callable, Iterable
@@ -33,6 +34,7 @@ MOBILEDE_HTTP_BACKOFF_BASE_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BAC
MOBILEDE_HTTP_BACKOFF_MAX_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_BACKOFF_MAX_SECONDS", "20")))
MOBILEDE_HTTP_JITTER_SECONDS = max(0.0, float(os.getenv("MOBILEDE_HTTP_JITTER_SECONDS", "0.5")))
MOBILEDE_HTTP_RETRY_STATUSES = {403, 429, 500, 502, 503, 504}
MOBILEDE_DETAIL_TIMEOUT_SECONDS = max(1, int(float(os.getenv("MOBILEDE_DETAIL_TIMEOUT_SECONDS", "15"))))
MOBILEDE_FLARESOLVERR_ENABLED = os.getenv("MOBILEDE_FLARESOLVERR_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_FLARESOLVERR_URL = os.getenv("MOBILEDE_FLARESOLVERR_URL", "http://flaresolverr:8191/v1").strip()
MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS = max(1.0, float(os.getenv("MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS", "120")))
@@ -241,6 +243,8 @@ class MobileDeClient:
page_number: int = 1,
*,
search_url: str | None = None,
timeout: int = 30,
max_retries: int | None = None,
**params: str | int | None,
) -> MobileDeSearchPage:
url = (
@@ -248,7 +252,15 @@ class MobileDeClient:
if search_url
else self.build_search_url(page_number=page_number, **params)
)
html = self.fetch_html(url)
if max_retries is None:
html = self.fetch_html(url, timeout=timeout)
else:
previous_retries = os.environ.get("MOBILEDE_HTTP_MAX_RETRIES")
try:
globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(max_retries))
html = self.fetch_html(url, timeout=timeout)
finally:
globals()["MOBILEDE_HTTP_MAX_RETRIES"] = max(0, int(previous_retries or "4"))
raw = extract_search_results(html)
listings = [self._map_listing(item) for item in raw.get("listings", []) if isinstance(item, dict)]
return MobileDeSearchPage(
@@ -364,7 +376,7 @@ class MobileDeClient:
return ordered_pages
def fetch_detail(self, listing_id: str | int) -> dict:
html = self.fetch_html(self.build_detail_url(listing_id))
html = self.fetch_html(self.build_detail_url(listing_id), timeout=MOBILEDE_DETAIL_TIMEOUT_SECONDS)
return extract_detail_listing(html)
@staticmethod

View File

@@ -15,7 +15,8 @@ def extract_next_flight_strings(html: str) -> list[str]:
try:
chunks.append(json.loads(f'"{raw}"'))
except json.JSONDecodeError:
# Fallback keeps parser useful if one chunk has non-standard escaping.
# Резервный вариант: сохраняем работоспособность парсера,
# даже если один из фрагментов имеет нестандартное экранирование.
chunks.append(raw.encode("utf-8", errors="ignore").decode("unicode_escape", errors="ignore"))
return chunks

View File

@@ -4,6 +4,7 @@ import hashlib
import re
from datetime import datetime, timezone
from typing import Any
from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
from ..storage.schemas import CarRecord, ImageRecord
from .client import MobileDeClient
@@ -11,22 +12,47 @@ from .models import MobileDeListing
_BODY_MAP = {
"cabrio": "OPEN",
"cabriolet": "OPEN",
"кабриолет": "OPEN",
"roadster": "OPEN",
"кабрио": "OPEN",
"limousine": "SEDAN",
"saloon": "SEDAN",
"sedan": "SEDAN",
"сeдан": "SEDAN",
"седан": "SEDAN",
"suv": "SUV",
"offroad": "SUV",
"gelandewagen": "SUV",
"geländewagen": "SUV",
"pickup": "PICKUP",
"pick-up": "PICKUP",
"пикап": "PICKUP",
"внедорож": "SUV",
"kombi": "STATION_WAGON",
"estatecar": "STATION_WAGON",
"touring": "STATION_WAGON",
"estate": "STATION_WAGON",
"универсал": "STATION_WAGON",
"van": "MINIVAN",
"kleinbus": "MINIVAN",
"bus": "MINIVAN",
"active tourer": "MINIVAN",
"gran tourer": "MINIVAN",
"минивэн": "MINIVAN",
"фургон": "MINIVAN",
"coupe": "COUPE",
"sportscar": "COUPE",
"sports car": "COUPE",
"купе": "COUPE",
"hatchback": "HATCHBACK",
"kleinwagen": "HATCHBACK",
"кляйнваген": "HATCHBACK",
"малолитраж": "HATCHBACK",
"хэтч": "HATCHBACK",
"smallcar": "HATCHBACK",
"small car": "HATCHBACK",
"compact": "HATCHBACK",
"compactcar": "HATCHBACK",
}
_GEARBOX_MAP = {
@@ -40,30 +66,87 @@ _GEARBOX_MAP = {
_COLOR_MAP = {
"schwarz": "black",
"saphirschwarz": "black",
"carbonschwarz": "black",
"obsidianschwarz": "black",
"jet black": "black",
"jetblack": "black",
"черн": "black",
"black": "black",
"weiss": "white",
"weiß": "white",
"alpinweiss": "white",
"alpine white": "white",
"mineralweiss": "white",
"бел": "white",
"white": "white",
"grau": "gray",
"сер": "gray",
"gray": "gray",
"silber": "silver",
"argent": "silver",
"сереб": "silver",
"silver": "silver",
"grau": "gray",
"grey": "gray",
"anthrazit": "gray",
"anthracite": "gray",
"graphit": "gray",
"graphite": "gray",
"spacegrau": "gray",
"brooklyn grau": "gray",
"brooklyn grey": "gray",
"sophistograu": "gray",
"skyscraper grau": "gray",
"сер": "gray",
"gray": "gray",
"rot": "red",
"burgundy": "red",
"bordeaux": "red",
"maroon": "red",
"красн": "red",
"red": "red",
"blau": "blue",
"turquoise": "blue",
"cyan": "blue",
"син": "blue",
"blue": "blue",
"grün": "green",
"gruen": "green",
"зелен": "green",
"green": "green",
"braun": "brown",
"корич": "brown",
"brown": "brown",
"beige": "beige",
"champagner": "beige",
"champagne": "beige",
"creme": "beige",
"cream": "beige",
"ivory": "beige",
"беж": "beige",
"gelb": "yellow",
"желт": "yellow",
"yellow": "yellow",
"orange": "orange",
"оранж": "orange",
"gold": "gold",
"золот": "gold",
"bronze": "bronze",
"бронз": "bronze",
"violett": "purple",
"lila": "purple",
"фиолет": "purple",
"purple": "purple",
}
_IMAGE_FIELD_HINTS = ["image", "images", "media", "gallery", "photo", "pic", "picture", "url", "src", "uri", "ref"]
_IMAGE_URL_MARKERS = ["img.classistatic.de", "/images/", "/image/", "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp"]
_MOBILEDE_IMAGE_RULE = "mo-640.jpg"
_COLOR_VALUE_KEYS = {"ecol", "color", "exteriorcolor", "manufacturercolorname", "vehiclecolor", "paint"}
_COUNTRY_VALUE_KEYS = {"country", "countrycode"}
_BODY_VALUE_KEYS = {"category", "bodytype", "vehiclecategory", "body"}
_ENGINE_VOLUME_VALUE_KEYS = {"cubiccapacity", "enginevolume", "displacement", "enginedisplacement"}
_DRIVE_VALUE_KEYS = {"wheeldrive", "drivetrain", "drive"}
_GEARBOX_VALUE_KEYS = {"transmission", "gearbox", "transmissiontype"}
class MobileDeMapper:
"""Map mobile.de payloads into CarRecord."""
@@ -78,35 +161,124 @@ class MobileDeMapper:
model = self._text(model_payload.get("localized") or self._model_from_title(listing.title, brand) or listing.subtitle or "UNKNOWN")
origin_id = self.origin_id(str(listing.id))
title = " ".join(part for part in [listing.title, listing.subtitle] if part)
contact = raw.get("contact") if isinstance(raw.get("contact"), dict) else {}
nested_country = self._find_first_value(raw, _COUNTRY_VALUE_KEYS)
nested_color = self._find_first_value(raw, _COLOR_VALUE_KEYS)
nested_body = self._find_first_value(raw, _BODY_VALUE_KEYS)
nested_engine = self._find_first_value(raw, _ENGINE_VOLUME_VALUE_KEYS)
nested_drive = self._find_first_value(raw, _DRIVE_VALUE_KEYS)
nested_gearbox = self._find_first_value(raw, _GEARBOX_VALUE_KEYS)
attr_country = self._mapping_value(attr, "cn", "countryCode", "country")
attr_color = self._mapping_value(attr, "ecol", "color", "exteriorColor", "manufacturerColorName", "paint")
attr_body = self._mapping_value(attr, "c", "category", "bodyType", "body")
attr_engine = self._mapping_value(attr, "cc", "cubicCapacity", "engineVolume", "displacement", "engineDisplacement")
attr_drive = self._mapping_value(attr, "wd", "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart")
attr_gearbox = self._mapping_value(attr, "tr", "transmission", "gearbox", "transmissionType")
drive_text = " ".join(
part
for part in [
listing.title,
listing.subtitle,
self._mapping_value(attr, "an"),
attr_drive,
raw.get("wheelDrive"),
raw.get("drivetrain"),
raw.get("drive"),
raw.get("driveType"),
raw.get("modelDescription"),
raw.get("variant"),
raw.get("trim"),
nested_drive,
]
if isinstance(part, str) and part.strip()
)
damage_text = self._text(
raw.get("damageCondition")
or attr.get("damageCondition")
or attr.get("dc")
).lower()
is_damaged = (
bool(raw.get("hasDamage"))
or ("дтп" in damage_text and "без дтп" not in damage_text)
or ("accident" in damage_text and "no accident" not in damage_text)
)
return CarRecord(
parser_id=self._parser_id(origin_id),
brand=brand[:50] or "UNKNOWN",
model=model[:50] or "UNKNOWN",
year=self._year_from_first_registration(listing.first_registration or attr.get("fr")),
price=self._money_to_int(listing.price or raw.get("p")),
year=self._year_from_first_registration(
self._first_present(
listing.first_registration,
attr.get("fr"),
attr.get("yc"),
raw.get("firstRegistration"),
raw.get("firstRegistrationYear"),
raw.get("year"),
)
),
price=self._money_to_int(self._first_present(listing.price, raw.get("p"), raw.get("price"))),
currency="EUR",
mileage=self._int_from_text(listing.mileage or attr.get("ml")) or 0,
country=self._normalize_country("DE"),
mileage=self._int_from_text(
self._first_present(listing.mileage, attr.get("ml"), raw.get("mileage"))
) or 0,
country=self._normalize_country(
self._first_present(
attr_country,
contact.get("countryCode"),
contact.get("country"),
raw.get("countryCode"),
raw.get("country"),
nested_country,
"DE",
)
),
is_sold=False,
color=self._normalize_color(attr.get("ecol")),
drive=None,
gearbox=self._normalize_gearbox(listing.transmission or attr.get("tr")),
color=self._normalize_color(
self._first_present(
attr_color,
raw.get("color"),
raw.get("manufacturerColorName"),
raw.get("exteriorColor"),
nested_color,
)
),
drive=self._normalize_drive(drive_text),
gearbox=self._normalize_gearbox(listing.transmission or attr_gearbox or nested_gearbox),
steering_wheel="LEFT",
body_type=self._normalize_body(attr.get("c")),
engine_volume=self._int_from_text(attr.get("cc")),
body_type=self._normalize_body_from_candidates(
attr_body,
raw.get("category"),
raw.get("bodyType"),
nested_body,
listing.subtitle,
listing.title,
),
engine_volume=self._engine_volume_from_candidates(
attr_engine,
raw.get("cubicCapacity"),
raw.get("cc"),
raw.get("engineVolume"),
raw.get("displacement"),
raw.get("engineDisplacement"),
nested_engine,
raw.get("modelDescription"),
raw.get("variant"),
listing.subtitle,
listing.title,
),
selling_type="CLASSIFIED",
one_owner=(str(attr.get("pvo") or "").strip() == "1"),
new_car=False,
one_owner=self._is_one_owner(attr.get("pvo") or raw.get("numPreviousOwners")),
new_car=bool(raw.get("isNew") or raw.get("isConditionNew")),
is_hidden=False,
origin="MOBILE_DE",
origin_url=listing.url,
origin_id=origin_id,
is_damaged=bool(raw.get("hasDamage")),
evaluation=self._text(raw.get("priceRating") or raw.get("rating")) or None,
is_damaged=is_damaged,
evaluation=self._rating_text(raw.get("priceRating") or raw.get("rating")),
non_smoking=True,
rental=False,
repair_history=bool(raw.get("hasDamage")),
repair_history=is_damaged,
slug=self._slugify(title or f"{brand} {model}"),
last_seen_at=datetime.now(timezone.utc),
images=self._images_from_listing(raw),
@@ -127,17 +299,45 @@ class MobileDeMapper:
origin_id = self.origin_id(str(listing_id))
price_amount, price_currency = self._detail_price(detail.get("price"))
mileage = self._int_from_text(attrs.get("mileage")) or 0
year = self._year_from_first_registration(attrs.get("firstRegistration"))
gearbox = self._normalize_gearbox(attrs.get("transmission"))
body_type = self._normalize_body(attrs.get("category") or detail.get("category"))
color = self._normalize_color(attrs.get("color") or attrs.get("manufacturerColorName"))
mileage = self._int_from_text(self._mapping_value(attrs, "mileage")) or 0
year = self._year_from_first_registration(self._mapping_value(attrs, "firstRegistration", "year", "registrationDate"))
gearbox = self._normalize_gearbox(self._first_present(self._mapping_value(attrs, "transmission", "gearbox", "transmissionType"), detail.get("transmission"), detail.get("gearbox")))
body_type = self._normalize_body_from_candidates(
self._mapping_value(attrs, "category", "bodyType", "body", "vehicleCategory"),
detail.get("category"),
detail.get("bodyType"),
subtitle,
short_title,
)
color = self._normalize_color(
self._first_present(
self._mapping_value(attrs, "color", "exteriorColor", "manufacturerColorName", "paint"),
detail.get("color"),
detail.get("manufacturerColorName"),
detail.get("exteriorColor"),
self._find_first_value(detail, _COLOR_VALUE_KEYS),
)
)
drive_text = " ".join(
part
for part in [
short_title,
subtitle,
self._mapping_value(attrs, "wheelDrive", "drivetrain", "drive", "driveType", "antriebsart"),
detail.get("wheelDrive"),
detail.get("drivetrain"),
detail.get("drive"),
detail.get("driveType"),
self._find_first_value(detail, _DRIVE_VALUE_KEYS),
]
if isinstance(part, str) and part.strip()
)
damage_text = self._text(attrs.get("damageCondition")).lower()
is_damaged = ("дтп" in damage_text and "без дтп" not in damage_text) or bool(detail.get("hasDamage"))
owners_text = self._text(attrs.get("numPreviousOwners"))
one_owner = owners_text in {"1", "01", "1.0"}
one_owner = self._is_one_owner(owners_text)
return CarRecord(
parser_id=self._parser_id(origin_id),
@@ -150,11 +350,21 @@ class MobileDeMapper:
country=self._normalize_country(contact.get("countryCode") or contact.get("country") or "DE"),
is_sold=False,
color=color,
drive=self._normalize_drive(attrs.get("wheelDrive") or attrs.get("drivetrain")),
drive=self._normalize_drive(drive_text),
gearbox=gearbox,
steering_wheel="LEFT",
body_type=body_type,
engine_volume=self._int_from_text(attrs.get("cubicCapacity") or attrs.get("cc")),
engine_volume=self._engine_volume_from_candidates(
self._mapping_value(attrs, "cubicCapacity", "cc", "engineVolume", "displacement", "engineDisplacement"),
detail.get("cubicCapacity"),
detail.get("cc"),
detail.get("engineVolume"),
detail.get("displacement"),
detail.get("engineDisplacement"),
self._find_first_value(detail, _ENGINE_VOLUME_VALUE_KEYS),
subtitle,
short_title,
),
selling_type="CLASSIFIED",
one_owner=one_owner,
new_car=bool(detail.get("isNew") or detail.get("isConditionNew")),
@@ -163,7 +373,7 @@ class MobileDeMapper:
origin_url=MobileDeClient.build_detail_url(listing_id),
origin_id=origin_id,
is_damaged=is_damaged,
evaluation=self._text(detail.get("priceRating") or detail.get("rating")) or None,
evaluation=self._rating_text(detail.get("priceRating") or detail.get("rating")),
non_smoking=True,
rental=False,
repair_history=is_damaged,
@@ -185,6 +395,53 @@ class MobileDeMapper:
def _text(value: Any) -> str:
return "" if value is None else str(value).strip()
@staticmethod
def _first_present(*values: Any) -> Any:
for value in values:
if value is None:
continue
if isinstance(value, str) and not value.strip():
continue
return value
return None
@staticmethod
def _normalized_key(value: Any) -> str:
return re.sub(r"[^a-z0-9]", "", str(value or "").lower())
@classmethod
def _find_first_value(cls, value: Any, keys: set[str], *, depth: int = 0) -> Any:
if depth > 6:
return None
if isinstance(value, dict):
for key, item in value.items():
if cls._normalized_key(key) in keys and cls._first_present(item) is not None:
if not isinstance(item, (dict, list)):
return item
for item in value.values():
found = cls._find_first_value(item, keys, depth=depth + 1)
if cls._first_present(found) is not None:
return found
elif isinstance(value, list):
for item in value:
found = cls._find_first_value(item, keys, depth=depth + 1)
if cls._first_present(found) is not None:
return found
return None
@classmethod
def _mapping_value(cls, mapping: Any, *keys: str) -> Any:
if not isinstance(mapping, dict):
return None
for key in keys:
if key in mapping and cls._first_present(mapping.get(key)) is not None:
return mapping.get(key)
normalized_keys = {cls._normalized_key(key) for key in keys if key}
for existing_key, value in mapping.items():
if cls._normalized_key(existing_key) in normalized_keys and cls._first_present(value) is not None:
return value
return None
@classmethod
def _money_to_int(cls, value: Any) -> int | None:
if isinstance(value, dict):
@@ -231,11 +488,11 @@ class MobileDeMapper:
@staticmethod
def _normalize_drive(value: Any) -> str | None:
text = "" if value is None else str(value).lower()
if any(marker in text for marker in ("front", "fwd", "перед")):
if any(marker in text for marker in ("front", "fwd", "перед", "frontantrieb", "vorderrad", "antrieb vorne", "front-wheel", "front wheel")):
return "FWD"
if any(marker in text for marker in ("rear", "rwd", "зад")):
if any(marker in text for marker in ("rear", "rwd", "зад", "heckantrieb", "hinterrad", "antrieb hinten", "rear-wheel", "rear wheel")):
return "RWD"
if any(marker in text for marker in ("all", "awd", "4x4", "quattro", "полный")):
if any(marker in text for marker in ("awd", "4wd", "4x4", "quattro", "полный", "xdrive", "4matic", "4motion", "allrad", "all-wheel", "all wheel", "four-wheel", "four wheel")):
return "4WD"
return None
@@ -248,28 +505,122 @@ class MobileDeMapper:
return "US"
if text in {"CA", "CANADA"}:
return "CA"
if text in {"IT", "ITALY", "ITALIA"}:
return "IT"
if text in {"JP", "JAPAN"}:
return "JP"
if text in {"KR", "KOREA", "SOUTH KOREA"}:
return "KR"
if re.fullmatch(r"[A-Z]{2}", text):
return text
return "NA"
@staticmethod
def _normalize_body(value: Any) -> str:
text = "" if value is None else str(value).lower()
text = text.replace("ё", "е")
if re.search(r"\bbmw\s+x(?:[1-7]|m)\b", text):
return "SUV"
for marker, mapped in _BODY_MAP.items():
if marker in text:
return mapped
return "OTHER"
@classmethod
def _normalize_body_from_candidates(cls, *values: Any) -> str:
for value in values:
normalized = cls._normalize_body(value)
if normalized != "OTHER":
return normalized
return "OTHER"
@classmethod
def _engine_volume_from_candidates(cls, *values: Any) -> int | None:
fallback_texts: list[str] = []
free_text_start = max(0, len(values) - 2)
for index, value in enumerate(values):
if value is None:
continue
if isinstance(value, str):
text = value.strip()
if not text:
continue
fallback_texts.append(text)
if index < free_text_start and re.fullmatch(r"[\d\s.,]+", text):
parsed = cls._int_from_text(text)
if parsed and 500 <= parsed <= 9000:
return parsed
if re.search(r"(ccm|cm3|cm³|\bcc\b|куб|cubic|displacement)", text, re.IGNORECASE):
parsed = cls._engine_volume_from_cc_text(text)
if parsed:
return parsed
if re.search(r"\b(?:liter|litre|l)\b", text, re.IGNORECASE):
parsed = cls._engine_volume_from_liter_text(text)
if parsed:
return parsed
continue
parsed = cls._int_from_text(value)
if parsed and 500 <= parsed <= 9000:
return parsed
for text in fallback_texts:
match = re.search(r"(?<![\w])([1-6])[\.,](\d{1,2})(?!\d)", text.lower())
if match:
liters = float(f"{match.group(1)}.{match.group(2)}")
return int(round(liters * 1000))
return None
@classmethod
def _engine_volume_from_cc_text(cls, value: str) -> int | None:
text = str(value or "")
match = re.search(r"(\d{1,2}(?:[\s.,]\d{3})|\d{3,5})\s*(?:ccm|cm3|cm³|cc|куб)", text, re.IGNORECASE)
if not match:
return None
parsed = cls._int_from_text(match.group(1))
if parsed and 500 <= parsed <= 9000:
return parsed
return None
@classmethod
def _engine_volume_from_liter_text(cls, value: str) -> int | None:
text = str(value or "")
match = re.search(r"(?<!\d)([1-8])(?:[\.,](\d{1,2}))?\s*(?:l|liter|litre)(?![a-z])", text, re.IGNORECASE)
if not match:
return None
decimals = (match.group(2) or "0").ljust(1, "0")
liters = float(f"{match.group(1)}.{decimals}")
parsed = int(round(liters * 1000))
if 500 <= parsed <= 9000:
return parsed
return None
@staticmethod
def _normalize_color(value: Any) -> str:
text = "" if value is None else str(value).lower().strip()
text = text.replace("ё", "е")
text = text.replace("ä", "a").replace("ö", "o").replace("ü", "u").replace("ß", "ss")
text = re.sub(r"[_\-/]+", " ", text)
for marker, mapped in _COLOR_MAP.items():
if marker in text:
return mapped
return text[:50] if text else "other"
@staticmethod
def _is_one_owner(value: Any) -> bool:
text = "" if value is None else str(value).strip().lower()
return text in {"1", "01", "1.0", "one", "one owner", "1 owner", "1 previous owner"}
@staticmethod
def _rating_text(value: Any) -> str | None:
if isinstance(value, dict):
for key in ("rating", "ratingLabel", "label", "value"):
text = MobileDeMapper._text(value.get(key))
if text:
return text
return None
text = MobileDeMapper._text(value)
return text or None
@staticmethod
def _slugify(value: str) -> str:
slug = re.sub(r"[^a-zA-Z0-9а-яА-ЯёЁ]+", "-", value.lower()).strip("-")
@@ -304,38 +655,69 @@ class MobileDeMapper:
@staticmethod
def _images_from_listing(raw: dict[str, Any]) -> list[ImageRecord]:
urls: list[str] = []
image = raw.get("image")
if isinstance(image, str):
urls.append(MobileDeMapper._normalize_image_url(image))
images = raw.get("images")
if isinstance(images, list):
for item in images:
if isinstance(item, str):
urls.append(MobileDeMapper._normalize_image_url(item))
elif isinstance(item, dict):
src = item.get("src") or item.get("url") or item.get("uri")
if src:
urls.append(MobileDeMapper._normalize_image_url(str(src)))
media_gallery = raw.get("mediaGallery")
if isinstance(media_gallery, list):
for item in media_gallery:
if isinstance(item, dict):
src = item.get("uri") or item.get("url")
if src:
urls.append(MobileDeMapper._normalize_image_url(str(src)))
urls = MobileDeMapper._extract_image_urls(raw)
return [
ImageRecord(fullres_image=url, preview_image=url, order_index=index)
for index, url in enumerate(dict.fromkeys(url for url in urls if url))
]
@staticmethod
def _extract_image_urls(value: Any, *, parent_key: str = "", depth: int = 0) -> list[str]:
if depth > 8:
return []
urls: list[str] = []
parent_hint = MobileDeMapper._has_image_field_hint(parent_key)
if isinstance(value, str):
if parent_hint or MobileDeMapper._looks_like_image_url(value):
normalized = MobileDeMapper._normalize_image_url(value)
if MobileDeMapper._looks_like_image_url(normalized):
urls.append(normalized)
return urls
if isinstance(value, list):
for item in value:
urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=parent_key, depth=depth + 1))
return urls
if isinstance(value, dict):
for key, item in value.items():
key_text = str(key or "")
urls.extend(MobileDeMapper._extract_image_urls(item, parent_key=key_text, depth=depth + 1))
return urls
return urls
@staticmethod
def _has_image_field_hint(value: str) -> bool:
normalized = re.sub(r"[^a-z0-9]", "", str(value or "").lower())
return normalized in _IMAGE_FIELD_HINTS or any(hint in normalized for hint in _IMAGE_FIELD_HINTS)
@staticmethod
def _looks_like_image_url(value: str) -> bool:
text = str(value or "").strip().lower()
if not text:
return False
if not (text.startswith("http://") or text.startswith("https://") or text.startswith("//") or text.startswith("/")):
return False
return any(marker in text for marker in _IMAGE_URL_MARKERS)
@staticmethod
def _normalize_image_url(value: str) -> str:
url = str(value).strip()
if not url:
return ""
if url.startswith("//"):
return f"https:{url}"
if url.startswith("http://") or url.startswith("https://"):
url = f"https:{url}"
elif not (url.startswith("http://") or url.startswith("https://")):
url = f"https://{url.lstrip('/')}"
return MobileDeMapper._normalize_mobilede_image_rule(url)
@staticmethod
def _normalize_mobilede_image_rule(url: str) -> str:
parsed = urlsplit(url)
if "img.classistatic.de" not in parsed.netloc.lower():
return url
return f"https://{url.lstrip('/')}"
if "/api/v1/mo-prod/images/" not in parsed.path:
return url
query_pairs = parse_qsl(parsed.query, keep_blank_values=True)
if any(key.lower() == "rule" for key, _value in query_pairs):
return url
query_pairs.append(("rule", _MOBILEDE_IMAGE_RULE))
return urlunsplit((parsed.scheme, parsed.netloc, parsed.path, urlencode(query_pairs), parsed.fragment))

View File

@@ -4,6 +4,7 @@ import logging
import os
from collections.abc import Callable
from dataclasses import asdict
from datetime import datetime, timezone
from typing import Any
import requests
@@ -19,6 +20,11 @@ logger = logging.getLogger("mobile_de.scraper")
MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_STOP_ON_EXISTING_STREAK", "2")))
MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS = max(0, int(os.getenv("MOBILEDE_ONLY_NEW_MIN_NEW_RECORDS", "1")))
MOBILEDE_LIGHT_REFRESH_EXISTING = os.getenv("MOBILEDE_LIGHT_REFRESH_EXISTING", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED = os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED = os.getenv("MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", "20")))
MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE = max(0, int(os.getenv("MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", "2")))
MOBILEDE_SEARCH_STRATEGY_NOTE = (
"mobile.de search pages return about 20 listings per page and are limited to about 50 pages; "
"for full coverage split into narrower segments and deduplicate by id."
@@ -92,6 +98,32 @@ class MobileDeScraper:
deduped_page_records.append(record)
return deduped_page_records
@staticmethod
def _record_key(record: CarRecord) -> str:
return record.origin_id or record.origin_url
@staticmethod
def _record_needs_detail_enrich(record: CarRecord) -> bool:
return any(
(
record.year is None,
record.mileage == 0,
record.engine_volume is None,
record.body_type == "OTHER",
record.color == "other",
)
)
@staticmethod
def _detail_enrich_priority(record: CarRecord) -> tuple[int, int, int, int, int]:
return (
int(record.year is None),
int(record.engine_volume is None),
int(record.mileage == 0),
int(record.body_type == "OTHER"),
int(record.color == "other"),
)
def _apply_only_new_page_policy(
self,
*,
@@ -102,13 +134,15 @@ class MobileDeScraper:
skipped_existing: int,
existing_streak: int,
new_records_kept: int,
existing_origin_ids: set[str] | None = None,
) -> tuple[list[CarRecord], int, int, int, bool]:
if not only_new or not page_records:
return page_records, skipped_existing, existing_streak, new_records_kept, False
existing_origin_ids = self.persistence.get_existing_origin_ids(
[record.origin_id for record in page_records if record.origin_id]
)
if existing_origin_ids is None:
existing_origin_ids = self.persistence.get_existing_origin_ids(
[record.origin_id for record in page_records if record.origin_id]
)
head_cut_triggered = False
should_cut_tail = self._should_cut_only_new_tail(sort_by, sort_order)
@@ -289,10 +323,14 @@ class MobileDeScraper:
mileage_max: str | None = None,
sort_by: str | None = None,
sort_order: str | None = None,
seen_at: datetime | None = None,
progress_callback: Callable[[str, dict[str, Any]], None] | None = None,
) -> dict[str, Any]:
self.persistence.create_tables()
run_id = self.persistence.start_sync_run(lane)
run_seen_at = seen_at or datetime.now(timezone.utc)
if run_seen_at.tzinfo is None:
run_seen_at = run_seen_at.replace(tzinfo=timezone.utc)
pages_payload: list[dict[str, Any]] = []
unique_ids: set[str] = set()
listing_count = 0
@@ -302,6 +340,8 @@ class MobileDeScraper:
inserted_total = 0
updated_total = 0
images_upserted = 0
detail_enriched = 0
detail_enrich_failed = 0
existing_streak = 0
new_records_kept = 0
head_cut_triggered = False
@@ -340,20 +380,48 @@ class MobileDeScraper:
pages_collected = 0
early_stopped = False
for page in self.client.iter_search_pages(
start_page=start_page,
max_pages=max_pages,
search_url=search_url,
progress_callback=_on_page,
**params,
):
concurrent_pages = max(1, int(os.getenv("MOBILEDE_CONCURRENT_PAGES", "1")))
if concurrent_pages > 1 and max_pages and max_pages > 1 and only_new is not True:
page_iterator = self.client.fetch_search_pages_concurrent(
start_page=start_page,
max_pages=max_pages,
workers=concurrent_pages,
search_url=search_url,
progress_callback=_on_page,
**params,
)
else:
page_iterator = self.client.iter_search_pages(
start_page=start_page,
max_pages=max_pages,
search_url=search_url,
progress_callback=_on_page,
**params,
)
for page in page_iterator:
pages_collected += 1
pages_payload.append(asdict(page))
listing_count += len(page.listings)
unique_ids.update(str(listing.id) for listing in page.listings if listing.id)
page_records = [self.mapper.listing_to_car_record(listing) for listing in page.listings]
listing_by_record_key = {
self._record_key(record): listing
for listing, record in zip(page.listings, page_records, strict=False)
if self._record_key(record)
}
page_records = self._dedupe_page_records(page_records, seen_record_keys)
for record in page_records:
record.is_sold = False
record.first_seen_at = run_seen_at
record.last_seen_at = run_seen_at
record.sold_at = None
record.skip_image_sync = False
existing_origin_ids: set[str] = set()
if page_records and (only_new or MOBILEDE_LIGHT_REFRESH_EXISTING or MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED):
existing_origin_ids = self.persistence.get_existing_origin_ids(
[record.origin_id for record in page_records if record.origin_id]
)
page_records, skipped_existing, existing_streak, new_records_kept, head_cut_triggered = (
self._apply_only_new_page_policy(
page_records=page_records,
@@ -363,8 +431,73 @@ class MobileDeScraper:
skipped_existing=skipped_existing,
existing_streak=existing_streak,
new_records_kept=new_records_kept,
existing_origin_ids=existing_origin_ids,
)
)
if MOBILEDE_LIGHT_REFRESH_EXISTING and existing_origin_ids:
for record in page_records:
if record.origin_id and record.origin_id in existing_origin_ids:
record.skip_image_sync = True
if (
MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED
and detail_enriched < MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN
and page_records
):
remaining_budget = MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN - detail_enriched
page_budget = min(MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE, remaining_budget)
candidate_records = [
record
for record in page_records
if record.origin_id
and record.origin_id not in existing_origin_ids
and self._record_needs_detail_enrich(record)
]
candidate_records.sort(key=self._detail_enrich_priority, reverse=True)
selected_keys = {
self._record_key(record)
for record in candidate_records[:page_budget]
}
if selected_keys:
enriched_records: list[CarRecord] = []
for record in page_records:
record_key = self._record_key(record)
listing = listing_by_record_key.get(record_key)
if record_key not in selected_keys or listing is None:
enriched_records.append(record)
continue
try:
if progress_callback is not None:
progress_callback(
"detail_enriching",
{
"run_id": run_id,
"pages_collected": pages_collected,
"page_number": page.page_number,
"listing_id": str(listing.id),
"detail_enriched": detail_enriched,
"detail_enrich_failed": detail_enrich_failed,
},
)
enriched = self.mapper.detail_to_car_record(str(listing.id), self.client.fetch_detail(listing.id))
enriched.is_sold = False
enriched.first_seen_at = run_seen_at
enriched.last_seen_at = run_seen_at
enriched.sold_at = None
enriched.skip_image_sync = False
if not MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:
enriched.images = record.images
enriched_records.append(enriched)
detail_enriched += 1
except Exception:
logger.warning(
"mobile.de selective detail enrich failed: listing_id=%s title=%s",
getattr(listing, "id", None),
getattr(listing, "title", None),
exc_info=True,
)
detail_enrich_failed += 1
enriched_records.append(record)
page_records = enriched_records
if progress_callback is not None:
progress_callback(
@@ -372,6 +505,8 @@ class MobileDeScraper:
{
"record_count": len(page_records),
"skipped_existing": skipped_existing,
"detail_enriched": detail_enriched,
"detail_enrich_failed": detail_enrich_failed,
"only_new": bool(only_new),
"run_id": run_id,
"pages_collected": pages_collected,
@@ -462,6 +597,8 @@ class MobileDeScraper:
"updated": updated_total,
"images_upserted": images_upserted,
},
"detail_enriched": detail_enriched,
"detail_enrich_failed": detail_enrich_failed,
"skipped_existing": skipped_existing,
**data,
}

View File

@@ -1,9 +1,10 @@
import logging
import os
from contextlib import contextmanager
from datetime import datetime, timezone
from typing import Any, Iterator
from sqlalchemy import create_engine, delete, or_, select, text, update
from sqlalchemy import create_engine, delete, func, or_, select, text, update
from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.orm import Session, sessionmaker
@@ -12,12 +13,14 @@ from .models import Base, Car, Image, SyncRun
from .schemas import CarRecord
logger = logging.getLogger("MOBILEDE_scraper.db")
MOBILEDE_SKIP_IMAGES_FOR_UPDATED = os.getenv("MOBILEDE_SKIP_IMAGES_FOR_UPDATED", "1").strip().lower() in {"1", "true", "yes", "on"}
CAR_DB_FIELDS = {
col.key for col in Car.__table__.columns
if col.key not in ("id",)
}
CAR_UPDATE_FIELDS = CAR_DB_FIELDS - {"first_seen_at"}
_IN_CHUNK_SIZE = 5000
CAR_TABLE_NAME = Car.__tablename__
@@ -168,6 +171,16 @@ class PersistenceService:
payload = record.model_dump(mode="python")
return {key: value for key, value in payload.items() if key in CAR_DB_FIELDS}
@staticmethod
def _apply_update_payload(car: Car, payload: dict[str, object]) -> None:
for key, value in payload.items():
if key in CAR_UPDATE_FIELDS:
setattr(car, key, value)
@staticmethod
def _skip_image_sync(record: CarRecord) -> bool:
return bool(getattr(record, "skip_image_sync", False))
def _is_postgres(self) -> bool:
return self.engine.dialect.name == "postgresql"
@@ -225,7 +238,7 @@ class PersistenceService:
def _postgres_upsert_set_map(insert_stmt) -> dict[str, object]:
return {
key: getattr(insert_stmt.excluded, key)
for key in CAR_DB_FIELDS
for key in CAR_UPDATE_FIELDS
}
def _replace_images_for_car(
@@ -264,17 +277,23 @@ class PersistenceService:
images = [image.model_dump(mode="python") for image in record.images]
car_by_id = existing_by_id.get(record.origin_id)
car_by_url = existing_by_url.get(record.origin_url)
entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
entry: dict[str, object] = {
"record": record,
"images": images,
"car_id": None,
"action": "inserted",
"skip_image_sync": self._skip_image_sync(record),
}
if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
for key, value in payload.items():
setattr(car_by_url, key, value)
car_by_url.last_seen_at = record.last_seen_at
self._apply_update_payload(car_by_url, payload)
entry["car_id"] = int(car_by_url.id)
entry["action"] = "updated"
updated += 1
else:
upsert_payloads.append(payload)
if car_by_id is not None:
entry["action"] = "updated"
updated += 1
else:
inserted += 1
@@ -299,34 +318,54 @@ class PersistenceService:
raise RuntimeError(f"PostgreSQL upsert did not return car_id for {record.origin_id}")
entry["car_id"] = car_id
car_ids = {int(entry["car_id"]) for entry in entries if entry["car_id"] is not None}
existing_images_map = self._load_existing_image_urls(session, car_ids)
images_by_car_id: dict[int, list[dict[str, object]]] = {}
replace_ids: list[int] = []
insert_images_by_car_id: dict[int, list[dict[str, object]]] = {}
updated_entries_needing_compare: list[dict[str, object]] = []
for entry in entries:
car_id = int(entry["car_id"])
action = str(entry.get("action") or "updated")
images = entry["images"]
new_image_urls = {
str(img.get("fullres_image", ""))
for img in images
if img.get("fullres_image")
}
old_image_urls = existing_images_map.get(car_id, set())
if new_image_urls != old_image_urls:
replace_ids.append(car_id)
images_by_car_id[car_id] = images
else:
images_total += len(old_image_urls)
skip_image_sync = bool(entry.get("skip_image_sync"))
if action == "updated" and (MOBILEDE_SKIP_IMAGES_FOR_UPDATED or skip_image_sync):
continue
if action == "inserted":
insert_images_by_car_id[car_id] = images
continue
updated_entries_needing_compare.append(entry)
if replace_ids:
for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
for car_id in replace_ids:
images = images_by_car_id[car_id]
self._add_images(session, car_id, images)
images_total += len(images)
for car_id, images in insert_images_by_car_id.items():
self._add_images(session, car_id, images)
images_total += len(images)
if updated_entries_needing_compare:
car_ids = {int(entry["car_id"]) for entry in updated_entries_needing_compare}
existing_images_map = self._load_existing_image_urls(session, car_ids)
images_by_car_id: dict[int, list[dict[str, object]]] = {}
replace_ids: list[int] = []
for entry in updated_entries_needing_compare:
car_id = int(entry["car_id"])
images = entry["images"]
new_image_urls = {
str(img.get("fullres_image", ""))
for img in images
if img.get("fullres_image")
}
old_image_urls = existing_images_map.get(car_id, set())
if new_image_urls != old_image_urls:
replace_ids.append(car_id)
images_by_car_id[car_id] = images
else:
images_total += len(old_image_urls)
if replace_ids:
for i in range(0, len(replace_ids), _IN_CHUNK_SIZE):
chunk = replace_ids[i:i + _IN_CHUNK_SIZE]
session.execute(delete(Image).where(Image.car_id.in_(chunk)))
for car_id in replace_ids:
images = images_by_car_id[car_id]
self._add_images(session, car_id, images)
images_total += len(images)
return {"inserted": inserted, "updated": updated, "images_upserted": images_total}
@@ -340,9 +379,7 @@ class PersistenceService:
select(Car).where(Car.origin_url == record.origin_url)
).scalar_one_or_none()
if car_by_url is not None and car_by_url.origin_id != record.origin_id:
for key, value in payload.items():
setattr(car_by_url, key, value)
car_by_url.last_seen_at = record.last_seen_at
self._apply_update_payload(car_by_url, payload)
session.flush()
car_id = int(car_by_url.id)
action = "updated"
@@ -373,9 +410,7 @@ class PersistenceService:
session.flush()
else:
action = "updated"
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
self._apply_update_payload(car, payload)
session.flush()
images_upserted = self._replace_images_for_car(session, int(car.id), images, record.origin_id)
return {"car_id": int(car.id), "images_upserted": images_upserted, "action": action}
@@ -429,18 +464,8 @@ class PersistenceService:
existing_by_id, existing_by_url = self._load_existing_cars(session, origin_ids, origin_urls)
# Предзагружаем изображения.
existing_car_ids = set()
for record in records:
car = existing_by_id.get(record.origin_id) or existing_by_url.get(record.origin_url)
if car is not None:
existing_car_ids.add(int(car.id))
# Готовим map car_id -> image_urls.
existing_images_map = self._load_existing_image_urls(session, existing_car_ids)
new_cars: list[tuple[Car, list[dict]]] = []
update_cars_needing_images: list[tuple[Car, list[dict]]] = []
update_image_candidates: list[tuple[Car, list[dict]]] = []
for record in records:
payload = self._car_payload(record)
@@ -453,18 +478,11 @@ class PersistenceService:
inserted += 1
new_cars.append((car, images))
else:
for key, value in payload.items():
setattr(car, key, value)
car.last_seen_at = record.last_seen_at
self._apply_update_payload(car, payload)
updated += 1
# Проверяем изменения картинок.
new_image_urls = {img.get("fullres_image", "") for img in images}
old_image_urls = existing_images_map.get(int(car.id), set())
if new_image_urls != old_image_urls:
update_cars_needing_images.append((car, images))
else:
images_total += len(old_image_urls)
if MOBILEDE_SKIP_IMAGES_FOR_UPDATED or self._skip_image_sync(record):
continue
update_image_candidates.append((car, images))
# Один flush.
session.flush()
@@ -474,6 +492,18 @@ class PersistenceService:
self._add_images(session, int(car.id), images)
images_total += len(images)
update_cars_needing_images: list[tuple[Car, list[dict]]] = []
if update_image_candidates:
update_ids = [int(car.id) for car, _ in update_image_candidates]
existing_images_map = self._load_existing_image_urls(session, set(update_ids))
for car, images in update_image_candidates:
new_image_urls = {img.get("fullres_image", "") for img in images}
old_image_urls = existing_images_map.get(int(car.id), set())
if new_image_urls != old_image_urls:
update_cars_needing_images.append((car, images))
else:
images_total += len(old_image_urls)
# Обновляем только изменённые картинки.
if update_cars_needing_images:
update_ids = [int(car.id) for car, _ in update_cars_needing_images]
@@ -514,7 +544,7 @@ class PersistenceService:
.where(Car.origin_id.notin_(active_origin_ids))
.where(Car.is_sold == False) # noqa: E712
.where(_origin_prefix_filter(Car.origin_id))
.values(is_sold=True)
.values(is_sold=True, sold_at=datetime.now(timezone.utc))
)
result = session.execute(stmt)
count = result.rowcount or 0
@@ -591,7 +621,7 @@ class PersistenceService:
# Массовая пометка sold.
result = session.execute(text("""
UPDATE {car_table}
SET is_sold = TRUE
SET is_sold = TRUE, sold_at = NOW()
FROM (
SELECT c.id
FROM {car_table} c
@@ -609,7 +639,7 @@ class PersistenceService:
update(Car)
.where(Car.is_sold == False) # noqa: E712
.where(_origin_prefix_filter(Car.origin_id))
.values(is_sold=True)
.values(is_sold=True, sold_at=datetime.now(timezone.utc))
)
# Загружаем active URL.
all_active = session.execute(
@@ -631,7 +661,7 @@ class PersistenceService:
for i in range(0, len(mark_ids), _IN_CHUNK_SIZE):
chunk = mark_ids[i:i + _IN_CHUNK_SIZE]
session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True))
session.execute(update(Car).where(Car.id.in_(chunk)).values(is_sold=True, sold_at=datetime.now(timezone.utc)))
count = len(mark_ids)
if count:
@@ -692,7 +722,7 @@ class PersistenceService:
.where(_origin_prefix_filter(Car.origin_id, prefixes))
.where(Car.is_sold == False) # noqa: E712
.where(Car.last_seen_at < since_ts)
.values(is_sold=True)
.values(is_sold=True, sold_at=since_ts)
)
count = int(result.rowcount or 0)
if count:
@@ -755,3 +785,69 @@ class PersistenceService:
).order_by(Car.last_seen_at.asc()).offset(offset).limit(limit)
)
return [str(row[0]) for row in result if row and row[0]]
def get_active_cars_batch_for_sold_probe(
self,
prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
*,
limit: int = 200,
newest_first: bool = True,
) -> list[tuple[int, str, datetime]]:
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
order_by = Car.last_seen_at.desc() if newest_first else Car.last_seen_at.asc()
with self.session_scope() as session:
result = session.execute(
select(Car.id, Car.origin_url, Car.last_seen_at).where(
_origin_prefix_filter(Car.origin_id, prefixes),
Car.is_sold == False, # noqa: E712
).order_by(order_by).limit(limit)
)
return [
(int(row[0]), str(row[1]), row[2])
for row in result
if row and row[0] and row[1] and row[2]
]
def get_active_cars_batch_for_image_enrich(
self,
prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
*,
limit: int = 50,
max_existing_images: int = 1,
) -> list[tuple[int, str, str, int]]:
prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
with self.session_scope() as session:
image_count = func.count(Image.id)
result = session.execute(
select(Car.id, Car.origin_id, Car.origin_url, image_count.label("image_count"))
.outerjoin(Image, Image.car_id == Car.id)
.where(
_origin_prefix_filter(Car.origin_id, prefixes),
Car.is_sold == False, # noqa: E712
)
.group_by(Car.id, Car.origin_id, Car.origin_url)
.having(image_count <= max(0, int(max_existing_images)))
.order_by(Car.last_seen_at.desc())
.limit(max(1, int(limit)))
)
return [
(int(row[0]), str(row[1]), str(row[2]), int(row[3] or 0))
for row in result
if row and row[0] and row[1] and row[2]
]
def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int:
if not car_ids:
return 0
ts = sold_at or datetime.now(timezone.utc)
with self.session_scope() as session:
result = session.execute(
update(Car)
.where(Car.id.in_(car_ids))
.where(Car.is_sold == False) # noqa: E712
.values(is_sold=True, sold_at=ts)
)
count = int(result.rowcount or 0)
if count:
logger.info("Marked %d cars as sold by explicit id probe", count)
return count

View File

@@ -16,7 +16,44 @@ BODY_TYPE_ENUM_VALUES = (
"OTHER",
"NA",
)
COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "NA")
COUNTRY_ENUM_VALUES = (
"AT",
"BE",
"BG",
"CA",
"CH",
"CY",
"CZ",
"DE",
"DK",
"EE",
"ES",
"FI",
"FR",
"GB",
"GR",
"HR",
"HU",
"IE",
"IT",
"JP",
"KR",
"LI",
"LT",
"LU",
"LV",
"MT",
"NA",
"NL",
"NO",
"PL",
"PT",
"RO",
"SE",
"SI",
"SK",
"US",
)
ORIGIN_ENUM_VALUES = (
"MOBILEDE",
"MOBILE_DE",

View File

@@ -5,7 +5,6 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
from .enums import (
BODY_TYPE_ENUM_VALUES,
COUNTRY_ENUM_VALUES,
CURRENCY_ENUM_VALUES,
DRIVE_ENUM_VALUES,
GEARBOX_ENUM_VALUES,
@@ -33,7 +32,7 @@ class Car(Base):
price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
country: Mapped[str] = mapped_column(String(10), nullable=False, default="NA")
is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
@@ -54,7 +53,9 @@ class Car(Base):
rental: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
repair_history: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False)
slug: Mapped[str] = mapped_column(String(), nullable=False)
first_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
last_seen_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
sold_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True, index=True)
images: Mapped[list["Image"]] = relationship("Image", back_populates="car", cascade="all, delete-orphan")

View File

@@ -37,7 +37,10 @@ class CarRecord(BaseModel):
rental: bool = False
repair_history: bool = False
slug: str
first_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
last_seen_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
sold_at: datetime | None = None
skip_image_sync: bool = False
images: list[ImageRecord] = Field(default_factory=list)
@@ -82,6 +85,8 @@ class CarRead(BaseModel):
rental: bool = False
repair_history: bool = False
slug: str = ""
first_seen_at: datetime | None = None
last_seen_at: datetime | None = None
sold_at: datetime | None = None
images: list[ImageRead] = Field(default_factory=list)

View File

@@ -1,4 +1,4 @@
# Инициализация Celery-приложения и периодических задач.
# Celery-приложение и периодические задачи.
import json
import logging
@@ -11,11 +11,22 @@ from redis import Redis
from ..core.config import settings
from ..core.logs import setup_logging
from .constants import (
GLOBAL_DB_PROGRESS_TS_KEY,
GLOBAL_PROGRESS_TS_KEY,
MOBILEDE_RUNTIME_SEGMENTS_TASK,
MOBILEDE_SYNC_QUEUE,
MOBILEDE_SYNC_TASK_NAME,
)
logger = logging.getLogger("mobilede_scraper.worker.celery_app")
STARTUP_SYNC_DISPATCH_KEY = "mobilede:state:startup_sync_dispatched"
MOBILEDE_SYNC_QUEUE = "mobilede_sync"
PROGRESS_KEY_PREFIX = "mobilede:state:task_progress:"
MOBILEDE_SYNC_DETAIL_TASK = "mobilede.sync_detail"
MOBILEDE_ENRICH_IMAGES_TASK = "mobilede.enrich_images_batch"
STARTUP_SYNC_DISPATCH_TTL_SECONDS = 10 * 60
STARTUP_PROGRESS_MAX_AGE_SECONDS = 180
STARTUP_GLOBAL_PROGRESS_MAX_AGE_SECONDS = 300
def _env_bool(name: str, default: bool) -> bool:
@@ -23,7 +34,26 @@ def _env_bool(name: str, default: bool) -> bool:
return raw in {"1", "true", "yes", "on"}
def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 180) -> bool:
MOBILEDE_BEAT_SYNC_ENABLED = _env_bool("MOBILEDE_BEAT_SYNC_ENABLED", True)
def _runtime_sync_kwargs() -> dict[str, bool | float]:
return {
"delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
"use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
"continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
}
def _runtime_sync_expires_seconds() -> float:
return settings.celery.beat_sync_interval_minutes * 60.0
def _has_fresh_active_progress(
redis_client: Redis,
*,
max_age_seconds: int = STARTUP_PROGRESS_MAX_AGE_SECONDS,
) -> bool:
now = int(time.time())
try:
for raw_key in redis_client.scan_iter(f"{PROGRESS_KEY_PREFIX}*"):
@@ -43,17 +73,56 @@ def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 18
return False
def _has_recent_global_progress(
redis_client: Redis,
*,
max_age_seconds: int = STARTUP_GLOBAL_PROGRESS_MAX_AGE_SECONDS,
) -> bool:
now = int(time.time())
try:
progress_ts = int(redis_client.get(GLOBAL_PROGRESS_TS_KEY) or 0)
db_progress_ts = int(redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY) or 0)
except Exception:
logger.warning("Failed to inspect global startup progress keys", exc_info=True)
return False
freshest_ts = max(progress_ts, db_progress_ts)
return freshest_ts > 0 and now - freshest_ts <= max_age_seconds
def _has_live_startup_progress(redis_client: Redis, *, queue_len: int) -> bool:
if _has_recent_global_progress(redis_client):
return True
if queue_len <= 0:
return False
return _has_fresh_active_progress(redis_client)
def _claim_startup_dispatch(redis_client: Redis, *, reset_stale: bool) -> bool:
if redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=STARTUP_SYNC_DISPATCH_TTL_SECONDS):
return True
if not reset_stale:
return False
redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
return bool(
redis_client.set(
STARTUP_SYNC_DISPATCH_KEY,
"1",
nx=True,
ex=STARTUP_SYNC_DISPATCH_TTL_SECONDS,
)
)
@celery_setup_logging.connect
def _configure_logging(loglevel=None, **kwargs):
# Перехватываем логирование Celery и пишем только в stderr (Docker logs).
# Пишем логи Celery в stderr.
level = settings.log_level if settings.log_level else "INFO"
setup_logging(level, None)
@worker_process_init.connect
def _on_worker_process_init(**kwargs):
# Повторно настраиваем логирование в каждом дочернем prefork-процессе,
# чтобы StreamHandler(stderr) корректно работал после fork.
# Повторно настраиваем логирование после fork.
level = settings.log_level if settings.log_level else "INFO"
setup_logging(level, None)
@@ -72,8 +141,7 @@ celery_app = Celery(
backend=_result_backend(),
)
# Auto-clamp: если hard limit слишком далёк от soft (> soft + 120),
# ограничиваем, чтобы зависший worker не жил вечно.
# Если hard limit слишком большой, сжимаем его до soft + 120.
_soft = settings.celery.task_soft_time_limit
_hard = settings.celery.task_time_limit
_max_hard = _soft + 120 if _soft else _hard
@@ -85,6 +153,21 @@ if _hard > _max_hard:
)
_hard = _max_hard
beat_schedule = {}
if MOBILEDE_BEAT_SYNC_ENABLED:
beat_schedule = {
"periodic-mobilede-sync-search": {
"task": MOBILEDE_RUNTIME_SEGMENTS_TASK,
"schedule": _runtime_sync_expires_seconds(),
"args": (),
"kwargs": _runtime_sync_kwargs(),
"options": {
"queue": MOBILEDE_SYNC_QUEUE,
"expires": _runtime_sync_expires_seconds(),
},
}
}
celery_app.conf.update(
task_serializer="json",
accept_content=["json"],
@@ -107,26 +190,12 @@ celery_app.conf.update(
result_expires=86400,
worker_redirect_stdouts=False,
worker_hijack_root_logger=False,
beat_schedule={
"periodic-mobilede-sync-search": {
"task": "mobilede.sync_runtime_segments",
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
"args": (),
"kwargs": {
"delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
"use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
"continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
},
"options": {
"queue": MOBILEDE_SYNC_QUEUE,
"expires": settings.celery.beat_sync_interval_minutes * 60.0,
},
}
},
beat_schedule=beat_schedule,
task_routes={
"mobilede.sync_runtime_segments": {"queue": MOBILEDE_SYNC_QUEUE},
"mobilede.sync_search": {"queue": MOBILEDE_SYNC_QUEUE},
"mobilede.sync_detail": {"queue": MOBILEDE_SYNC_QUEUE},
MOBILEDE_RUNTIME_SEGMENTS_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
MOBILEDE_SYNC_TASK_NAME: {"queue": MOBILEDE_SYNC_QUEUE},
MOBILEDE_SYNC_DETAIL_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
MOBILEDE_ENRICH_IMAGES_TASK: {"queue": MOBILEDE_SYNC_QUEUE},
"mobilede_scraper.worker.tasks.*": {"queue": MOBILEDE_SYNC_QUEUE},
},
)
@@ -141,6 +210,7 @@ def _on_worker_ready(**kwargs):
logger.info("Worker ready: startup sync dispatch disabled by MOBILEDE_STARTUP_SYNC_ENABLED")
return
should_dispatch = False
redis_client = None
try:
redis_client = Redis.from_url(
@@ -152,22 +222,27 @@ def _on_worker_ready(**kwargs):
retry_on_timeout=True,
)
has_fresh_progress = _has_fresh_active_progress(redis_client)
queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
has_live_progress = _has_live_startup_progress(redis_client, queue_len=queue_len)
try:
queue_len = int(redis_client.llen(MOBILEDE_SYNC_QUEUE) or 0)
except Exception:
queue_len = 0
if queue_len > 0:
if queue_len > 0 and has_live_progress:
logger.info("Worker ready: MOBILEDE_sync queue already has %d task(s); skip startup dispatch", queue_len)
return
has_fresh_progress = _has_fresh_active_progress(redis_client)
if queue_len > 0 and not has_live_progress:
logger.warning(
"Worker ready: MOBILEDE_sync queue has %d task(s), but no fresh progress is visible; forcing runtime sync dispatch",
queue_len,
)
elif queue_len <= 0 and has_fresh_progress:
logger.info("Worker ready: queue is empty but active progress is still visible; relying on startup dedupe")
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
if not should_dispatch and not has_fresh_progress:
redis_client.delete(STARTUP_SYNC_DISPATCH_KEY)
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
if should_dispatch:
logger.info("Worker ready: stale startup dedupe key ignored because queue is empty and no fresh active progress exists")
should_dispatch = _claim_startup_dispatch(
redis_client,
reset_stale=not has_live_progress,
)
if should_dispatch and not has_live_progress:
logger.info("Worker ready: claimed startup dispatch after stale or missing dedupe state")
except Exception:
logger.warning("Worker ready startup sync dedupe check failed; skipping immediate dispatch", exc_info=True)
return
@@ -182,14 +257,10 @@ def _on_worker_ready(**kwargs):
logger.info("Worker ready immediate sync already dispatched recently; skipping duplicate enqueue")
return
logger.info("Worker ready dispatching initial mobile.de sync_search task")
logger.info("Worker ready - dispatching initial mobile.de sync_runtime_segments task")
celery_app.send_task(
"mobilede.sync_runtime_segments",
kwargs={
"delay_seconds": float(os.getenv("MOBILEDE_REQUEST_DELAY_SECONDS", "0.7")),
"use_cursor": _env_bool("MOBILEDE_CURSOR_ENABLED", True),
"continuous": _env_bool("MOBILEDE_CONTINUOUS_SYNC_ENABLED", True),
},
MOBILEDE_RUNTIME_SEGMENTS_TASK,
kwargs=_runtime_sync_kwargs(),
queue=MOBILEDE_SYNC_QUEUE,
expires=settings.celery.beat_sync_interval_minutes * 60.0,
expires=_runtime_sync_expires_seconds(),
)

View File

@@ -16,6 +16,7 @@ MOBILEDE_CONTINUOUS_SYNC_ENABLED = os.getenv("MOBILEDE_CONTINUOUS_SYNC_ENABLED",
MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_CONTINUOUS_SYNC_DELAY_SECONDS", "15"))))
MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS = max(60, int(float(os.getenv("MOBILEDE_FULL_PASS_REPEAT_DELAY_SECONDS", "3600"))))
MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS", "5"))))
MOBILEDE_ANTIBOT_BACKOFF_SECONDS = max(300, int(float(os.getenv("MOBILEDE_ANTIBOT_BACKOFF_SECONDS", "1800"))))
MOBILEDE_PROGRESS_LOG_EVERY_PAGES = max(1, int(os.getenv("MOBILEDE_PROGRESS_LOG_EVERY_PAGES", "10")))
MOBILEDE_SKIP_EMPTY_WINDOW = os.getenv("MOBILEDE_SKIP_EMPTY_WINDOW", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_ROTATE_RUNTIME_SEGMENTS = os.getenv("MOBILEDE_ROTATE_RUNTIME_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
@@ -31,14 +32,19 @@ MOBILEDE_DYNAMIC_SEGMENT_PROBES = os.getenv("MOBILEDE_DYNAMIC_SEGMENT_PROBES", "
MOBILEDE_PREPLAN_SEGMENT_PROBES = os.getenv("MOBILEDE_PREPLAN_SEGMENT_PROBES", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_PREPLAN_MAX_SEGMENTS = max(1, int(os.getenv("MOBILEDE_PREPLAN_MAX_SEGMENTS", "1000")))
MOBILEDE_PREPLAN_MAX_PROBES = max(0, int(os.getenv("MOBILEDE_PREPLAN_MAX_PROBES", "40")))
MOBILEDE_ADAPTIVE_URL_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_ADAPTIVE_URL_MAX_SECONDS", "300"))))
MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = min(
5.0,
max(1.0, float(os.getenv("MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO", "2.5"))),
)
MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = max(0, min(3, int(os.getenv("MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH", "1"))))
MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS = max(0, int(float(os.getenv("MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS", "180"))))
MOBILEDE_COMPACT_SEGMENTS = os.getenv("MOBILEDE_COMPACT_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = os.getenv("MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE", "false").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS = os.getenv("MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_SEGMENTS_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED = os.getenv("MOBILEDE_RUNTIME_BRAND_ROOT_PROBES_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS = max(300, int(os.getenv("MOBILEDE_SITE_MAKE_OPTIONS_CACHE_TTL_SECONDS", str(6 * 60 * 60))))
MOBILEDE_HOT_BASE_SPLIT_ENABLED = os.getenv("MOBILEDE_HOT_BASE_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_HOT_BASE_PRICE_MAX = max(5000, int(os.getenv("MOBILEDE_HOT_BASE_PRICE_MAX", "30000")))
MOBILEDE_HOT_RECENT_YEAR_MIN = max(2000, int(os.getenv("MOBILEDE_HOT_RECENT_YEAR_MIN", "2018")))
@@ -80,6 +86,10 @@ MOBILEDE_REFRESH_CYCLE_DONE_KEY = "mobilede:state:refresh_cycle:done"
MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT = "mobilede:state:refresh_cycle:done_segments:{cycle_id}"
MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT = "mobilede:state:refresh_cycle:finalized:{cycle_id}"
MOBILEDE_REFRESH_CYCLE_TTL_SECONDS = max(60 * 60, int(os.getenv("MOBILEDE_REFRESH_CYCLE_TTL_SECONDS", str(24 * 60 * 60))))
MOBILEDE_OVERFLOW_CHILDREN_QUEUED_KEY_FMT = "mobilede:state:overflow_children_queued:{scope}:{parent_key}"
MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED = os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE = max(0, int(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_BATCH_SIZE", "200")))
MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS = max(0, int(float(os.getenv("MOBILEDE_POST_REFRESH_SOLD_PROBE_DELAY_SECONDS", "5"))))
MOBILEDE_OVERFLOW_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_OVERFLOW_SPLIT_THRESHOLD_RATIO = min(
@@ -98,6 +108,59 @@ MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH = max(
1,
min(6, int(os.getenv("MOBILEDE_OVERFLOW_MAX_SPLIT_DEPTH", "6"))),
)
MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED = os.getenv("MOBILEDE_OVERFLOW_SMART_SPLIT_ENABLED", "true").strip().lower() in {"1", "true", "yes", "on"}
MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES = max(
1,
min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CANDIDATES", "3"))),
)
MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN = max(
1,
min(4, int(os.getenv("MOBILEDE_OVERFLOW_SPLIT_PROBE_CHILDREN", "3"))),
)
MOBILEDE_SEGMENT_TARGET_MIN_RATIO = min(
1.0,
max(0.2, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MIN_RATIO", "0.65"))),
)
MOBILEDE_SEGMENT_TARGET_MAX_RATIO = min(
1.2,
max(0.5, float(os.getenv("MOBILEDE_SEGMENT_TARGET_MAX_RATIO", "0.98"))),
)
MOBILEDE_SEGMENT_TINY_RATIO = min(
0.8,
max(0.1, float(os.getenv("MOBILEDE_SEGMENT_TINY_RATIO", "0.45"))),
)
MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN = max(
1,
min(8, int(os.getenv("MOBILEDE_OVERFLOW_MIN_YEAR_SPLIT_SPAN", "4"))),
)
MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH = max(
0,
min(6, int(os.getenv("MOBILEDE_OVERFLOW_YEAR_DEEP_SPLIT_DEPTH", "1"))),
)
MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY = max(
0,
int(os.getenv("MOBILEDE_OVERFLOW_SCORE_DEPTH_PENALTY", "220")),
)
MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY = max(
0,
int(os.getenv("MOBILEDE_OVERFLOW_SCORE_YEAR_PENALTY", "320")),
)
MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY = max(
0,
int(os.getenv("MOBILEDE_OVERFLOW_SCORE_MILEAGE_PENALTY", "80")),
)
MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY = max(
0,
int(os.getenv("MOBILEDE_OVERFLOW_SCORE_MICRO_CHILD_PENALTY", "420")),
)
MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO = min(
1.0,
max(0.2, float(os.getenv("MOBILEDE_OVERFLOW_MIN_USEFUL_CHILD_RATIO", "0.55"))),
)
MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN = max(
0,
min(3, int(os.getenv("MOBILEDE_OVERFLOW_MAX_MICRO_CHILDREN", "1"))),
)
MOBILEDE_INCREMENTAL_CYCLE_KEY = "mobilede:state:incremental_cycle"
MOBILEDE_INCREMENTAL_CYCLE_SEEN_COUNT_KEY = "mobilede:state:incremental_cycle_seen_count"
@@ -116,6 +179,7 @@ STALL_WATCHDOG_NAVIGATION_STAGES = {
STALL_WATCHDOG_LONG_RUNNING_STAGES = {
"search_collection_done",
"records_mapped",
"detail_enriching",
}
STALL_WATCHDOG_NAVIGATION_GRACE_SECONDS = max(
300,

File diff suppressed because it is too large Load Diff

View File

@@ -79,9 +79,7 @@ def _update_task_progress(
json.dumps(data, ensure_ascii=False),
ex=ttl,
)
# Глобальный маркер активности для внешнего guard-процесса.
# Нужен, чтобы контейнер мог самовосстанавливаться при полном зависании воркера
# (когда PID жив, но прогресс по задачам не двигается).
# Глобальный маркер активности для self-heal.
pipe.set(GLOBAL_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))
if stage in DB_PROGRESS_STAGES:
pipe.set(GLOBAL_DB_PROGRESS_TS_KEY, str(now_ts), ex=max(ttl, 7 * 24 * 60 * 60))

View File

@@ -0,0 +1,260 @@
from __future__ import annotations
import logging
import uuid
from datetime import datetime, timezone
from typing import Callable
from redis import Redis
from .constants import (
MOBILEDE_REFRESH_CYCLE_DONE_KEY,
MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT,
MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT,
MOBILEDE_REFRESH_CYCLE_ID_KEY,
MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
MOBILEDE_REFRESH_CYCLE_TTL_SECONDS,
)
def _mobilede_refresh_cycle_done_set_key(cycle_id: str) -> str:
return MOBILEDE_REFRESH_CYCLE_DONE_SEGMENTS_KEY_FMT.format(cycle_id=cycle_id)
def _mobilede_refresh_cycle_finalized_key(cycle_id: str) -> str:
return MOBILEDE_REFRESH_CYCLE_FINALIZED_KEY_FMT.format(cycle_id=cycle_id)
def _mobilede_redis_text(value: object) -> str:
if isinstance(value, bytes):
return value.decode("utf-8", errors="ignore")
return str(value or "")
def _mobilede_claim_refresh_cycle_finalization(redis_client: Redis, *, cycle_id: str) -> bool:
ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
return bool(
redis_client.set(
_mobilede_refresh_cycle_finalized_key(cycle_id),
"1",
nx=True,
ex=ttl,
)
)
def _mobilede_clear_refresh_cycle(redis_client: Redis) -> None:
cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
keys = [
MOBILEDE_REFRESH_CYCLE_ID_KEY,
MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY,
MOBILEDE_REFRESH_CYCLE_TOTAL_KEY,
MOBILEDE_REFRESH_CYCLE_DONE_KEY,
]
if cycle_id:
keys.append(_mobilede_refresh_cycle_done_set_key(cycle_id))
keys.append(_mobilede_refresh_cycle_finalized_key(cycle_id))
redis_client.delete(*keys)
def _mobilede_start_refresh_cycle(
redis_client: Redis,
*,
total_segments: int,
logger: logging.Logger,
) -> str:
cycle_id = uuid.uuid4().hex[:16]
started_at = datetime.now(timezone.utc).isoformat()
total = max(0, int(total_segments))
ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
pipe = redis_client.pipeline()
pipe.set(MOBILEDE_REFRESH_CYCLE_ID_KEY, cycle_id, ex=ttl)
pipe.set(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY, started_at, ex=ttl)
pipe.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total), ex=ttl)
pipe.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, "0", ex=ttl)
pipe.delete(done_set_key)
pipe.delete(_mobilede_refresh_cycle_finalized_key(cycle_id))
pipe.execute()
logger.info("mobile.de refresh cycle started: id=%s total=%s started_at=%s", cycle_id, total, started_at)
return cycle_id
def _mobilede_get_or_start_refresh_cycle(
redis_client: Redis,
*,
total_segments: int,
logger: logging.Logger,
) -> str:
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id:
done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments or 0)
if total_now <= 0 or done_now < total_now:
return active_cycle_id
return _mobilede_start_refresh_cycle(redis_client, total_segments=total_segments, logger=logger)
def _mobilede_refresh_cycle_seen_at(redis_client: Redis, *, cycle_id: str | None, logger: logging.Logger) -> datetime | None:
if not cycle_id:
return None
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id != cycle_id:
return None
started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
if not started_at_raw:
return None
try:
seen_at = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
if seen_at.tzinfo is None:
seen_at = seen_at.replace(tzinfo=timezone.utc)
return seen_at
except Exception:
logger.warning("mobile.de refresh cycle seen_at is invalid: cycle=%s value=%s", cycle_id, started_at_raw)
return None
def _mobilede_refresh_cycle_progress(
redis_client: Redis,
*,
cycle_id: str | None,
total_segments_hint: int = 0,
) -> tuple[int, int, int]:
if not cycle_id:
return 0, max(0, int(total_segments_hint)), 0
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id != cycle_id:
return 0, max(0, int(total_segments_hint)), 0
done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
if total_now <= 0:
total_now = max(done_now, int(total_segments_hint or 0))
left_now = max(0, total_now - min(done_now, total_now)) if total_now > 0 else 0
return done_now, total_now, left_now
def _mobilede_track_refresh_cycle_segment(
redis_client: Redis,
*,
cycle_id: str | None,
segment: dict[str, object] | None,
total_segments_hint: int,
segment_fingerprint: Callable[[dict[str, object] | None], str],
) -> tuple[int, int, bool]:
if not cycle_id or not segment:
return 0, max(0, int(total_segments_hint)), False
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id != cycle_id:
return 0, 0, False
done_set_key = _mobilede_refresh_cycle_done_set_key(cycle_id)
if int(redis_client.sadd(done_set_key, segment_fingerprint(segment)) or 0) <= 0:
done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
return done_now, total_now, False
ttl = max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS))
redis_client.expire(done_set_key, ttl)
done_now = int(redis_client.incr(MOBILEDE_REFRESH_CYCLE_DONE_KEY))
redis_client.expire(MOBILEDE_REFRESH_CYCLE_DONE_KEY, ttl)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or total_segments_hint or 0)
if total_now <= 0:
total_now = max(done_now, int(total_segments_hint or 0))
redis_client.set(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY, str(total_now), ex=ttl)
if total_now > 0 and done_now > total_now:
done_now = total_now
redis_client.set(MOBILEDE_REFRESH_CYCLE_DONE_KEY, str(done_now), ex=ttl)
should_finalize = False
if total_now > 0 and done_now >= total_now:
should_finalize = _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id)
return done_now, total_now, should_finalize
def _mobilede_finalize_refresh_cycle_sold_marking(
redis_client: Redis,
*,
cycle_id: str,
logger: logging.Logger,
get_persistence: Callable[[], object],
origin_prefixes: tuple[str, ...],
post_refresh_probe_enabled: bool,
post_refresh_probe_batch_size: int,
schedule_post_refresh_probe: Callable[[], None],
) -> int:
started_at_raw = redis_client.get(MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY)
if not started_at_raw:
logger.warning("mobile.de refresh sold marking skipped: missing started_at for cycle=%s", cycle_id)
return 0
try:
cutoff = datetime.fromisoformat(_mobilede_redis_text(started_at_raw))
if cutoff.tzinfo is None:
cutoff = cutoff.replace(tzinfo=timezone.utc)
except Exception:
logger.warning(
"mobile.de refresh sold marking skipped: invalid started_at=%s cycle=%s",
started_at_raw,
cycle_id,
)
return 0
sold_marked = get_persistence().mark_sold_not_seen_since(
cutoff,
prefix=origin_prefixes,
safety_ratio=0.8,
)
done_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_DONE_KEY) or 0)
total_now = int(redis_client.get(MOBILEDE_REFRESH_CYCLE_TOTAL_KEY) or 0)
logger.info(
"mobile.de refresh sold marking completed: cycle=%s progress=%s/%s cutoff=%s sold_marked=%s",
cycle_id,
done_now,
total_now,
cutoff.isoformat(),
sold_marked,
)
if post_refresh_probe_enabled and post_refresh_probe_batch_size > 0:
try:
schedule_post_refresh_probe()
except Exception:
logger.warning(
"mobile.de post-refresh sold probe scheduling failed: cycle=%s",
cycle_id,
exc_info=True,
)
return sold_marked
def _mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
redis_client: Redis,
*,
cycle_id: str | None,
total_segments_hint: int = 0,
logger: logging.Logger,
finalize_refresh_cycle_sold_marking: Callable[[Redis], int] | Callable[..., int],
) -> bool:
if not cycle_id:
return False
active_cycle_id = _mobilede_redis_text(redis_client.get(MOBILEDE_REFRESH_CYCLE_ID_KEY)).strip()
if active_cycle_id != cycle_id:
return False
done_now, total_now, _left_now = _mobilede_refresh_cycle_progress(
redis_client,
cycle_id=cycle_id,
total_segments_hint=total_segments_hint,
)
if total_now <= 0:
return False
if done_now < total_now:
redis_client.set(
MOBILEDE_REFRESH_CYCLE_DONE_KEY,
str(total_now),
ex=max(3600, int(MOBILEDE_REFRESH_CYCLE_TTL_SECONDS)),
)
logger.warning(
"mobile.de refresh finalize fallback: bootstrap completed while refresh progress lagged cycle=%s done=%s/%s",
cycle_id,
done_now,
total_now,
)
if not _mobilede_claim_refresh_cycle_finalization(redis_client, cycle_id=cycle_id):
return False
finalize_refresh_cycle_sold_marking(redis_client, cycle_id=cycle_id)
return True

File diff suppressed because it is too large Load Diff

View File

@@ -63,8 +63,7 @@ def _read_last_progress_ts(redis_client: Redis) -> int | None:
if ts > 0:
return ts
# Fallback: если глобальный ключ не найден, берём max(ts) из task_progress:*.
# Это дороже, но выполняется только при отсутствии основного маркера.
# Резервно берём max(ts) из task_progress:*.
max_ts = 0
for key in redis_client.scan_iter(match="mobilede:state:task_progress:*"):
try:
@@ -153,7 +152,7 @@ def _kill_worker_process() -> None:
time.sleep(20)
try:
# Если процесс ещё жив — принудительно убиваем.
# Если процесс жив, добиваем SIGKILL.
os.kill(pid, 0)
logger.error("Self-heal: worker pid=%s did not stop after SIGTERM; sending SIGKILL", pid)
os.kill(pid, SIGKILL_FALLBACK)
@@ -230,7 +229,7 @@ def main() -> None:
db_idle_restart = True
restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s"
# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
# Не даём нескольким воркерам рестартовать одновременно.
acquired = bool(
redis_client.set(
SELF_HEAL_RESTART_LOCK_KEY,
@@ -251,12 +250,10 @@ def main() -> None:
if db_idle_restart:
logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1")
_reset_bootstrap_checkpoint_for_db_idle(redis_client)
# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
# перезапуск был не строго одновременно.
# Добавляем небольшой джиттер перед рестартом.
time.sleep(random.uniform(0.3, 2.0))
_kill_worker_process()
# После kill pid1 контейнер будет перезапущен Docker restart-policy.
# На случай неуспеха не молотим цикл.
# Даём Docker время на рестарт.
time.sleep(check_interval)
except Exception:

File diff suppressed because it is too large Load Diff

View File

@@ -6,7 +6,7 @@
"ids_max_pages": null,
"condition_check_enabled": false,
"lane": "MOBILEDE_cars",
"only_new": false,
"only_new": true,
"limit": null
},
"filters": {

View File

@@ -2,6 +2,7 @@
import tempfile
import unittest
from datetime import datetime, timedelta, timezone
from pathlib import Path
from sqlalchemy import select
@@ -48,6 +49,14 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
],
)
@staticmethod
def _as_utc(value: datetime | None) -> datetime | None:
if value is None:
return None
if value.tzinfo is None:
return value.replace(tzinfo=timezone.utc)
return value.astimezone(timezone.utc)
def test_insert_update_and_skip_flow(self) -> None:
first = self._record("777", price=1000)
inserted = self.persistence.upsert_car(first)
@@ -122,6 +131,87 @@ class TestPersistenceServiceIntegration(unittest.TestCase):
self.assertEqual(len(cars), 1)
self.assertEqual(cars[0].origin_id, "NEW-ID")
def test_upsert_preserves_first_seen_and_reactivates_seen_car(self) -> None:
first_seen = datetime(2026, 5, 1, tzinfo=timezone.utc)
sold_at = datetime(2026, 5, 2, tzinfo=timezone.utc)
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
first = self._record("mobile.de:777", price=1000)
first.first_seen_at = first_seen
first.last_seen_at = first_seen
self.persistence.upsert_car(first)
with self.persistence.session_scope() as session:
car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
car.is_sold = True
car.sold_at = sold_at
updated = self._record("mobile.de:777", price=1500)
updated.first_seen_at = seen_at
updated.last_seen_at = seen_at
updated.is_sold = False
updated.sold_at = None
self.persistence.upsert_car(updated)
with self.persistence.session_scope() as session:
car = session.execute(select(Car).where(Car.origin_id == "mobile.de:777")).scalar_one()
self.assertEqual(self._as_utc(car.first_seen_at), first_seen)
self.assertEqual(self._as_utc(car.last_seen_at), seen_at)
self.assertFalse(car.is_sold)
self.assertIsNone(car.sold_at)
self.assertEqual(car.price, 1500)
def test_mark_sold_not_seen_since_sets_sold_at_to_cycle_seen_at(self) -> None:
seen_at = datetime(2026, 5, 5, tzinfo=timezone.utc)
old = self._record("mobile.de:old")
old.first_seen_at = seen_at - timedelta(days=2)
old.last_seen_at = seen_at - timedelta(days=1)
current = self._record("mobile.de:current")
current.first_seen_at = seen_at
current.last_seen_at = seen_at
self.persistence.upsert_car(old)
self.persistence.upsert_car(current)
marked = self.persistence.mark_sold_not_seen_since(seen_at)
self.assertEqual(marked, 1)
with self.persistence.session_scope() as session:
cars = session.execute(select(Car).order_by(Car.origin_id.asc())).scalars().all()
sold_map = {car.origin_id: (car.is_sold, self._as_utc(car.sold_at)) for car in cars}
self.assertEqual(sold_map["mobile.de:old"], (True, seen_at))
self.assertEqual(sold_map["mobile.de:current"], (False, None))
def test_get_active_cars_batch_for_image_enrich_selects_low_image_active_cars(self) -> None:
low = self._record("mobile.de:low")
low.images = [
ImageRecord(
fullres_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-640.jpg",
preview_image="https://img.classistatic.de/api/v1/mo-prod/images/low?rule=mo-200.jpg",
order_index=0,
)
]
rich = self._record("mobile.de:rich")
rich.images = [
ImageRecord(
fullres_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-640.jpg",
preview_image=f"https://img.classistatic.de/api/v1/mo-prod/images/rich-{idx}?rule=mo-200.jpg",
order_index=idx,
)
for idx in range(3)
]
sold = self._record("mobile.de:sold")
sold.images = []
sold.is_sold = True
self.persistence.upsert_car(low)
self.persistence.upsert_car(rich)
self.persistence.upsert_car(sold)
candidates = self.persistence.get_active_cars_batch_for_image_enrich(limit=10, max_existing_images=1)
self.assertEqual([(origin_id, image_count) for _id, origin_id, _url, image_count in candidates], [("mobile.de:low", 1)])
if __name__ == "__main__":
unittest.main()

View File

@@ -27,8 +27,51 @@ class TestMobileDeMapper(unittest.TestCase):
)
self.assertEqual(len(record.images), 2)
self.assertEqual(record.images[0].fullres_image, urls[0])
self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2")
self.assertEqual(record.images[0].fullres_image, f"{urls[0]}?rule=mo-640.jpg")
self.assertEqual(record.images[1].fullres_image, "https://img.classistatic.de/api/v1/mo-prod/images/2?rule=mo-640.jpg")
def test_extracts_nested_gallery_images_without_detail_fetch(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="124",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=124",
title="Honda Civic",
raw={
"image": "https://img.classistatic.de/api/v1/mo-prod/images/main",
"images": [{"ref": "img.classistatic.de/api/v1/mo-prod/images/ref-1"}],
"mediaGallery": [
{"uri": "//img.classistatic.de/api/v1/mo-prod/images/gallery-1"},
{"picture": {"src": "https://img.classistatic.de/api/v1/mo-prod/images/gallery-2"}},
],
"trackingUrl": "https://example.test/not-an-image",
},
)
)
self.assertEqual(
[image.fullres_image for image in record.images],
[
"https://img.classistatic.de/api/v1/mo-prod/images/main?rule=mo-640.jpg",
"https://img.classistatic.de/api/v1/mo-prod/images/ref-1?rule=mo-640.jpg",
"https://img.classistatic.de/api/v1/mo-prod/images/gallery-1?rule=mo-640.jpg",
"https://img.classistatic.de/api/v1/mo-prod/images/gallery-2?rule=mo-640.jpg",
],
)
def test_keeps_existing_mobilede_image_rule(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="125",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=125",
title="BMW 320",
raw={"images": [{"uri": "img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"}]},
)
)
self.assertEqual(
[image.fullres_image for image in record.images],
["https://img.classistatic.de/api/v1/mo-prod/images/abc?rule=mo-1024.jpg"],
)
def test_origin_id_uses_canonical_prefix(self) -> None:
record = self.mapper.listing_to_car_record(
@@ -84,6 +127,255 @@ class TestMobileDeMapper(unittest.TestCase):
self.assertEqual(record.year, 2019)
self.assertEqual(record.mileage, 50100)
def test_listing_uses_mobilede_attr_fallbacks(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1001",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1001",
title="BMW X1 xDrive",
subtitle="Sport",
raw={
"attr": {
"yc": "2016",
"cn": "DE",
"ecol": "Серебряный",
"c": "OffRoad",
"cc": "1 998 ccm",
}
},
)
)
self.assertEqual(record.year, 2016)
self.assertEqual(record.country, "DE")
self.assertEqual(record.color, "silver")
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.engine_volume, 1998)
self.assertEqual(record.drive, "4WD")
def test_listing_body_and_color_normalization_is_broader(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1002",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1002",
title="BMW 320 Touring",
subtitle="EstateCar",
raw={"attr": {"ecol": "Коричневый"}},
)
)
self.assertEqual(record.body_type, "STATION_WAGON")
self.assertEqual(record.color, "brown")
def test_listing_maps_available_search_payload_fields(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1003",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1003",
title="BMW 120 Frontantrieb",
raw={
"price": {"grs": {"amount": "12250"}},
"contact": {"country": "IT"},
"priceRating": {"rating": "GOOD_PRICE", "ratingLabel": "Good price"},
"hasDamage": True,
"isConditionNew": True,
"cubicCapacity": "1 995 ccm",
"color": "Black Metallic",
"attr": {
"yc": "2020",
"ml": "55 000 km",
"c": "SmallCar",
"tr": "Manual",
"pvo": "1",
},
},
)
)
self.assertEqual(record.year, 2020)
self.assertEqual(record.price, 12250)
self.assertEqual(record.mileage, 55000)
self.assertEqual(record.country, "IT")
self.assertEqual(record.color, "black")
self.assertEqual(record.drive, "FWD")
self.assertEqual(record.gearbox, "MT")
self.assertEqual(record.body_type, "HATCHBACK")
self.assertEqual(record.engine_volume, 1995)
self.assertTrue(record.one_owner)
self.assertTrue(record.new_car)
self.assertTrue(record.is_damaged)
self.assertTrue(record.repair_history)
self.assertEqual(record.evaluation, "GOOD_PRICE")
def test_drive_mapping_uses_specific_markers_only(self) -> None:
allrad = self.mapper.listing_to_car_record(
MobileDeListing(
id="1004",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1004",
title="BMW X3 Allrad",
)
)
unrelated = self.mapper.listing_to_car_record(
MobileDeListing(
id="1005",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1005",
title="BMW 320 All inclusive",
)
)
self.assertEqual(allrad.drive, "4WD")
self.assertIsNone(unrelated.drive)
def test_search_mapping_uses_country_codes_and_body_fallbacks(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1006",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1006",
title="BMW X3",
subtitle="M40 d Facelift 1 Hand Scheckheft BMW",
raw={
"category": "Иное",
"attr": {
"cn": "NL",
"c": "OtherCar",
"cc": "2 993 ccm",
},
},
)
)
self.assertEqual(record.country, "NL")
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.engine_volume, 2993)
def test_search_mapping_infers_explicit_decimal_engine_volume(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1007",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1007",
title="Honda Civic",
subtitle="1.6 Automatik",
raw={"attr": {"cn": "AT", "c": "SmallCar"}},
)
)
self.assertEqual(record.country, "AT")
self.assertEqual(record.body_type, "HATCHBACK")
self.assertEqual(record.engine_volume, 1600)
def test_search_mapping_does_not_treat_plain_year_text_as_engine_volume(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1009",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1009",
title="Honda Civic",
subtitle="1983",
raw={"attr": {"cn": "SK", "c": "SmallCar"}},
)
)
self.assertIsNone(record.engine_volume)
def test_search_mapping_extracts_ccm_without_concatenating_all_title_digits(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1010",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1010",
title="BMW 320",
subtitle="E 46 320i 2.200 ccm 170 PS 2005",
raw={"attr": {"cn": "DE", "c": "Cabrio"}},
)
)
self.assertEqual(record.engine_volume, 2200)
def test_search_mapping_reads_nested_safe_field_aliases(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1008",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1008",
title="BMW 218",
raw={
"vehicle": {
"specs": {
"exteriorColor": "Silver",
"bodyType": "Cabrio",
"cubicCapacity": "1998",
"wheelDrive": "Allrad",
"transmission": "Automatic",
}
}
},
)
)
self.assertEqual(record.color, "silver")
self.assertEqual(record.body_type, "OPEN")
self.assertEqual(record.engine_volume, 1998)
self.assertEqual(record.drive, "4WD")
self.assertEqual(record.gearbox, "AT")
def test_listing_mapping_supports_attr_aliases_and_liter_engine_text(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1011",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1011",
title="BMW 320 Touring",
subtitle="2.0 l xDrive",
raw={
"attr": {
"exteriorColor": "Skyscraper Grey Metallic",
"wheelDrive": "Antrieb vorne",
"engineDisplacement": "2.0 l",
},
"driveType": "xDrive",
},
)
)
self.assertEqual(record.color, "gray")
self.assertEqual(record.drive, "FWD")
self.assertEqual(record.engine_volume, 2000)
def test_detail_mapping_uses_aliases_for_color_drive_and_engine(self) -> None:
record = self.mapper.detail_to_car_record(
"1012",
{
"shortTitle": "BMW X3",
"subTitle": "xDrive30d",
"make": {"localized": "BMW"},
"model": {"localized": "X3"},
"price": {"grs": {"amount": "31990", "currency": "EUR"}},
"contact": {"countryCode": "DE"},
"attributes": [
{"tag": "exteriorColor", "value": "Brooklyn Grey"},
{"tag": "driveType", "value": "All wheel drive"},
{"tag": "engineDisplacement", "value": "2993 cc"},
{"tag": "mileage", "value": "145 000 km"},
{"tag": "year", "value": "2019"},
{"tag": "bodyType", "value": "Geländewagen"},
],
},
)
self.assertEqual(record.color, "gray")
self.assertEqual(record.drive, "4WD")
self.assertEqual(record.engine_volume, 2993)
self.assertEqual(record.body_type, "SUV")
self.assertEqual(record.year, 2019)
def test_drive_prefers_explicit_attr_marker_over_title_noise(self) -> None:
record = self.mapper.listing_to_car_record(
MobileDeListing(
id="1013",
url="https://suchen.mobile.de/fahrzeuge/details.html?id=1013",
title="BMW 320 xDrive",
raw={"attr": {"wheelDrive": "Front-wheel drive"}},
)
)
self.assertEqual(record.drive, "FWD")
if __name__ == "__main__":
unittest.main()

View File

@@ -1,15 +1,19 @@
from __future__ import annotations
import unittest
from datetime import datetime, timezone
from unittest.mock import patch
from mobilede_scraper.mobile_de.models import MobileDeListing, MobileDeSearchPage
from mobilede_scraper.mobile_de.scraper import MobileDeScraper
class _FakePersistence:
def __init__(self) -> None:
def __init__(self, existing_ids: set[str] | None = None) -> None:
self.upsert_calls: list[list[str]] = []
self.upsert_records: list[object] = []
self.finish_payload: dict[str, object] | None = None
self.existing_ids = existing_ids or set()
def create_tables(self) -> None:
return None
@@ -18,10 +22,11 @@ class _FakePersistence:
return 1
def get_existing_origin_ids(self, origin_ids: list[str]) -> set[str]:
return set()
return {origin_id for origin_id in origin_ids if origin_id in self.existing_ids}
def upsert_cars_batch(self, records):
self.upsert_calls.append([record.origin_id for record in records])
self.upsert_records.extend(records)
return {
"inserted": len(records),
"updated": 0,
@@ -35,6 +40,7 @@ class _FakePersistence:
class _FakeClient:
def __init__(self, pages: list[MobileDeSearchPage]) -> None:
self.pages = pages
self.detail_calls: list[str] = []
def build_make_model_param(self, make_id: str, model_id: str | None) -> str:
return make_id if not model_id else f"{make_id};{model_id}"
@@ -43,6 +49,26 @@ class _FakeClient:
del kwargs
yield from self.pages
def fetch_detail(self, listing_id: str) -> dict[str, object]:
self.detail_calls.append(str(listing_id))
return {
"shortTitle": "BMW X1",
"subTitle": "xDrive20i",
"make": {"localized": "BMW"},
"model": {"localized": "X1"},
"price": {"grs": {"amount": "19999", "currency": "EUR"}},
"contact": {"countryCode": "DE"},
"attributes": [
{"tag": "firstRegistration", "value": "05/2016"},
{"tag": "mileage", "value": "71 500 km"},
{"tag": "category", "value": "OffRoad"},
{"tag": "color", "value": "Серый"},
{"tag": "wheelDrive", "value": "xDrive"},
{"tag": "cubicCapacity", "value": "1 998 ccm"},
],
"images": ["https://img.example.test/1.jpg"],
}
class TestMobileDeScraperStreamingSync(unittest.TestCase):
def test_sync_search_upserts_each_page_during_run(self) -> None:
@@ -82,6 +108,75 @@ class TestMobileDeScraperStreamingSync(unittest.TestCase):
self.assertIsNotNone(persistence.finish_payload)
self.assertEqual(int(persistence.finish_payload["cars_upserted"]), 3)
def test_sync_search_applies_one_seen_at_to_all_records_in_run(self) -> None:
seen_at = datetime(2026, 5, 5, 12, 30, tzinfo=timezone.utc)
pages = [
MobileDeSearchPage(
url="https://example.test/page-1",
page_number=1,
total_results=2,
listings=[
MobileDeListing(id="1", url="https://example.test/1", title="Car 1"),
MobileDeListing(id="2", url="https://example.test/2", title="Car 2"),
],
),
]
persistence = _FakePersistence()
scraper = MobileDeScraper(client=_FakeClient(pages), persistence=persistence)
scraper.sync_search(max_pages=1, seen_at=seen_at)
records = persistence.upsert_calls
self.assertEqual(records, [["mobile.de:1", "mobile.de:2"]])
self.assertTrue(all(record.first_seen_at == seen_at for record in persistence.upsert_records))
self.assertTrue(all(record.last_seen_at == seen_at for record in persistence.upsert_records))
self.assertTrue(all(record.is_sold is False for record in persistence.upsert_records))
self.assertTrue(all(record.sold_at is None for record in persistence.upsert_records))
def test_sync_search_selectively_enriches_new_records_with_missing_fields(self) -> None:
pages = [
MobileDeSearchPage(
url="https://example.test/page-1",
page_number=1,
total_results=2,
listings=[
MobileDeListing(
id="1",
url="https://example.test/1",
title="BMW X1",
subtitle="xDrive20i",
raw={"attr": {"fr": "", "ml": "", "c": "", "ecol": "", "cc": ""}},
),
MobileDeListing(
id="2",
url="https://example.test/2",
title="BMW 320",
raw={"attr": {"fr": "09/2016", "ml": "115 000 km", "c": "EstateCar", "ecol": "Серый", "cc": "1 998 ccm"}},
),
],
),
]
client = _FakeClient(pages)
persistence = _FakePersistence(existing_ids={"mobile.de:2"})
scraper = MobileDeScraper(client=client, persistence=persistence)
with (
patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED", True),
patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_RUN", 1),
patch("mobilede_scraper.mobile_de.scraper.MOBILEDE_SELECTIVE_DETAIL_ENRICH_MAX_PER_PAGE", 1),
):
result = scraper.sync_search(max_pages=1)
self.assertEqual(client.detail_calls, ["1"])
self.assertEqual(result["detail_enriched"], 1)
self.assertEqual(result["detail_enrich_failed"], 0)
record_by_id = {record.origin_id: record for record in persistence.upsert_records}
self.assertEqual(record_by_id["mobile.de:1"].drive, "4WD")
self.assertEqual(record_by_id["mobile.de:1"].engine_volume, 1998)
self.assertEqual(record_by_id["mobile.de:1"].body_type, "SUV")
self.assertEqual(record_by_id["mobile.de:1"].year, 2016)
self.assertEqual(record_by_id["mobile.de:2"].engine_volume, 1998)
if __name__ == "__main__":
unittest.main()

View File

@@ -65,7 +65,7 @@ class TestSelfHeal(unittest.TestCase):
@patch("builtins.open")
def test_kill_worker_process_sends_term_and_kill(self, open_mock, kill_mock, _sleep_mock) -> None:
open_mock.return_value.__enter__.return_value.read.return_value = "123"
# SIGTERM -> process alive check (pid,0) -> SIGKILL
# SIGTERM -> проверка, что процесс жив (pid,0) -> SIGKILL
kill_mock.side_effect = [None, None, None]
self_heal._kill_worker_process()

View File

@@ -1,7 +1,9 @@
from __future__ import annotations
from datetime import datetime, timezone
from types import SimpleNamespace
import unittest
from unittest.mock import MagicMock
from unittest.mock import MagicMock, patch
from mobilede_scraper.worker import tasks
@@ -11,6 +13,7 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
def __init__(self) -> None:
self.store: dict[str, str] = {}
self.sets: dict[str, set[str]] = {}
self.lists: dict[str, list[str]] = {}
def get(self, key: str):
return self.store.get(key)
@@ -33,6 +36,26 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
bucket.add(member)
return 1 if len(bucket) > before else 0
def scard(self, key: str):
return len(self.sets.get(key, set()))
def llen(self, key: str):
return len(self.lists.get(key, []))
def delete(self, *keys: str):
removed = 0
for key in keys:
if key in self.store:
del self.store[key]
removed += 1
if key in self.sets:
del self.sets[key]
removed += 1
if key in self.lists:
del self.lists[key]
removed += 1
return removed
def expire(self, key: str, ttl: int): # noqa: ARG002
return True
@@ -83,6 +106,34 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(key1, key2)
self.assertEqual(len(key1), 16)
def test_prune_overflow_parent_segments_keeps_only_leaf_segments(self) -> None:
parent = {
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:100000",
"make_id": "3500",
"price_min": "1",
"price_max": "5000",
"year_max": "2004",
"mileage_max": "100000",
}
parent_key = tasks._mobilede_segment_fingerprint(parent)
child_a = {
**parent,
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=:50000",
"mileage_max": "50000",
"overflow_parent": parent_key,
}
child_b = {
**parent,
"search_url": "https://www.mobile.de/ru/search.html?ms=3500&p=1:5000&fr=:2004&ml=50001:100000",
"mileage_min": "50001",
"mileage_max": "100000",
"overflow_parent": parent_key,
}
pruned = tasks._mobilede_prune_overflow_parent_segments([parent, child_a, child_b])
self.assertEqual(pruned, [child_a, child_b])
def test_runtime_segment_reservation_uses_pending_cache(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
@@ -102,86 +153,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(segment, (1, {"make_id": "AUDI"}))
def test_preplan_splits_dense_segment_before_queueing(self) -> None:
original_probe = tasks._mobilede_probe_segment_total
original_children = tasks._mobilede_build_overflow_child_segments
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
parent = {
"label": "Cars",
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
child_a = {**parent, "search_url": parent["search_url"] + "&ml=:100000", "mileage_max": "100000", "overflow_depth": 1}
child_b = {**parent, "search_url": parent["search_url"] + "&ml=100001:", "mileage_min": "100001", "overflow_depth": 1}
try:
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 450
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 700
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
tasks._mobilede_probe_segment_total = lambda segment: 2000 if segment is parent or segment.get("overflow_depth") is None else 100
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [child_a, child_b]
planned = tasks._mobilede_preplan_runtime_segments([parent])
finally:
tasks._mobilede_probe_segment_total = original_probe
tasks._mobilede_build_overflow_child_segments = original_children
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
self.assertEqual(len(planned), 2)
self.assertTrue(all(item.get("total_results") == 100 for item in planned))
self.assertTrue(all(int(item.get("max_pages") or 0) <= tasks.MOBILEDE_MAX_PAGE_NUMBER for item in planned))
def test_preplan_has_hard_segment_limit(self) -> None:
original_probe = tasks._mobilede_probe_segment_total
original_children = tasks._mobilede_build_overflow_child_segments
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
original_max_segments = tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS
original_max_probes = tasks.MOBILEDE_PREPLAN_MAX_PROBES
original_threshold_ratio = tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO
original_max_preplan_depth = tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH
parent = {
"label": "Cars",
"search_url": "https://suchen.mobile.de/fahrzeuge/search.html?isSearchRequest=true",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
try:
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = True
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = 3
tasks.MOBILEDE_PREPLAN_MAX_PROBES = 10
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = 1.0
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = 2
tasks._mobilede_probe_segment_total = lambda segment: 5000
tasks._mobilede_build_overflow_child_segments = lambda *, segment, max_pages: [
{**segment, "search_url": str(segment["search_url"]) + "&a=1", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
{**segment, "search_url": str(segment["search_url"]) + "&a=2", "overflow_depth": int(segment.get("overflow_depth") or 0) + 1},
]
planned = tasks._mobilede_preplan_runtime_segments([parent])
finally:
tasks._mobilede_probe_segment_total = original_probe
tasks._mobilede_build_overflow_child_segments = original_children
tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES = original_preplan_enabled
tasks.MOBILEDE_PREPLAN_MAX_SEGMENTS = original_max_segments
tasks.MOBILEDE_PREPLAN_MAX_PROBES = original_max_probes
tasks.MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO = original_threshold_ratio
tasks.MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH = original_max_preplan_depth
self.assertGreaterEqual(len(planned), 1)
self.assertLessEqual(len(planned), 3)
def test_preplan_probe_budget_is_global(self) -> None:
original_probe = tasks._mobilede_probe_segment_total
original_preplan_enabled = tasks.MOBILEDE_PREPLAN_SEGMENT_PROBES
@@ -215,30 +186,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertEqual(calls["count"], 2)
self.assertEqual(len(planned), 5)
def test_overflow_root_mileage_split_keeps_high_mileage_tail_with_two_children(self) -> None:
original_max_children = tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS
try:
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = 2
children = tasks._mobilede_build_overflow_child_segments(
segment={
"label": "Cars",
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=5001:10000&fr=2005:2009",
"price_min": "5001",
"price_max": "10000",
"year_min": "2005",
"year_max": "2009",
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
},
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
)
finally:
tasks.MOBILEDE_OVERFLOW_MAX_CHILD_SEGMENTS = original_max_children
self.assertEqual(len(children), 2)
self.assertEqual(children[0].get("mileage_max"), "150000")
self.assertEqual(children[1].get("mileage_min"), "150001")
self.assertIsNone(children[1].get("mileage_max"))
def test_bootstrap_finalization_waits_for_dispatched_segments(self) -> None:
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: {
@@ -278,6 +225,86 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
redis_client.set.assert_any_call(tasks.MOBILEDE_BOOTSTRAP_DONE_KEY, "1")
redis_client.set.assert_any_call(tasks.MOBILEDE_RUNTIME_SEGMENTS_PLAN_FINALIZED_KEY, "1", ex=30 * 24 * 60 * 60)
def test_force_full_scan_only_new_stays_full_pass_for_continuous_hourly_cycle(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10"
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10"
effective_only_new = tasks._mobilede_force_full_scan_only_new(
True,
redis_client=redis_client,
continuous=True,
)
self.assertFalse(effective_only_new)
def test_post_bootstrap_refresh_disabled_for_continuous_hourly_cycle(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "10"
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "10"
post_bootstrap_refresh = tasks._mobilede_post_bootstrap_full_refresh(
redis_client,
False,
continuous=True,
)
self.assertFalse(post_bootstrap_refresh)
def test_stalled_bootstrap_recovery_ignores_stale_global_progress_key(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13"
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261"
redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "1"
redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"}
with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000):
recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client)
self.assertTrue(recovered)
self.assertNotIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets)
def test_stalled_bootstrap_recovery_keeps_recent_progress(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_DONE_KEY] = "13"
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_SEGMENTS_TOTAL_KEY] = "261"
redis_client.store[tasks.GLOBAL_PROGRESS_TS_KEY] = "9950"
redis_client.sets[tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY] = {"seg-a", "seg-b"}
with patch("mobilede_scraper.worker.tasks.time.time", return_value=10_000):
recovered = tasks._mobilede_try_recover_stalled_bootstrap_queue(redis_client)
self.assertFalse(recovered)
self.assertIn(tasks.MOBILEDE_BOOTSTRAP_DISPATCHED_SEGMENTS_KEY, redis_client.sets)
def test_bootstrap_completion_forces_refresh_sold_finalize_when_progress_lags(self) -> None:
redis_client = self._FakeRedis()
started_at = datetime(2026, 5, 6, 10, 0, tzinfo=timezone.utc)
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_ID_KEY] = "cycle-lag"
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_STARTED_AT_KEY] = started_at.isoformat()
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_TOTAL_KEY] = "377"
redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY] = "141"
persistence = MagicMock()
persistence.mark_sold_not_seen_since.return_value = 42
with (
patch.object(tasks, "_get_persistence", return_value=persistence),
patch.object(tasks, "MOBILEDE_POST_REFRESH_SOLD_PROBE_ENABLED", False),
):
finalized = tasks._mobilede_try_finalize_refresh_cycle_after_bootstrap_completion(
redis_client,
cycle_id="cycle-lag",
total_segments_hint=377,
)
self.assertTrue(finalized)
self.assertEqual(redis_client.store[tasks.MOBILEDE_REFRESH_CYCLE_DONE_KEY], "377")
persistence.mark_sold_not_seen_since.assert_called_once_with(
started_at,
prefix=("mobile.de:", "mobilede:"),
safety_ratio=0.8,
)
def test_completed_bootstrap_segment_is_skipped_during_active_full_pass(self) -> None:
redis_client = MagicMock()
redis_client.get.side_effect = lambda key: "1" if str(key).startswith("mobilede:state:bootstrap_segment_done:") else None
@@ -291,6 +318,58 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(should_skip)
def test_queue_mobilede_bootstrap_recovery_can_force_requeue_existing_pending(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1"
with patch.object(tasks.mobilede_sync_runtime_segments_task, "apply_async") as apply_async:
queued = tasks._queue_mobilede_bootstrap_recovery(
redis_client,
lane="mobile_de_cars",
delay_seconds=0.7,
use_cursor=True,
continuous=True,
segment_label="runtime_segments",
reason="waiting_for_active_bootstrap_tasks",
countdown=5,
force=True,
)
self.assertTrue(queued)
apply_async.assert_called_once()
self.assertEqual(redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY], "1")
def test_runtime_segments_task_requeues_pending_bootstrap_recovery_while_tasks_are_still_active(self) -> None:
redis_client = self._FakeRedis()
redis_client.store[tasks.MOBILEDE_BOOTSTRAP_RECOVERY_PENDING_KEY] = "1"
runtime_config = SimpleNamespace(sync=SimpleNamespace(only_new=False))
with (
patch.object(tasks, "_get_redis", return_value=redis_client),
patch.object(tasks, "_mobilede_try_recover_stalled_bootstrap_queue", return_value=False),
patch.object(tasks, "_get_cached_mobilede_runtime_segments", return_value=[{"make_id": "3500"}]),
patch.object(tasks, "Settings", return_value=SimpleNamespace(runtime_config_file="runtime_config.json")),
patch.object(tasks.RuntimeConfig, "from_file", return_value=runtime_config),
patch.object(tasks, "_mobilede_force_full_scan_only_new", return_value=False),
patch.object(tasks, "_mobilede_bootstrap_done", return_value=False),
patch.object(tasks, "_mobilede_post_bootstrap_full_refresh", return_value=False),
patch.object(tasks, "_mobilede_bootstrap_progress", return_value=(13, 261, 248)),
patch.object(tasks, "_mobilede_bootstrap_progress_snapshot", return_value=(13, 261, 248, 2)),
patch.object(tasks, "_has_recent_global_progress", return_value=False),
patch.object(tasks, "_queue_mobilede_bootstrap_recovery", return_value=True) as queue_recovery,
):
result = tasks.mobilede_sync_runtime_segments_task.run(
lane="mobile_de_cars",
delay_seconds=0.7,
use_cursor=True,
continuous=True,
full_pass_repeat=False,
)
self.assertEqual(result["status"], "bootstrap_active")
queue_recovery.assert_called_once()
def test_exhausted_non_cursor_segment_window_is_skipped(self) -> None:
exhausted = tasks._mobilede_segment_window_is_exhausted(
{"make_id": "3500", "max_pages": 50},
@@ -357,9 +436,58 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
)
self.assertEqual(done, 0)
self.assertEqual(total, 2)
self.assertEqual(total, 0)
self.assertFalse(should_finalize)
def test_overflow_skip_when_known_total_fits_page_cap(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
segment = {
"label": "Cars",
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=1:5000",
"total_results": 995,
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
added = tasks._mobilede_try_expand_overflow_segment(
redis_client,
settings,
segment=segment,
listing_count=1000,
unique_count=995,
max_pages=tasks.MOBILEDE_MAX_PAGE_NUMBER,
segment_end_page=tasks.MOBILEDE_MAX_PAGE_NUMBER,
)
self.assertEqual(added, 0)
redis_client.sadd.assert_not_called()
def test_late_overflow_children_are_not_queued_during_preplanned_bootstrap(self) -> None:
redis_client = MagicMock()
settings = MagicMock()
segment = {
"label": "Cars | ms=3500 | price=15001-20000",
"search_url": "https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&p=15001:20000",
"make_id": "3500",
}
with patch.object(tasks, "os") as os_mock:
os_mock.getenv.return_value = "true"
queued = tasks._queue_mobilede_overflow_child_segments(
redis_client,
settings,
parent_segment=segment,
lane="mobile_de_cars",
delay_seconds=0,
use_cursor=True,
only_new=False,
bootstrap_run=True,
refresh_cycle_id="cycle-a",
)
self.assertEqual(queued, 0)
redis_client.get.assert_not_called()
def test_pre_split_mileage_requires_explicit_flag(self) -> None:
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
try:
@@ -390,58 +518,6 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
def test_filtered_multi_make_url_expands_to_make_price_year_segments(self) -> None:
search_url = (
"https://www.mobile.de/ru/????????????????????????-????????????????/??????????.html"
"?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000&od=up&sb=rel&ref=dsp"
)
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
original_dynamic = tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES
try:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = False
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = False
segments = tasks._expand_mobilede_search_url_segment(
{
"label": "Cars",
"search_url": search_url,
"start_page": 1,
"max_pages": tasks.MOBILEDE_MAX_PAGE_NUMBER,
}
)
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
tasks.MOBILEDE_DYNAMIC_SEGMENT_PROBES = original_dynamic
make_counts: dict[str, int] = {}
for segment in segments:
make_id = str(segment.get("make_id") or "")
make_counts[make_id] = make_counts.get(make_id, 0) + 1
self.assertIsNone(segment.get("mileage_min"))
self.assertIsNone(segment.get("mileage_max"))
self.assertEqual(make_counts, {"3500": 59, "11000": 59})
self.assertEqual(len(segments), 118)
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all(str(segment.get("max_pages")) == str(tasks.MOBILEDE_MAX_PAGE_NUMBER) for segment in segments))
def test_build_runtime_segments_expands_filtered_search_urls_by_default(self) -> None:
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
]
segments = tasks._build_mobilede_runtime_segments(settings)
self.assertEqual(len(segments), 118)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
self.assertTrue(all("p=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("fr=" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
def test_build_runtime_segments_can_fast_start_filtered_search_urls_when_enabled(self) -> None:
import os
@@ -466,6 +542,37 @@ class TestWorkerRuntimeTaskHelpers(unittest.TestCase):
self.assertTrue(all("fr=" not in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ml=" not in str(segment.get("search_url")) for segment in segments))
def test_build_runtime_segments_splits_multi_make_filtered_search_url(self) -> None:
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=3500&ms=11000"
]
segments = tasks._build_mobilede_runtime_segments(settings)
self.assertGreater(len(segments), 2)
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"3500", "11000"})
self.assertTrue(all("pageNumber=1" in str(segment.get("search_url")) for segment in segments))
self.assertTrue(all("ms=3500&ms=11000" not in str(segment.get("search_url")) for segment in segments))
def test_full_link_coverage_can_pre_split_every_segment_by_mileage(self) -> None:
settings = MagicMock()
settings.listing.filtered_search_urls = [
"https://www.mobile.de/ru/search.html?isSearchRequest=true&s=Car&vc=Car&ms=111&ms=222"
]
original_split = tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE
try:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = True
segments = tasks._build_mobilede_runtime_segments(settings)
finally:
tasks.MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE = original_split
self.assertGreater(len(segments), 300)
self.assertTrue(all("ml=" in str(segment.get("search_url")) for segment in segments))
self.assertEqual({str(segment.get("make_id")) for segment in segments}, {"111", "222"})
self.assertTrue(all("ms=111&ms=222" not in str(segment.get("search_url")) for segment in segments))
if __name__ == "__main__":
unittest.main()

2
uv.lock generated
View File

@@ -342,7 +342,7 @@ wheels = [
]
[[package]]
name = "iaai-scraper"
name = "mobile-de-scraper"
version = "0.1.0"
source = { editable = "." }
dependencies = [