qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix docker scraping

Browse Source 
improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup
This commit is contained in:
qananasikq
2026-04-13 16:35:08 +03:00
parent b9557922ed
commit a4c93a1df1
30 changed files with 2239 additions and 624 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
docker-compose.yml
View File

@@ -1,20 +1,26 @@
x-app-env: &app-env
IAAI_DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper}
# NB: hardcoded to Postgres — .env may contain sqlite for local CLI, don't let it leak here
IAAI_DATABASE_URL: postgresql+psycopg2://iaai:iaai@postgres:5432/iaai_scraper
IAAI_REDIS_URL: ${IAAI_REDIS_URL:-redis://redis:6379/0}
CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
IAAI_DATABASE_POOL_RECYCLE_SECONDS: ${IAAI_DATABASE_POOL_RECYCLE_SECONDS:-1800}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300}
CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600}
CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-20}
# Без лимита beat может забирать слишком большой объём за один запуск.
# Консервативный дефолт для anti-fraud: 300 авто за запуск.
CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-300}
CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
IAAI_STORAGE_STATE_PATH: ${IAAI_STORAGE_STATE_PATH:-/data/storage_state.json}
IAAI_SESSION_MAX_AGE_DAYS: ${IAAI_SESSION_MAX_AGE_DAYS:-30}
IAAI_SESSION_SAVE_ON_EXIT: ${IAAI_SESSION_SAVE_ON_EXIT:-true}
IAAI_LOGIN_EMAIL: ${IAAI_LOGIN_EMAIL:-}
IAAI_LOGIN_PASSWORD: ${IAAI_LOGIN_PASSWORD:-}
IAAI_BROWSER_ENGINE: ${IAAI_BROWSER_ENGINE:-auto}
TZ: ${TZ:-UTC}
x-env-file: &env-file
@@ -135,9 +141,9 @@ services:
stop_grace_period: 60s
command: >
celery -A iaai_scraper.worker.celery_app worker
--loglevel=info --concurrency=1 --pool=prefork
--loglevel=info --concurrency=4 --pool=prefork
--pidfile=/tmp/celery-worker.pid
-Q scraping --max-tasks-per-child=20
-Q scraping --max-tasks-per-child=5
healthcheck:
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
interval: 60s