fix: eliminate greenlet crash + memory leak protection
- docker-compose: switch worker to --pool=solo --concurrency=1 --max-tasks-per-child=1 - tasks.py: remove ThreadPoolExecutor wrapper from _run_browser_job (greenlet crash) - scraper.py: browser fallback runs sequentially instead of ThreadPoolExecutor - scraper.py: add gc.collect() after scraper close to prevent memory leaks
This commit is contained in:
@@ -144,9 +144,9 @@ services:
|
||||
stop_grace_period: 60s
|
||||
command: >
|
||||
celery -A iaai_scraper.worker.celery_app worker
|
||||
--loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-2} --pool=prefork
|
||||
--loglevel=info --concurrency=1 --pool=solo
|
||||
--pidfile=/tmp/celery-worker.pid
|
||||
-Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-3}
|
||||
-Q scraping --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-1}
|
||||
healthcheck:
|
||||
test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid)"]
|
||||
interval: 60s
|
||||
|
||||
Reference in New Issue
Block a user