56 lines
1.6 KiB
Python
56 lines
1.6 KiB
Python
# Инициализация Celery-приложения и периодических задач.
|
|
|
|
from celery import Celery
|
|
|
|
from ..core.config import settings
|
|
|
|
|
|
def _broker_url() -> str:
|
|
return settings.celery.broker_url or settings.redis.url
|
|
|
|
|
|
def _result_backend() -> str:
|
|
return settings.celery.result_backend or settings.redis.url
|
|
|
|
|
|
celery_app = Celery(
|
|
"iaai_scraper",
|
|
broker=_broker_url(),
|
|
backend=_result_backend(),
|
|
)
|
|
|
|
celery_app.conf.update(
|
|
task_serializer="json",
|
|
accept_content=["json"],
|
|
result_serializer="json",
|
|
timezone="UTC",
|
|
enable_utc=True,
|
|
task_soft_time_limit=settings.celery.task_soft_time_limit,
|
|
task_time_limit=settings.celery.task_time_limit,
|
|
task_acks_late=True,
|
|
task_reject_on_worker_lost=True,
|
|
task_track_started=True,
|
|
worker_concurrency=settings.celery.worker_concurrency,
|
|
worker_max_tasks_per_child=settings.celery.worker_max_tasks_per_child,
|
|
worker_pool="prefork",
|
|
worker_prefetch_multiplier=1,
|
|
broker_connection_retry_on_startup=True,
|
|
broker_transport_options={
|
|
"visibility_timeout": settings.celery.broker_visibility_timeout,
|
|
},
|
|
result_expires=86400,
|
|
beat_schedule={
|
|
"periodic-sync-listing": {
|
|
"task": "iaai_scraper.worker.tasks.sync_listing_task",
|
|
"schedule": settings.celery.beat_sync_interval_minutes * 60.0,
|
|
"args": (),
|
|
"kwargs": {"limit": settings.celery.beat_sync_limit},
|
|
"options": {"queue": "scraping"},
|
|
}
|
|
},
|
|
task_routes={
|
|
"iaai_scraper.worker.tasks.*": {"queue": "scraping"}
|
|
},
|
|
)
|
|
|
|
celery_app.autodiscover_tasks(["iaai_scraper.worker"]) |