qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepared for production

Browse Source
This commit is contained in:
Andrey Shershnev
2026-04-23 13:54:53 +03:00
parent 09fecdae31
commit f221aebef0
8 changed files with 159 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
alembic/env.py
View File

@@ -14,6 +14,7 @@ from iaai_scraper.core.config import Settings
from iaai_scraper.storage.models import Base from iaai_scraper.storage.models import Base
config = context.config config = context.config
VERSION_TABLE = "iaai_parser_alembic_version"
# Logging # Logging
if config.config_file_name is not None: if config.config_file_name is not None:
@@ -32,6 +33,7 @@ def run_migrations_offline() -> None:
context.configure( context.configure(
url=url, url=url,
target_metadata=target_metadata, target_metadata=target_metadata,
version_table=VERSION_TABLE,
literal_binds=True, literal_binds=True,
dialect_opts={"paramstyle": "named"}, dialect_opts={"paramstyle": "named"},
) )
@@ -47,7 +49,11 @@ def run_migrations_online() -> None:
poolclass=pool.NullPool, poolclass=pool.NullPool,
) )
with connectable.connect() as connection: with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata) context.configure(
connection=connection,
target_metadata=target_metadata,
version_table=VERSION_TABLE,
)
with context.begin_transaction(): with context.begin_transaction():
context.run_migrations() context.run_migrations()

50
alembic/versions/001_initial.py
View File

@@ -1,4 +1,4 @@
# Начальная схема: cars, images, sync_runs # Начальная схема: iaai_cars, iaai_images, iaai_sync_runs
from typing import Sequence, Union from typing import Sequence, Union
from alembic import op from alembic import op
@@ -10,10 +10,29 @@ branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None depends_on: Union[str, Sequence[str], None] = None
def _schema_name() -> str | None:
bind = op.get_bind()
return "public" if bind.dialect.name == "postgresql" else None
def _table_exists(table_name: str) -> bool:
inspector = sa.inspect(op.get_bind())
return table_name in inspector.get_table_names(schema=_schema_name())
def _index_exists(table_name: str, index_name: str) -> bool:
if not _table_exists(table_name):
return False
inspector = sa.inspect(op.get_bind())
indexes = inspector.get_indexes(table_name, schema=_schema_name())
return any(index.get("name") == index_name for index in indexes)
def upgrade() -> None: def upgrade() -> None:
# Таблица cars — аукционные машины с IAAI # Таблица iaai_cars — аукционные машины с IAAI
if not _table_exists("iaai_cars"):
op.create_table( op.create_table(
"cars", "iaai_cars",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("parser_id", sa.String(50), nullable=False, unique=True), sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
sa.Column("brand", sa.String(50), nullable=False), sa.Column("brand", sa.String(50), nullable=False),
@@ -45,22 +64,27 @@ def upgrade() -> None:
sa.Column("slug", sa.String, nullable=False), sa.Column("slug", sa.String, nullable=False),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
) )
op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
# Таблица images — изображения машин if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"):
op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"])
if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"):
op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True)
# Таблица iaai_images — изображения машин
if not _table_exists("iaai_images"):
op.create_table( op.create_table(
"images", "iaai_images",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("fullres_image", sa.String, nullable=False), sa.Column("fullres_image", sa.String, nullable=False),
sa.Column("preview_image", sa.String, nullable=False), sa.Column("preview_image", sa.String, nullable=False),
sa.Column("order_index", sa.Integer, nullable=False), sa.Column("order_index", sa.Integer, nullable=False),
sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False), sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False),
) )
# Таблица sync_runs — логирование синхронизаций # Таблица iaai_sync_runs — логирование синхронизаций
if not _table_exists("iaai_sync_runs"):
op.create_table( op.create_table(
"sync_runs", "iaai_sync_runs",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
@@ -73,7 +97,7 @@ def upgrade() -> None:
sa.Column("error_summary", sa.Text, nullable=True), sa.Column("error_summary", sa.Text, nullable=True),
) )
def downgrade() -> None: def downgrade() -> None:
op.drop_table("sync_runs") # Compatibility-only migration for shared production databases.
op.drop_table("images") pass
op.drop_table("cars")

23
alembic/versions/002_add_indexes.py
View File

@@ -10,20 +10,15 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None: def upgrade() -> None:
op.create_index("ix_cars_brand", "cars", ["brand"]) op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)")
op.create_index("ix_cars_brand_model", "cars", ["brand", "model"]) op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)")
op.create_index("ix_cars_year", "cars", ["year"]) op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)")
op.create_index("ix_cars_is_sold", "cars", ["is_sold"]) op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)")
op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"]) op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)")
op.create_index("ix_images_car_id", "images", ["car_id"]) op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)")
op.create_index("ix_sync_runs_status", "sync_runs", ["status"]) op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)")
def downgrade() -> None: def downgrade() -> None:
op.drop_index("ix_sync_runs_status", table_name="sync_runs") # Compatibility-only migration for shared production databases.
op.drop_index("ix_images_car_id", table_name="images") pass
op.drop_index("ix_cars_last_seen_at", table_name="cars")
op.drop_index("ix_cars_is_sold", table_name="cars")
op.drop_index("ix_cars_year", table_name="cars")
op.drop_index("ix_cars_brand_model", table_name="cars")
op.drop_index("ix_cars_brand", table_name="cars")

20
alembic/versions/003_add_composite_indexes.py
View File

@@ -13,26 +13,24 @@ def upgrade() -> None:
# Partial index для активных машин IAAI (is_sold = FALSE) # Partial index для активных машин IAAI (is_sold = FALSE)
# Ускоряет поиск при mark_sold операциях # Ускоряет поиск при mark_sold операциях
op.execute( op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_active_iaai " "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai "
"ON cars (origin_url) " "ON iaai_cars (origin_url) "
"WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'" "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
) )
# Composite index для проверки дубликатов изображений # Composite index для проверки дубликатов изображений
op.create_index( op.execute(
"ix_images_car_id_fullres", "CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres "
"images", "ON iaai_images (car_id, fullres_image)"
["car_id", "fullres_image"],
) )
# Index для быстрого поиска existing машин по origin_url # Index для быстрого поиска existing машин по origin_url
op.execute( op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id " "CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id "
"ON cars (origin_url, origin_id)" "ON iaai_cars (origin_url, origin_id)"
) )
def downgrade() -> None: def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id") # Compatibility-only migration for shared production databases.
op.drop_index("ix_images_car_id_fullres", table_name="images") pass
op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")

15
iaai_scraper/storage/db.py
View File

@@ -20,6 +20,7 @@ CAR_DB_FIELDS = {
} }
_IN_CHUNK_SIZE = 5000 _IN_CHUNK_SIZE = 5000
CAR_TABLE_NAME = Car.__tablename__
class PersistenceService: class PersistenceService:
@@ -532,7 +533,7 @@ class PersistenceService:
if is_postgres: if is_postgres:
# Считаем кандидатов на sold. # Считаем кандидатов на sold.
total_active = session.execute( total_active = session.execute(
text("SELECT count(*) FROM cars WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'") text(f"SELECT count(*) FROM {CAR_TABLE_NAME} WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%%'")
).scalar() or 0 ).scalar() or 0
if total_active == 0: if total_active == 0:
@@ -556,12 +557,12 @@ class PersistenceService:
# Считаем будущие sold. # Считаем будущие sold.
would_mark = session.execute(text(""" would_mark = session.execute(text("""
SELECT count(*) SELECT count(*)
FROM cars c FROM {car_table} c
LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
WHERE a.url IS NULL WHERE a.url IS NULL
AND c.is_sold = FALSE AND c.is_sold = FALSE
AND c.origin_id LIKE 'iaai:%%' AND c.origin_id LIKE 'iaai:%%'
""")).scalar() or 0 """.format(car_table=CAR_TABLE_NAME))).scalar() or 0
# Защита от аномалии. # Защита от аномалии.
if total_active > 100 and would_mark > total_active * 0.8: if total_active > 100 and would_mark > total_active * 0.8:
@@ -573,18 +574,18 @@ class PersistenceService:
# Массовая пометка sold. # Массовая пометка sold.
result = session.execute(text(""" result = session.execute(text("""
UPDATE cars UPDATE {car_table}
SET is_sold = TRUE SET is_sold = TRUE
FROM ( FROM (
SELECT c.id SELECT c.id
FROM cars c FROM {car_table} c
LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url LEFT JOIN _active_urls a ON replace(c.origin_url, '~', '-') = a.url
WHERE a.url IS NULL WHERE a.url IS NULL
AND c.is_sold = FALSE AND c.is_sold = FALSE
AND c.origin_id LIKE 'iaai:%%' AND c.origin_id LIKE 'iaai:%%'
) sub ) sub
WHERE cars.id = sub.id WHERE {car_table}.id = sub.id
""")) """.format(car_table=CAR_TABLE_NAME)))
count = result.rowcount or 0 count = result.rowcount or 0
else: else:
# Упрощённый путь для SQLite. # Упрощённый путь для SQLite.

12
iaai_scraper/storage/models.py
View File

@@ -20,10 +20,10 @@ class Base(DeclarativeBase):
class Car(Base): class Car(Base):
__tablename__ = "cars" __tablename__ = "iaai_cars"
__table_args__ = ( __table_args__ = (
Index("ix_cars_brand_model", "brand", "model"), Index("ix_iaai_cars_brand_model", "brand", "model"),
Index("ix_cars_origin_id_not_sold", "origin_id", "is_sold"), Index("ix_iaai_cars_origin_id_not_sold", "origin_id", "is_sold"),
) )
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True) parser_id: Mapped[str] = mapped_column(String(50), nullable=False, unique=True)
@@ -59,17 +59,17 @@ class Car(Base):
class Image(Base): class Image(Base):
__tablename__ = "images" __tablename__ = "iaai_images"
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
fullres_image: Mapped[str] = mapped_column(String(), nullable=False) fullres_image: Mapped[str] = mapped_column(String(), nullable=False)
preview_image: Mapped[str] = mapped_column(String(), nullable=False) preview_image: Mapped[str] = mapped_column(String(), nullable=False)
order_index: Mapped[int] = mapped_column(Integer, nullable=False) order_index: Mapped[int] = mapped_column(Integer, nullable=False)
car_id: Mapped[int] = mapped_column(Integer, ForeignKey("cars.id", ondelete="CASCADE"), nullable=False, index=True) car_id: Mapped[int] = mapped_column(Integer, ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False, index=True)
car: Mapped[Car] = relationship("Car", back_populates="images") car: Mapped[Car] = relationship("Car", back_populates="images")
class SyncRun(Base): class SyncRun(Base):
__tablename__ = "sync_runs" __tablename__ = "iaai_sync_runs"
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True) id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now()) started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now())
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)

26
iaai_scraper/worker/celery_app.py
View File

@@ -1,6 +1,7 @@
# Инициализация Celery-приложения и периодических задач. # Инициализация Celery-приложения и периодических задач.
import logging import logging
import os
from celery import Celery from celery import Celery
from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging from celery.signals import worker_process_init, worker_ready, setup_logging as celery_setup_logging
@@ -11,6 +12,12 @@ from ..core.logs import setup_logging
logger = logging.getLogger("iaai_scraper.worker.celery_app") logger = logging.getLogger("iaai_scraper.worker.celery_app")
STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched" STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
IAAI_SYNC_QUEUE = "iaai_sync"
def _env_bool(name: str, default: bool) -> bool:
raw = os.getenv(name, "true" if default else "false").strip().lower()
return raw in {"1", "true", "yes", "on"}
@celery_setup_logging.connect @celery_setup_logging.connect
@@ -79,18 +86,19 @@ celery_app.conf.update(
worker_hijack_root_logger=False, worker_hijack_root_logger=False,
beat_schedule={ beat_schedule={
"periodic-sync-listing": { "periodic-sync-listing": {
"task": "iaai_scraper.worker.tasks.sync_listing_task", "task": "iaai.sync_cars_feed",
"schedule": settings.celery.beat_sync_interval_minutes * 60.0, "schedule": settings.celery.beat_sync_interval_minutes * 60.0,
"args": (), "args": (),
"kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True}, "kwargs": {"limit": settings.celery.beat_sync_limit, "only_new": True},
"options": { "options": {
"queue": "scraping", "queue": IAAI_SYNC_QUEUE,
"expires": settings.celery.beat_sync_interval_minutes * 60.0, "expires": settings.celery.beat_sync_interval_minutes * 60.0,
}, },
} }
}, },
task_routes={ task_routes={
"iaai_scraper.worker.tasks.*": {"queue": "scraping"} "iaai.sync_cars_feed": {"queue": IAAI_SYNC_QUEUE},
"iaai_scraper.worker.tasks.*": {"queue": IAAI_SYNC_QUEUE},
}, },
) )
@@ -100,6 +108,10 @@ celery_app.autodiscover_tasks(["iaai_scraper.worker"])
@worker_ready.connect @worker_ready.connect
def _on_worker_ready(**kwargs): def _on_worker_ready(**kwargs):
"""При старте worker отправляем первый sync_listing, если очередь пуста.""" """При старте worker отправляем первый sync_listing, если очередь пуста."""
if not _env_bool("IAAI_STARTUP_SYNC_ENABLED", True):
logger.info("Worker ready: startup sync dispatch disabled by IAAI_STARTUP_SYNC_ENABLED")
return
redis_client = None redis_client = None
try: try:
redis_client = Redis.from_url( redis_client = Redis.from_url(
@@ -121,11 +133,11 @@ def _on_worker_ready(**kwargs):
logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True) logger.warning("Failed to clear stale lock %s on startup", stale_key, exc_info=True)
try: try:
queue_len = int(redis_client.llen("scraping") or 0) queue_len = int(redis_client.llen(IAAI_SYNC_QUEUE) or 0)
except Exception: except Exception:
queue_len = 0 queue_len = 0
if queue_len > 0: if queue_len > 0:
logger.info("Worker ready: scraping queue already has %d task(s); skip startup dispatch", queue_len) logger.info("Worker ready: iaai_sync queue already has %d task(s); skip startup dispatch", queue_len)
return return
should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600)) should_dispatch = bool(redis_client.set(STARTUP_SYNC_DISPATCH_KEY, "1", nx=True, ex=600))
@@ -145,8 +157,8 @@ def _on_worker_ready(**kwargs):
logger.info("Worker ready — dispatching initial sync_listing task") logger.info("Worker ready — dispatching initial sync_listing task")
celery_app.send_task( celery_app.send_task(
"iaai_scraper.worker.tasks.sync_listing_task", "iaai.sync_cars_feed",
kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False}, kwargs={"limit": settings.celery.beat_sync_limit, "only_new": False},
queue="scraping", queue=IAAI_SYNC_QUEUE,
expires=settings.celery.beat_sync_interval_minutes * 60.0, expires=settings.celery.beat_sync_interval_minutes * 60.0,
) )

21
iaai_scraper/worker/tasks.py
View File

@@ -19,6 +19,8 @@ from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitema
logger = logging.getLogger("iaai_scraper.worker.tasks") logger = logging.getLogger("iaai_scraper.worker.tasks")
IAAI_SYNC_QUEUE = "iaai_sync"
# Минимальная пауза между батчами (секунды) — не давит IAAI. # Минимальная пауза между батчами (секунды) — не давит IAAI.
_INTER_BATCH_DELAY = max(float(os.getenv("IAAI_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0) _INTER_BATCH_DELAY = max(float(os.getenv("IAAI_INTER_BATCH_DELAY_SECONDS", "0.3")), 0.0)
# Если доля failed в батче превышает порог — прерываем (IAAI блокирует). # Если доля failed в батче превышает порог — прерываем (IAAI блокирует).
@@ -44,7 +46,7 @@ HOURLY_FAILURE_STREAK_KEY = "iaai:state:hourly_failure_streak"
HOURLY_FAILURE_STREAK_LIMIT = 3 HOURLY_FAILURE_STREAK_LIMIT = 3
HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # сброс через 6 часов HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60 # сброс через 6 часов
SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending" SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task" SYNC_LISTING_TASK_NAME = "iaai.sync_cars_feed"
SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}" SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}"
SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress" SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress"
SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total" SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total"
@@ -482,7 +484,7 @@ def _start_stall_watchdog(
celery_app.send_task( celery_app.send_task(
SYNC_LISTING_TASK_NAME, SYNC_LISTING_TASK_NAME,
kwargs={}, kwargs={},
queue="scraping", queue=IAAI_SYNC_QUEUE,
countdown=15, countdown=15,
expires=followup_ttl, expires=followup_ttl,
) )
@@ -903,6 +905,7 @@ def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[in
@shared_task( @shared_task(
name="iaai_scraper.worker.tasks.sync_segment_task", name="iaai_scraper.worker.tasks.sync_segment_task",
queue=IAAI_SYNC_QUEUE,
bind=True, bind=True,
max_retries=2, max_retries=2,
default_retry_delay=60, default_retry_delay=60,
@@ -1076,6 +1079,7 @@ def sync_segment_task(
@shared_task( @shared_task(
name="iaai_scraper.worker.tasks.sync_vehicle_task", name="iaai_scraper.worker.tasks.sync_vehicle_task",
queue=IAAI_SYNC_QUEUE,
bind=True, bind=True,
max_retries=2, max_retries=2,
default_retry_delay=30, default_retry_delay=30,
@@ -1106,7 +1110,8 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
@shared_task( @shared_task(
name="iaai_scraper.worker.tasks.sync_listing_task", name=SYNC_LISTING_TASK_NAME,
queue=IAAI_SYNC_QUEUE,
bind=True, bind=True,
max_retries=3, max_retries=3,
default_retry_delay=120, default_retry_delay=120,
@@ -1178,7 +1183,7 @@ def sync_listing_task(
try: try:
followup_expires = max(int(lock_ttl), int(delay_seconds) + 300) followup_expires = max(int(lock_ttl), int(delay_seconds) + 300)
self.app.send_task( self.app.send_task(
"iaai_scraper.worker.tasks.sync_listing_task", SYNC_LISTING_TASK_NAME,
kwargs={ kwargs={
"make": make, "make": make,
"model": model, "model": model,
@@ -1186,7 +1191,7 @@ def sync_listing_task(
"limit": limit, "limit": limit,
"only_new": only_new, "only_new": only_new,
}, },
queue="scraping", queue=IAAI_SYNC_QUEUE,
countdown=max(0, int(delay_seconds)), countdown=max(0, int(delay_seconds)),
expires=followup_expires, expires=followup_expires,
) )
@@ -1480,7 +1485,7 @@ def sync_listing_task(
"only_new": effective_only_new, "only_new": effective_only_new,
"is_bootstrap": force_bootstrap_full_scan, "is_bootstrap": force_bootstrap_full_scan,
}, },
queue="scraping", queue=IAAI_SYNC_QUEUE,
) )
dispatched += 1 dispatched += 1
except Exception: except Exception:
@@ -1654,7 +1659,7 @@ def sync_listing_task(
if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl): if _try_set_followup_pending(redis_client, ttl_seconds=followup_ttl):
try: try:
self.app.send_task( self.app.send_task(
"iaai_scraper.worker.tasks.sync_listing_task", SYNC_LISTING_TASK_NAME,
kwargs={ kwargs={
"make": make, "make": make,
"model": model, "model": model,
@@ -1662,7 +1667,7 @@ def sync_listing_task(
"limit": limit, "limit": limit,
"only_new": only_new, "only_new": only_new,
}, },
queue="scraping", queue=IAAI_SYNC_QUEUE,
countdown=10, countdown=10,
expires=followup_ttl, expires=followup_ttl,
) )