Update parser

iaai_scraper/worker/celery_app.py

@@ -16,6 +16,9 @@ logger = logging.getLogger("iaai_scraper.worker.celery_app")
 STARTUP_SYNC_DISPATCH_KEY = "iaai:state:startup_sync_dispatched"
 IAAI_SYNC_QUEUE = "iaai_sync"
 PROGRESS_KEY_PREFIX = "iaai:state:task_progress:"
+SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
+SYNC_LAST_COMPLETED_AT_KEY = "iaai:state:sync_listing_last_completed_at"
 
 
 def _env_bool(name: str, default: bool) -> bool:
@@ -43,6 +46,26 @@ def _has_fresh_active_progress(redis_client: Redis, *, max_age_seconds: int = 18
     return False
 
 
+def _seconds_until_next_allowed_sync(redis_client: Redis) -> int:
+    """Запрещает новый автозапуск раньше чем через интервал beat после завершения полного run."""
+    min_interval = max(0, int(settings.celery.beat_sync_interval_minutes * 60))
+    if min_interval <= 0:
+        return 0
+    try:
+        full_done = str(redis_client.get(SYNC_FULL_SCAN_DONE_KEY) or "").strip().lower() in {"1", "true", "yes", "on"}
+        if not full_done:
+            return 0
+        completed_raw = redis_client.get(SYNC_LAST_COMPLETED_AT_KEY)
+        if not completed_raw:
+            return 0
+        completed_at = int(float(completed_raw))
+    except Exception:
+        logger.warning("Failed to inspect last sync completion timestamp", exc_info=True)
+        return 0
+    elapsed = int(time.time()) - completed_at
+    return max(0, min_interval - elapsed)
+
+
 @celery_setup_logging.connect
 def _configure_logging(loglevel=None, **kwargs):
     # Перехватываем логирование Celery и пишем только в stderr (Docker logs).
@@ -147,7 +170,15 @@ def _on_worker_ready(**kwargs):
         )
 
         has_fresh_progress = _has_fresh_active_progress(redis_client)
-        for stale_key in ("iaai:locks:sync_listing",):
+        next_allowed_delay = _seconds_until_next_allowed_sync(redis_client)
+        if next_allowed_delay > 0:
+            logger.info(
+                "Worker ready: last full sync finished recently; next auto sync allowed in %ss, skip startup dispatch",
+                next_allowed_delay,
+            )
+            return
+
+        for stale_key in (SYNC_LISTING_LOCK_KEY,):
             try:
                 ttl = redis_client.ttl(stale_key)
                 if ttl is not None and ttl != -2 and not has_fresh_progress:

iaai_scraper/worker/fast_sync.py

@@ -183,6 +183,7 @@ class FastSyncEngine:
         prepared_rows: list[CarRecord] = []
         db_processed = 0
         retry_candidates: list[FastListingVehicle] = []
+        transient_failure_log_count = 0
         started_details = time.perf_counter()
         with concurrent.futures.ThreadPoolExecutor(max_workers=min(self.fetch_concurrency, len(candidates))) as executor:
             future_to_vehicle = {
@@ -232,7 +233,13 @@ class FastSyncEngine:
                         stats.protection_events += 1
                     if self._is_transient_detail_error(exc):
                         retry_candidates.append(vehicle)
-                        logger.info("Fast detail transient failure queued for retry inventory_id=%s: %s", vehicle.inventory_id, exc)
+                        transient_failure_log_count += 1
+                        if transient_failure_log_count % 100 == 0:
+                            logger.warning(
+                                "Fast detail transient failures queued for retry: count=%d latest_inventory_id=%s",
+                                transient_failure_log_count,
+                                vehicle.inventory_id,
+                            )
                     else:
                         logger.exception("Fast detail parse failed inventory_id=%s: %s", vehicle.inventory_id, exc)
 

iaai_scraper/worker/tasks.py

@@ -47,6 +47,7 @@ HOURLY_FAILURE_STREAK_LIMIT = 3
 HOURLY_FAILURE_STREAK_TTL_SECONDS = 6 * 60 * 60  # сброс через 6 часов
 SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
 SYNC_LISTING_TASK_NAME = "iaai.sync_cars_feed"
+SYNC_LAST_COMPLETED_AT_KEY = "iaai:state:sync_listing_last_completed_at"
 SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}"
 SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress"
 SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total"
@@ -744,6 +745,31 @@ def _clear_followup_pending(redis_client: Redis) -> None:
         logger.warning("Failed to clear follow-up pending flag", exc_info=True)
 
 
+def _mark_sync_completed(redis_client: Redis) -> None:
+    try:
+        redis_client.set(SYNC_LAST_COMPLETED_AT_KEY, str(int(time.time())), ex=7 * 24 * 60 * 60)
+    except Exception:
+        logger.warning("Failed to mark sync completion timestamp", exc_info=True)
+
+
+def _seconds_until_next_allowed_sync(redis_client: Redis, settings: Settings) -> int:
+    min_interval = max(0, int(settings.celery.beat_sync_interval_minutes * 60))
+    if min_interval <= 0:
+        return 0
+    try:
+        if not _is_full_scan_done(redis_client):
+            return 0
+        completed_raw = redis_client.get(SYNC_LAST_COMPLETED_AT_KEY)
+        if not completed_raw:
+            return 0
+        completed_at = int(float(completed_raw))
+    except Exception:
+        logger.warning("Failed to inspect next allowed sync time", exc_info=True)
+        return 0
+    elapsed = int(time.time()) - completed_at
+    return max(0, min_interval - elapsed)
+
+
 def _bump_bootstrap_failure_streak(
     redis_client: Redis,
     *,
@@ -1139,6 +1165,11 @@ def sync_listing_task(
     watchdog_stop: Event | None = None
     watchdog_thread: Thread | None = None
     force_bootstrap_full_scan = False
+    settings = Settings()
+    followup_min_delay_seconds = max(
+        0,
+        int(os.getenv("IAAI_SYNC_FOLLOWUP_MIN_DELAY_SECONDS", str(settings.celery.beat_sync_interval_minutes * 60))),
+    )
 
     def _enqueue_bootstrap_followup(
         reason: str,
@@ -1146,6 +1177,7 @@ def sync_listing_task(
         *,
         count_as_failure: bool = False,
     ) -> None:
+        delay_seconds = max(int(delay_seconds), followup_min_delay_seconds)
         flag_ttl = max(lock_ttl, delay_seconds + 300)
         if not _try_set_followup_pending(redis_client, ttl_seconds=flag_ttl):
             logger.info(
@@ -1220,7 +1252,18 @@ def sync_listing_task(
         }
 
     try:
-        settings = Settings()
+        next_allowed_delay = _seconds_until_next_allowed_sync(redis_client, settings)
+        if next_allowed_delay > 0:
+            logger.info(
+                "sync_listing_task skipped: previous full run finished recently; next run allowed in %ss",
+                next_allowed_delay,
+            )
+            return {
+                "status": "skipped",
+                "reason": "next_sync_not_due_yet",
+                "retry_after_seconds": next_allowed_delay,
+                "task_id": task_id,
+            }
 
         # Любой реально стартовавший sync_listing снимает pending-флаг followup,
         # чтобы watchdog/continuation могли корректно планировать следующий run
@@ -1641,6 +1684,7 @@ def sync_listing_task(
             summary["cars_failed"],
             summary["failures_count"],
         )
+        _mark_sync_completed(redis_client)
         return summary
 
     except SoftTimeLimitExceeded: