fast parser

iaai_scraper/worker/self_heal.py

@@ -12,8 +12,12 @@ logger = logging.getLogger("iaai_scraper.worker.self_heal")
 
 IAAI_SYNC_QUEUE = "iaai_sync"
 GLOBAL_PROGRESS_TS_KEY = "iaai:state:last_progress_ts"
+GLOBAL_DB_PROGRESS_TS_KEY = "iaai:state:last_db_progress_ts"
 SELF_HEAL_RESTART_LOCK_KEY = "iaai:state:self_heal_restart_in_progress"
 SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
+SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
+SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
+SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
 SIGKILL_FALLBACK = getattr(signal, "SIGKILL", signal.SIGTERM)
 
 
@@ -79,6 +83,41 @@ def _read_last_progress_ts(redis_client: Redis) -> int | None:
 	return max_ts or None
 
 
+def _read_last_db_progress_ts(redis_client: Redis) -> int | None:
+	raw = redis_client.get(GLOBAL_DB_PROGRESS_TS_KEY)
+	if raw:
+		ts = _safe_int(raw)
+		if ts > 0:
+			return ts
+
+	max_ts = 0
+	for key in redis_client.scan_iter(match="iaai:state:task_progress:*"):
+		try:
+			payload = redis_client.get(key)
+			if not payload:
+				continue
+			data = json.loads(payload)
+			if str(data.get("stage") or "") == "fast_db_progress":
+				ts = _safe_int(data.get("ts"), 0)
+			else:
+				ts = _safe_int(data.get("last_db_progress_ts"), 0)
+			if ts > max_ts:
+				max_ts = ts
+		except Exception:
+			continue
+	return max_ts or None
+
+
+def _reset_bootstrap_checkpoint_for_db_idle(redis_client: Redis) -> None:
+	pipe = redis_client.pipeline()
+	pipe.delete(SYNC_LISTING_CHECKPOINT_KEY)
+	pipe.delete(SYNC_LISTING_FOLLOWUP_PENDING_KEY)
+	pipe.delete(GLOBAL_PROGRESS_TS_KEY)
+	pipe.delete(GLOBAL_DB_PROGRESS_TS_KEY)
+	pipe.set(SYNC_FULL_SCAN_DONE_KEY, "0")
+	pipe.execute()
+
+
 def _has_inflight_work(redis_client: Redis, queue_name: str) -> tuple[bool, dict[str, int]]:
 	"""Есть ли признаки активной/зависшей работы, даже если очередь пуста."""
 	queue_len = _safe_int(redis_client.llen(queue_name), 0)
@@ -135,14 +174,16 @@ def main() -> None:
 	queue_name = os.getenv("IAAI_CELERY_QUEUE", IAAI_SYNC_QUEUE)
 	check_interval = max(5, _env_int("IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS", 30))
 	stall_seconds = max(180, _env_int("IAAI_SELF_HEAL_STALL_SECONDS", 720))
+	db_idle_seconds = max(60, _env_int("IAAI_DB_IDLE_RESTART_SECONDS", 3600))
 	startup_grace = max(30, _env_int("IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS", 300))
 	restart_cooldown = max(60, _env_int("IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS", 300))
 
-	logger.warning(
-		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss startup_grace=%ss cooldown=%ss",
+	logger.info(
+		"Self-heal watchdog enabled: queue=%s check_interval=%ss stall=%ss db_idle=%ss startup_grace=%ss cooldown=%ss",
 		queue_name,
 		check_interval,
 		stall_seconds,
+		db_idle_seconds,
 		startup_grace,
 		restart_cooldown,
 	)
@@ -175,9 +216,22 @@ def main() -> None:
 				)
 				age = stall_seconds + 1
 
+			restart_reason = f"progress_age={age}s > {stall_seconds}s"
+			db_idle_restart = False
 			if age <= stall_seconds:
-				time.sleep(check_interval)
-				continue
+				last_db_ts = _read_last_db_progress_ts(redis_client)
+				db_age = None if last_db_ts is None else max(0, now_ts - int(last_db_ts))
+				if db_age is None:
+					first_allowed_ts = int(started_at) + max(startup_grace, db_idle_seconds)
+					if now_ts < first_allowed_ts:
+						time.sleep(check_interval)
+						continue
+					db_age = db_idle_seconds + 1
+				if db_age <= db_idle_seconds:
+					time.sleep(check_interval)
+					continue
+				db_idle_restart = True
+				restart_reason = f"db_idle_age={db_age}s > {db_idle_seconds}s"
 
 			# Глобальный anti-storm lock: чтобы много воркеров не рестартились одновременно.
 			acquired = bool(
@@ -193,11 +247,13 @@ def main() -> None:
 				continue
 
 			logger.error(
-				"Self-heal: detected global stall (inflight=%s, progress_age=%ss > %ss). Restarting worker process...",
+				"Self-heal: detected stall (inflight=%s, %s). Restarting worker process...",
 				inflight,
-				age,
-				stall_seconds,
+				restart_reason,
 			)
+			if db_idle_restart:
+				logger.error("Self-heal: no DB writes for too long; clearing checkpoint to restart from segment 1")
+				_reset_bootstrap_checkpoint_for_db_idle(redis_client)
 			# Небольшой джиттер, чтобы при одинаковом событии у разных контейнеров
 			# перезапуск был не строго одновременно.
 			time.sleep(random.uniform(0.3, 2.0))