Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

iaai_scraper/storage/db.py

@@ -14,6 +14,13 @@ from .schemas import CarRecord
 logger = logging.getLogger("iaai_scraper.db")
 
 
+def get_db_session() -> Session:
+    """Получить новую сессию базы данных."""
+    settings = Settings()
+    persistence = PersistenceService(settings)
+    return persistence.session_factory()
+
+
 CAR_DB_FIELDS = {
 	col.key for col in Car.__table__.columns
 	if col.key not in ("id",)
@@ -517,21 +524,38 @@ class PersistenceService:
 		is_postgres = "postgresql" in self.settings.database.url
 
 		with self.session_scope() as session:
+			# Страховка от ложного mark_sold при битом/неполном full-scan:
+			# если текущий список активных URL аномально мал относительно уже активных машин в БД,
+			# ничего не помечаем проданным.
+			active_db_count = int(session.execute(
+				select(func.count())
+				.select_from(Car)
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like(f"{lane}:%"))
+			).scalar_one() or 0)
+
+			current_active_count = len(active_origin_urls)
+			if active_db_count >= 1000 and current_active_count < max(500, int(active_db_count * 0.25)):
+				logger.warning(
+					"Skipping mark_sold: suspiciously small active set (%d URLs vs %d active in DB)",
+					current_active_count,
+					active_db_count,
+				)
+				return 0
+
 			if is_postgres:
 				# Создаём временную таблицу с активными URL.
-				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT PRIMARY KEY) ON COMMIT DROP"))
 				session.execute(text("TRUNCATE _active_urls"))
 
-				# Вставляем активные URL чанками.
+				# Вставляем активные URL чанками через executemany.
 				url_list = list(active_origin_urls)
 				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
 					chunk = url_list[i:i + _IN_CHUNK_SIZE]
-					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
-					params = {f"u{j}": url for j, url in enumerate(chunk)}
-					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
-
-				# Создаём индекс на временной таблице для ускорения JOIN.
-				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
+					session.execute(
+						text("INSERT INTO _active_urls (url) VALUES (:url) ON CONFLICT DO NOTHING"),
+						[{"url": url} for url in chunk],
+					)
 
 				# Массовая пометка проданных в PostgreSQL.
 				result = session.execute(text("""
@@ -543,10 +567,10 @@ class PersistenceService:
 						LEFT JOIN _active_urls a ON c.origin_url = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
-						AND c.origin_id LIKE 'iaai:%%'
+						AND c.origin_id LIKE :lane_prefix
 					) sub
 					WHERE cars.id = sub.id
-				"""))
+				"""), {"lane_prefix": f"{lane}:%"})
 				count = result.rowcount or 0
 			else:
 				# Упрощённый путь для SQLite.
@@ -554,7 +578,7 @@ class PersistenceService:
 					update(Car)
 					.where(Car.origin_url.notin_(active_origin_urls))
 					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like("iaai:%"))
+					.where(Car.origin_id.like(f"{lane}:%"))
 					.values(is_sold=True)
 				)
 				result = session.execute(stmt)
@@ -573,4 +597,43 @@ class PersistenceService:
 			result = session.execute(
 				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
 			)
-			return {str(row[0]) for row in result if row and row[0]}
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все активные (не sold) origin_url для указанного lane/prefix."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		*,
+		prefix: str = "iaai:",
+		offset: int = 0,
+		limit: int = 3000,
+	) -> list[str]:
+		"""Возвращает батч активных origin_url для циклического hourly refresh."""
+		with self.session_scope() as session:
+			rows = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.order_by(Car.last_seen_at.asc(), Car.id.asc())
+				.offset(max(0, int(offset)))
+				.limit(max(1, int(limit)))
+			).all()
+			return [str(row[0]) for row in rows if row and row[0]]
+
+	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		with self.session_scope() as session:
+			return int(session.execute(
+				select(func.count())
+				.select_from(Car)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+			).scalar_one() or 0)