Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

2026-04-20 16:27:42 +03:00
parent 55203d33ea
commit 65d5f8e1eb
23 changed files with 3436 additions and 518 deletions
--- a/iaai_scraper/storage/db.py
+++ b/iaai_scraper/storage/db.py
@@ -14,6 +14,13 @@ from .schemas import CarRecord
 logger = logging.getLogger("iaai_scraper.db")


+def get_db_session() -> Session:
+    """Получить новую сессию базы данных."""
+    settings = Settings()
+    persistence = PersistenceService(settings)
+    return persistence.session_factory()
+
+
 CAR_DB_FIELDS = {
 	col.key for col in Car.__table__.columns
 	if col.key not in ("id",)
@@ -517,21 +524,38 @@ class PersistenceService:
 		is_postgres = "postgresql" in self.settings.database.url

 		with self.session_scope() as session:
+			# Страховка от ложного mark_sold при битом/неполном full-scan:
+			# если текущий список активных URL аномально мал относительно уже активных машин в БД,
+			# ничего не помечаем проданным.
+			active_db_count = int(session.execute(
+				select(func.count())
+				.select_from(Car)
+				.where(Car.is_sold == False)  # noqa: E712
+				.where(Car.origin_id.like(f"{lane}:%"))
+			).scalar_one() or 0)
+
+			current_active_count = len(active_origin_urls)
+			if active_db_count >= 1000 and current_active_count < max(500, int(active_db_count * 0.25)):
+				logger.warning(
+					"Skipping mark_sold: suspiciously small active set (%d URLs vs %d active in DB)",
+					current_active_count,
+					active_db_count,
+				)
+				return 0
+
 			if is_postgres:
 				# Создаём временную таблицу с активными URL.
-				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
+				session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT PRIMARY KEY) ON COMMIT DROP"))
 				session.execute(text("TRUNCATE _active_urls"))

-				# Вставляем активные URL чанками.
+				# Вставляем активные URL чанками через executemany.
 				url_list = list(active_origin_urls)
 				for i in range(0, len(url_list), _IN_CHUNK_SIZE):
 					chunk = url_list[i:i + _IN_CHUNK_SIZE]
-					values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
-					params = {f"u{j}": url for j, url in enumerate(chunk)}
-					session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
-
-				# Создаём индекс на временной таблице для ускорения JOIN.
-				session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
+					session.execute(
+						text("INSERT INTO _active_urls (url) VALUES (:url) ON CONFLICT DO NOTHING"),
+						[{"url": url} for url in chunk],
+					)

 				# Массовая пометка проданных в PostgreSQL.
 				result = session.execute(text("""
@@ -543,10 +567,10 @@ class PersistenceService:
 						LEFT JOIN _active_urls a ON c.origin_url = a.url
 						WHERE a.url IS NULL
 						AND c.is_sold = FALSE
-						AND c.origin_id LIKE 'iaai:%%'
+						AND c.origin_id LIKE :lane_prefix
 					) sub
 					WHERE cars.id = sub.id
-				"""))
+				"""), {"lane_prefix": f"{lane}:%"})
 				count = result.rowcount or 0
 			else:
 				# Упрощённый путь для SQLite.
@@ -554,7 +578,7 @@ class PersistenceService:
 					update(Car)
 					.where(Car.origin_url.notin_(active_origin_urls))
 					.where(Car.is_sold == False)  # noqa: E712
-					.where(Car.origin_id.like("iaai:%"))
+					.where(Car.origin_id.like(f"{lane}:%"))
 					.values(is_sold=True)
 				)
 				result = session.execute(stmt)
@@ -573,4 +597,43 @@ class PersistenceService:
 			result = session.execute(
 				select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
 			)
-			return {str(row[0]) for row in result if row and row[0]}
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
+		"""Возвращает все активные (не sold) origin_url для указанного lane/prefix."""
+		with self.session_scope() as session:
+			result = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.execution_options(yield_per=10000)
+			)
+			return {str(row[0]) for row in result if row and row[0]}
+
+	def get_active_origin_urls_batch_for_refresh(
+		self,
+		*,
+		prefix: str = "iaai:",
+		offset: int = 0,
+		limit: int = 3000,
+	) -> list[str]:
+		"""Возвращает батч активных origin_url для циклического hourly refresh."""
+		with self.session_scope() as session:
+			rows = session.execute(
+				select(Car.origin_url)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+				.order_by(Car.last_seen_at.asc(), Car.id.asc())
+				.offset(max(0, int(offset)))
+				.limit(max(1, int(limit)))
+			).all()
+			return [str(row[0]) for row in rows if row and row[0]]
+
+	def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
+		with self.session_scope() as session:
+			return int(session.execute(
+				select(func.count())
+				.select_from(Car)
+				.where(Car.origin_id.like(f"{prefix}%"))
+				.where(Car.is_sold == False)  # noqa: E712
+			).scalar_one() or 0)
--- a/iaai_scraper/storage/models.py
+++ b/iaai_scraper/storage/models.py
@@ -80,3 +80,65 @@ class SyncRun(Base):
 	cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
 	error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class VehicleCandidate(Base):
+	__tablename__ = "vehicle_candidates"
+	__table_args__ = (
+		Index("ix_vehicle_candidates_url", "url"),
+		Index("ix_vehicle_candidates_status_discovered", "status", "discovered_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	url: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
+	discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True)  # pending, processing, processed, failed
+	priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
+	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	raw_snapshots: Mapped[list["VehicleRawSnapshot"]] = relationship("VehicleRawSnapshot", back_populates="candidate", cascade="all, delete-orphan")
+
+
+class VehicleRawSnapshot(Base):
+	__tablename__ = "vehicle_raw_snapshots"
+	__table_args__ = (
+		Index("ix_vehicle_raw_snapshots_candidate_id", "candidate_id"),
+		Index("ix_vehicle_raw_snapshots_captured_at", "captured_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
+	candidate: Mapped[VehicleCandidate] = relationship("VehicleCandidate", back_populates="raw_snapshots")
+	captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	method: Mapped[str] = mapped_column(String(20), nullable=False)  # http, browser
+	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
+	raw_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # HTML or JSON content
+	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+	parse_results: Mapped[list["VehicleParseResult"]] = relationship("VehicleParseResult", back_populates="snapshot", cascade="all, delete-orphan")
+
+
+class VehicleParseResult(Base):
+	__tablename__ = "vehicle_parse_results"
+	__table_args__ = (
+		Index("ix_vehicle_parse_results_snapshot_id", "snapshot_id"),
+		Index("ix_vehicle_parse_results_parsed_at", "parsed_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	snapshot_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True)
+	snapshot: Mapped[VehicleRawSnapshot] = relationship("VehicleRawSnapshot", back_populates="parse_results")
+	parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
+	success: Mapped[bool] = mapped_column(Boolean, nullable=False)
+	parsed_data: Mapped[str | None] = mapped_column(Text, nullable=True)  # JSON with parsed vehicle data
+	error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
+
+
+class VehicleRetryQueue(Base):
+	__tablename__ = "vehicle_retry_queue"
+	__table_args__ = (
+		Index("ix_vehicle_retry_queue_candidate_id", "candidate_id"),
+		Index("ix_vehicle_retry_queue_retry_at", "retry_at"),
+	)
+	id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
+	candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
+	reason: Mapped[str] = mapped_column(String(50), nullable=False)  # parse_failed, capture_failed, etc.
+	retry_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
+	attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
+	max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)