qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

Browse Source
This commit is contained in:
qananasikq
2026-04-20 16:27:42 +03:00
parent 55203d33ea
commit 65d5f8e1eb
23 changed files with 3436 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
iaai_scraper/storage/models.py
View File

@@ -80,3 +80,65 @@ class SyncRun(Base):
cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
class VehicleCandidate(Base):
__tablename__ = "vehicle_candidates"
__table_args__ = (
Index("ix_vehicle_candidates_url", "url"),
Index("ix_vehicle_candidates_status_discovered", "status", "discovered_at"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
url: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True) # pending, processing, processed, failed
priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
raw_snapshots: Mapped[list["VehicleRawSnapshot"]] = relationship("VehicleRawSnapshot", back_populates="candidate", cascade="all, delete-orphan")
class VehicleRawSnapshot(Base):
__tablename__ = "vehicle_raw_snapshots"
__table_args__ = (
Index("ix_vehicle_raw_snapshots_candidate_id", "candidate_id"),
Index("ix_vehicle_raw_snapshots_captured_at", "captured_at"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
candidate: Mapped[VehicleCandidate] = relationship("VehicleCandidate", back_populates="raw_snapshots")
captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
method: Mapped[str] = mapped_column(String(20), nullable=False) # http, browser
success: Mapped[bool] = mapped_column(Boolean, nullable=False)
raw_data: Mapped[str | None] = mapped_column(Text, nullable=True) # HTML or JSON content
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
parse_results: Mapped[list["VehicleParseResult"]] = relationship("VehicleParseResult", back_populates="snapshot", cascade="all, delete-orphan")
class VehicleParseResult(Base):
__tablename__ = "vehicle_parse_results"
__table_args__ = (
Index("ix_vehicle_parse_results_snapshot_id", "snapshot_id"),
Index("ix_vehicle_parse_results_parsed_at", "parsed_at"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
snapshot_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True)
snapshot: Mapped[VehicleRawSnapshot] = relationship("VehicleRawSnapshot", back_populates="parse_results")
parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
success: Mapped[bool] = mapped_column(Boolean, nullable=False)
parsed_data: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON with parsed vehicle data
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
class VehicleRetryQueue(Base):
__tablename__ = "vehicle_retry_queue"
__table_args__ = (
Index("ix_vehicle_retry_queue_candidate_id", "candidate_id"),
Index("ix_vehicle_retry_queue_retry_at", "retry_at"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
reason: Mapped[str] = mapped_column(String(50), nullable=False) # parse_failed, capture_failed, etc.
retry_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)