77 lines
4.0 KiB
Python
77 lines
4.0 KiB
Python
# Добавление таблиц для новой архитектуры ingestion: candidates, raw snapshots, parse results, retry queue
|
|
from typing import Sequence, Union
|
|
|
|
from alembic import op
|
|
import sqlalchemy as sa
|
|
|
|
revision: str = "004_add_ingestion_tables"
|
|
down_revision: Union[str, None] = "003_add_composite_indexes"
|
|
branch_labels: Union[str, Sequence[str], None] = None
|
|
depends_on: Union[str, Sequence[str], None] = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
# Таблица vehicle_candidates
|
|
op.create_table(
|
|
"vehicle_candidates",
|
|
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
|
|
sa.Column("url", sa.String(), nullable=False, unique=True),
|
|
sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
|
|
sa.Column("priority", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
|
|
)
|
|
|
|
op.create_index("ix_vehicle_candidates_url", "vehicle_candidates", ["url"])
|
|
op.create_index("ix_vehicle_candidates_status_discovered", "vehicle_candidates", ["status", "discovered_at"])
|
|
|
|
# Таблица vehicle_raw_snapshots
|
|
op.create_table(
|
|
"vehicle_raw_snapshots",
|
|
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
|
|
sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("method", sa.String(20), nullable=False),
|
|
sa.Column("success", sa.Boolean(), nullable=False),
|
|
sa.Column("raw_data", sa.Text(), nullable=True),
|
|
sa.Column("error_message", sa.Text(), nullable=True),
|
|
)
|
|
|
|
op.create_index("ix_vehicle_raw_snapshots_candidate_id", "vehicle_raw_snapshots", ["candidate_id"])
|
|
op.create_index("ix_vehicle_raw_snapshots_captured_at", "vehicle_raw_snapshots", ["captured_at"])
|
|
|
|
# Таблица vehicle_parse_results
|
|
op.create_table(
|
|
"vehicle_parse_results",
|
|
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
|
|
sa.Column("snapshot_id", sa.Integer(), sa.ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("parsed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
|
|
sa.Column("success", sa.Boolean(), nullable=False),
|
|
sa.Column("parsed_data", sa.Text(), nullable=True),
|
|
sa.Column("error_message", sa.Text(), nullable=True),
|
|
)
|
|
|
|
op.create_index("ix_vehicle_parse_results_snapshot_id", "vehicle_parse_results", ["snapshot_id"])
|
|
op.create_index("ix_vehicle_parse_results_parsed_at", "vehicle_parse_results", ["parsed_at"])
|
|
|
|
# Таблица vehicle_retry_queue
|
|
op.create_table(
|
|
"vehicle_retry_queue",
|
|
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
|
|
sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False),
|
|
sa.Column("reason", sa.String(50), nullable=False),
|
|
sa.Column("retry_at", sa.DateTime(timezone=True), nullable=False),
|
|
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
|
|
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
|
|
)
|
|
|
|
op.create_index("ix_vehicle_retry_queue_candidate_id", "vehicle_retry_queue", ["candidate_id"])
|
|
op.create_index("ix_vehicle_retry_queue_retry_at", "vehicle_retry_queue", ["retry_at"])
|
|
|
|
|
|
def downgrade() -> None:
|
|
op.drop_table("vehicle_retry_queue")
|
|
op.drop_table("vehicle_parse_results")
|
|
op.drop_table("vehicle_raw_snapshots")
|
|
op.drop_table("vehicle_candidates") |