qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix scraper flow

Browse Source
This commit is contained in:
qananasikq
2026-04-22 21:40:40 +03:00
parent 133c125e9c
commit d5d6ba8b7c
33 changed files with 2632 additions and 2811 deletions
Download Patch File Download Diff File Expand all files Collapse all files

66
alembic/versions/004_add_ingestion_tables.py
View File

@@ -1,8 +1,7 @@
# Добавление таблиц для новой архитектуры ingestion: candidates, raw snapshots, parse results, retry queue
# Placeholder migration (ingestion tables not yet needed)
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
revision: str = "004_add_ingestion_tables"
down_revision: Union[str, None] = "003_add_composite_indexes"
@@ -11,67 +10,8 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Таблица vehicle_candidates
op.create_table(
"vehicle_candidates",
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
sa.Column("url", sa.String(), nullable=False, unique=True),
sa.Column("discovered_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("status", sa.String(20), nullable=False, server_default="pending"),
sa.Column("priority", sa.Integer(), nullable=False, server_default="0"),
sa.Column("last_attempt_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
)
op.create_index("ix_vehicle_candidates_url", "vehicle_candidates", ["url"])
op.create_index("ix_vehicle_candidates_status_discovered", "vehicle_candidates", ["status", "discovered_at"])
# Таблица vehicle_raw_snapshots
op.create_table(
"vehicle_raw_snapshots",
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False),
sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("method", sa.String(20), nullable=False),
sa.Column("success", sa.Boolean(), nullable=False),
sa.Column("raw_data", sa.Text(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
)
op.create_index("ix_vehicle_raw_snapshots_candidate_id", "vehicle_raw_snapshots", ["candidate_id"])
op.create_index("ix_vehicle_raw_snapshots_captured_at", "vehicle_raw_snapshots", ["captured_at"])
# Таблица vehicle_parse_results
op.create_table(
"vehicle_parse_results",
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
sa.Column("snapshot_id", sa.Integer(), sa.ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False),
sa.Column("parsed_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("success", sa.Boolean(), nullable=False),
sa.Column("parsed_data", sa.Text(), nullable=True),
sa.Column("error_message", sa.Text(), nullable=True),
)
op.create_index("ix_vehicle_parse_results_snapshot_id", "vehicle_parse_results", ["snapshot_id"])
op.create_index("ix_vehicle_parse_results_parsed_at", "vehicle_parse_results", ["parsed_at"])
# Таблица vehicle_retry_queue
op.create_table(
"vehicle_retry_queue",
sa.Column("id", sa.BigInteger().with_variant(sa.Integer(), "sqlite"), primary_key=True, autoincrement=True),
sa.Column("candidate_id", sa.Integer(), sa.ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False),
sa.Column("reason", sa.String(50), nullable=False),
sa.Column("retry_at", sa.DateTime(timezone=True), nullable=False),
sa.Column("attempts", sa.Integer(), nullable=False, server_default="0"),
sa.Column("max_attempts", sa.Integer(), nullable=False, server_default="3"),
)
op.create_index("ix_vehicle_retry_queue_candidate_id", "vehicle_retry_queue", ["candidate_id"])
op.create_index("ix_vehicle_retry_queue_retry_at", "vehicle_retry_queue", ["retry_at"])
pass
def downgrade() -> None:
op.drop_table("vehicle_retry_queue")
op.drop_table("vehicle_parse_results")
op.drop_table("vehicle_raw_snapshots")
op.drop_table("vehicle_candidates")
pass