Add detail job storage

2026-08-19 11:18:00 +03:00
parent f3329f76a5
commit 33b85ccecd
5 changed files with 832 additions and 265 deletions
@@ -0,0 +1,73 @@
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+revision: str = "007_runtime_storage"
+down_revision: Union[str, None] = "006_add_details_fetched_at"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute(
+        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de" '
+        'ON "MOBILEDE_cars" (last_seen_at) '
+        "WHERE is_sold = FALSE AND origin_id LIKE 'mobile.de:%'"
+    )
+    op.create_table(
+        "MOBILEDE_detail_jobs",
+        sa.Column("id", sa.BigInteger(), autoincrement=True, nullable=False),
+        sa.Column("car_id", sa.BigInteger(), nullable=False),
+        sa.Column("listing_id", sa.String(length=128), nullable=False),
+        sa.Column("status", sa.String(length=32), server_default="pending", nullable=False),
+        sa.Column("priority", sa.Integer(), server_default="3", nullable=False),
+        sa.Column("attempts", sa.Integer(), server_default="0", nullable=False),
+        sa.Column("available_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("lease_owner", sa.String(length=64), nullable=True),
+        sa.Column("lease_expires_at", sa.DateTime(timezone=True), nullable=True),
+        sa.Column("last_http_status", sa.Integer(), nullable=True),
+        sa.Column("error_kind", sa.String(length=64), nullable=True),
+        sa.Column("last_error", sa.Text(), nullable=True),
+        sa.Column("created_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.Column("updated_at", sa.DateTime(timezone=True), server_default=sa.func.now(), nullable=False),
+        sa.ForeignKeyConstraint(["car_id"], ["MOBILEDE_cars.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+        sa.UniqueConstraint("car_id"),
+        sa.UniqueConstraint("listing_id"),
+    )
+    op.create_index("ix_MOBILEDE_detail_jobs_listing_id", "MOBILEDE_detail_jobs", ["listing_id"])
+    op.create_index("ix_MOBILEDE_detail_jobs_status", "MOBILEDE_detail_jobs", ["status"])
+    op.create_index("ix_MOBILEDE_detail_jobs_available_at", "MOBILEDE_detail_jobs", ["available_at"])
+    op.create_index("ix_MOBILEDE_detail_jobs_lease_expires_at", "MOBILEDE_detail_jobs", ["lease_expires_at"])
+    op.create_index(
+        "ix_MOBILEDE_detail_jobs_ready",
+        "MOBILEDE_detail_jobs",
+        ["status", "available_at", "priority", "created_at"],
+    )
+    op.create_index("ix_MOBILEDE_detail_jobs_lease", "MOBILEDE_detail_jobs", ["status", "lease_expires_at"])
+    op.add_column("MOBILEDE_cars", sa.Column("gallery_fetched_at", sa.DateTime(timezone=True), nullable=True))
+    op.create_index("ix_MOBILEDE_cars_gallery_fetched_at", "MOBILEDE_cars", ["gallery_fetched_at"])
+    op.execute(
+        '''
+        UPDATE "MOBILEDE_cars" AS car
+        SET gallery_fetched_at = car.details_fetched_at
+        WHERE car.details_fetched_at IS NOT NULL
+          AND EXISTS (
+              SELECT 1 FROM "MOBILEDE_images" AS image WHERE image.car_id = car.id
+          )
+        '''
+    )
+
+
+def downgrade() -> None:
+    op.drop_index("ix_MOBILEDE_cars_gallery_fetched_at", table_name="MOBILEDE_cars")
+    op.drop_column("MOBILEDE_cars", "gallery_fetched_at")
+    op.drop_index("ix_MOBILEDE_detail_jobs_lease", table_name="MOBILEDE_detail_jobs")
+    op.drop_index("ix_MOBILEDE_detail_jobs_ready", table_name="MOBILEDE_detail_jobs")
+    op.drop_index("ix_MOBILEDE_detail_jobs_lease_expires_at", table_name="MOBILEDE_detail_jobs")
+    op.drop_index("ix_MOBILEDE_detail_jobs_available_at", table_name="MOBILEDE_detail_jobs")
+    op.drop_index("ix_MOBILEDE_detail_jobs_status", table_name="MOBILEDE_detail_jobs")
+    op.drop_index("ix_MOBILEDE_detail_jobs_listing_id", table_name="MOBILEDE_detail_jobs")
+    op.drop_table("MOBILEDE_detail_jobs")
+    op.execute('DROP INDEX IF EXISTS "ix_MOBILEDE_cars_active_last_seen_mobile_de"')