Prepare mobile de parser release

alembic/env.py

@@ -0,0 +1,64 @@
+# Alembic env.py — подключение к БД через Settings.
+
+import os
+import sys
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+# Добавляем корень проекта в sys.path
+sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
+
+from iaai_scraper.core.config import Settings
+from iaai_scraper.storage.models import Base
+
+config = context.config
+VERSION_TABLE = "iaai_parser_alembic_version"
+
+# Logging
+if config.config_file_name is not None:
+    fileConfig(config.config_file_name)
+
+# Подставляем URL из Settings (env vars имеют приоритет)
+settings = Settings()
+config.set_main_option("sqlalchemy.url", settings.database.url)
+
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+    # Run migrations in 'offline' mode.
+    url = config.get_main_option("sqlalchemy.url")
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        version_table=VERSION_TABLE,
+        literal_binds=True,
+        dialect_opts={"paramstyle": "named"},
+    )
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online() -> None:
+    # Run migrations in 'online' mode.
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section, {}),
+        prefix="sqlalchemy.",
+        poolclass=pool.NullPool,
+    )
+    with connectable.connect() as connection:
+        context.configure(
+            connection=connection,
+            target_metadata=target_metadata,
+            version_table=VERSION_TABLE,
+        )
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

alembic/script.py.mako

@@ -0,0 +1,25 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+    ${downgrades if downgrades else "pass"}

alembic/versions/001_initial.py

@@ -0,0 +1,103 @@
+# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+revision: str = "001_initial"
+down_revision: Union[str, None] = None
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def _schema_name() -> str | None:
+    bind = op.get_bind()
+    return "public" if bind.dialect.name == "postgresql" else None
+
+
+def _table_exists(table_name: str) -> bool:
+    inspector = sa.inspect(op.get_bind())
+    return table_name in inspector.get_table_names(schema=_schema_name())
+
+
+def _index_exists(table_name: str, index_name: str) -> bool:
+    if not _table_exists(table_name):
+        return False
+    inspector = sa.inspect(op.get_bind())
+    indexes = inspector.get_indexes(table_name, schema=_schema_name())
+    return any(index.get("name") == index_name for index in indexes)
+
+
+def upgrade() -> None:
+    # Таблица iaai_cars — аукционные машины с IAAI
+    if not _table_exists("iaai_cars"):
+        op.create_table(
+            "iaai_cars",
+            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+            sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
+            sa.Column("brand", sa.String(50), nullable=False),
+            sa.Column("model", sa.String(50), nullable=False),
+            sa.Column("year", sa.Integer, nullable=True),
+            sa.Column("price", sa.BigInteger, nullable=True),
+            sa.Column("currency", sa.String(10), nullable=False, server_default="USD"),
+            sa.Column("mileage", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("country", sa.String(10), nullable=False, server_default="NA"),
+            sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("color", sa.String, nullable=False, server_default="other"),
+            sa.Column("drive", sa.String(10), nullable=True),
+            sa.Column("gearbox", sa.String(10), nullable=True),
+            sa.Column("steering_wheel", sa.String(10), nullable=True),
+            sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"),
+            sa.Column("engine_volume", sa.Integer, nullable=True),
+            sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"),
+            sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("origin", sa.String(20), nullable=False, server_default="NA"),
+            sa.Column("origin_url", sa.String, nullable=False),
+            sa.Column("origin_id", sa.String, nullable=False, unique=True),
+            sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("evaluation", sa.String, nullable=True),
+            sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")),
+            sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
+            sa.Column("slug", sa.String, nullable=False),
+            sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+        )
+
+    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"):
+        op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"])
+    if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"):
+        op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True)
+
+    # Таблица iaai_images — изображения машин
+    if not _table_exists("iaai_images"):
+        op.create_table(
+            "iaai_images",
+            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+            sa.Column("fullres_image", sa.String, nullable=False),
+            sa.Column("preview_image", sa.String, nullable=False),
+            sa.Column("order_index", sa.Integer, nullable=False),
+            sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False),
+        )
+
+    # Таблица iaai_sync_runs — логирование синхронизаций
+    if not _table_exists("iaai_sync_runs"):
+        op.create_table(
+            "iaai_sync_runs",
+            sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
+            sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
+            sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
+            sa.Column("status", sa.Text, nullable=False),
+            sa.Column("lane", sa.Text, nullable=False),
+            sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"),
+            sa.Column("error_summary", sa.Text, nullable=True),
+        )
+
+
+def downgrade() -> None:
+    # Compatibility-only migration for shared production databases.
+    pass

alembic/versions/002_add_indexes.py

@@ -0,0 +1,24 @@
+# Индексы производительности
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "002_add_indexes"
+down_revision: Union[str, None] = "001_initial"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)")
+    op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)")
+
+
+def downgrade() -> None:
+    # Compatibility-only migration for shared production databases.
+    pass

alembic/versions/003_add_composite_indexes.py

@@ -0,0 +1,36 @@
+# Индексы для масштабированной БД (20k+ записей)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "003_add_composite_indexes"
+down_revision: Union[str, None] = "002_add_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Partial index для активных машин IAAI (is_sold = FALSE)
+    # Ускоряет поиск при mark_sold операциях
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai "
+        "ON iaai_cars (origin_url) "
+        "WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
+    )
+
+    # Composite index для проверки дубликатов изображений
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres "
+        "ON iaai_images (car_id, fullres_image)"
+    )
+
+    # Index для быстрого поиска existing машин по origin_url
+    op.execute(
+        "CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id "
+        "ON iaai_cars (origin_url, origin_id)"
+    )
+
+
+def downgrade() -> None:
+    # Compatibility-only migration for shared production databases.
+    pass

alembic/versions/004_add_ingestion_tables.py

@@ -0,0 +1,17 @@
+# Placeholder migration (ingestion tables not yet needed)
+from typing import Sequence, Union
+
+from alembic import op
+
+revision: str = "004_add_ingestion_tables"
+down_revision: Union[str, None] = "003_add_composite_indexes"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    pass
+
+
+def downgrade() -> None:
+    pass