From 3ce78666434b513c25d378713f8fba7497fb420d Mon Sep 17 00:00:00 2001 From: qananasikq Date: Thu, 16 Apr 2026 18:11:06 +0300 Subject: [PATCH] update alembic migrations --- alembic.ini | 40 ++++++++++ alembic/env.py | 58 ++++++++++++++ alembic/script.py.mako | 25 ++++++ alembic/versions/001_initial.py | 79 +++++++++++++++++++ alembic/versions/002_add_indexes.py | 29 +++++++ alembic/versions/003_add_composite_indexes.py | 38 +++++++++ alembic/versions/004_widen_string_columns.py | 25 ++++++ .../versions/005_unique_images_constraint.py | 33 ++++++++ alembic/versions/006_image_car_id_bigint.py | 31 ++++++++ 9 files changed, 358 insertions(+) create mode 100644 alembic.ini create mode 100644 alembic/env.py create mode 100644 alembic/script.py.mako create mode 100644 alembic/versions/001_initial.py create mode 100644 alembic/versions/002_add_indexes.py create mode 100644 alembic/versions/003_add_composite_indexes.py create mode 100644 alembic/versions/004_widen_string_columns.py create mode 100644 alembic/versions/005_unique_images_constraint.py create mode 100644 alembic/versions/006_image_car_id_bigint.py diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..8d1b2a3 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,40 @@ +# Alembic Configuration File + +[alembic] +script_location = alembic +prepend_sys_path = . +sqlalchemy.url = postgresql+psycopg2://encar:encar@localhost:5434/encar_db + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/alembic/env.py b/alembic/env.py new file mode 100644 index 0000000..375e0e3 --- /dev/null +++ b/alembic/env.py @@ -0,0 +1,58 @@ +# Alembic env.py — подключение к БД через Settings. + +import os +import sys +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +# Добавляем корень проекта в sys.path +sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))) + +from encar_scraper.core.config import Settings +from encar_scraper.storage.models import Base + +config = context.config + +# Logging +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Подставляем URL из Settings (env vars имеют приоритет) +settings = Settings() +config.set_main_option("sqlalchemy.url", settings.database.url) + +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + # Run migrations in 'offline' mode. + url = config.get_main_option("sqlalchemy.url") + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + # Run migrations in 'online' mode. + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/alembic/script.py.mako b/alembic/script.py.mako new file mode 100644 index 0000000..958df87 --- /dev/null +++ b/alembic/script.py.mako @@ -0,0 +1,25 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py new file mode 100644 index 0000000..a533529 --- /dev/null +++ b/alembic/versions/001_initial.py @@ -0,0 +1,79 @@ +# Начальная схема: cars, images, sync_runs +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa + +revision: str = "001_initial" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Таблица cars — автомобили с Encar + op.create_table( + "cars", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("parser_id", sa.String(50), nullable=False, unique=True), + sa.Column("brand", sa.String(50), nullable=False), + sa.Column("model", sa.String(50), nullable=False), + sa.Column("year", sa.Integer, nullable=True), + sa.Column("price", sa.BigInteger, nullable=True), + sa.Column("currency", sa.String(10), nullable=False, server_default="USD"), + sa.Column("mileage", sa.Integer, nullable=False, server_default="0"), + sa.Column("country", sa.String(10), nullable=False, server_default="NA"), + sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("color", sa.String, nullable=False, server_default="other"), + sa.Column("drive", sa.String(10), nullable=True), + sa.Column("gearbox", sa.String(10), nullable=True), + sa.Column("steering_wheel", sa.String(10), nullable=True), + sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"), + sa.Column("engine_volume", sa.Integer, nullable=True), + sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"), + sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("origin", sa.String(20), nullable=False, server_default="NA"), + sa.Column("origin_url", sa.String, nullable=False), + sa.Column("origin_id", sa.String, nullable=False, unique=True), + sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("evaluation", sa.String, nullable=True), + sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")), + sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")), + sa.Column("slug", sa.String, nullable=False), + sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + ) + op.create_index("ix_cars_origin_url", "cars", ["origin_url"]) + op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True) + + # Таблица images — изображения машин + op.create_table( + "images", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("fullres_image", sa.String, nullable=False), + sa.Column("preview_image", sa.String, nullable=False), + sa.Column("order_index", sa.Integer, nullable=False), + sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False), + ) + + # Таблица sync_runs — логирование синхронизаций + op.create_table( + "sync_runs", + sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True), + sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()), + sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True), + sa.Column("status", sa.Text, nullable=False), + sa.Column("lane", sa.Text, nullable=False), + sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"), + sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"), + sa.Column("error_summary", sa.Text, nullable=True), + ) + +def downgrade() -> None: + op.drop_table("sync_runs") + op.drop_table("images") + op.drop_table("cars") diff --git a/alembic/versions/002_add_indexes.py b/alembic/versions/002_add_indexes.py new file mode 100644 index 0000000..2c7b631 --- /dev/null +++ b/alembic/versions/002_add_indexes.py @@ -0,0 +1,29 @@ +# Индексы производительности +from typing import Sequence, Union + +from alembic import op + +revision: str = "002_add_indexes" +down_revision: Union[str, None] = "001_initial" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + op.create_index("ix_cars_brand", "cars", ["brand"]) + op.create_index("ix_cars_brand_model", "cars", ["brand", "model"]) + op.create_index("ix_cars_year", "cars", ["year"]) + op.create_index("ix_cars_is_sold", "cars", ["is_sold"]) + op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"]) + op.create_index("ix_images_car_id", "images", ["car_id"]) + op.create_index("ix_sync_runs_status", "sync_runs", ["status"]) + + +def downgrade() -> None: + op.drop_index("ix_sync_runs_status", table_name="sync_runs") + op.drop_index("ix_images_car_id", table_name="images") + op.drop_index("ix_cars_last_seen_at", table_name="cars") + op.drop_index("ix_cars_is_sold", table_name="cars") + op.drop_index("ix_cars_year", table_name="cars") + op.drop_index("ix_cars_brand_model", table_name="cars") + op.drop_index("ix_cars_brand", table_name="cars") diff --git a/alembic/versions/003_add_composite_indexes.py b/alembic/versions/003_add_composite_indexes.py new file mode 100644 index 0000000..5b33f10 --- /dev/null +++ b/alembic/versions/003_add_composite_indexes.py @@ -0,0 +1,38 @@ +# Индексы для масштабированной БД (20k+ записей) +from typing import Sequence, Union + +from alembic import op + +revision: str = "003_add_composite_indexes" +down_revision: Union[str, None] = "002_add_indexes" +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # Partial index для активных машин Encar (is_sold = FALSE) + # Ускоряет поиск при mark_sold операциях + op.execute( + "CREATE INDEX IF NOT EXISTS ix_cars_active_encar " + "ON cars (origin_url) " + "WHERE is_sold = FALSE AND origin_id LIKE 'encar:%'" + ) + + # Composite index для проверки дубликатов изображений + op.create_index( + "ix_images_car_id_fullres", + "images", + ["car_id", "fullres_image"], + ) + + # Index для быстрого поиска existing машин по origin_url + op.execute( + "CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id " + "ON cars (origin_url, origin_id)" + ) + + +def downgrade() -> None: + op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id") + op.drop_index("ix_images_car_id_fullres", table_name="images") + op.execute("DROP INDEX IF EXISTS ix_cars_active_encar") diff --git a/alembic/versions/004_widen_string_columns.py b/alembic/versions/004_widen_string_columns.py new file mode 100644 index 0000000..e83980c --- /dev/null +++ b/alembic/versions/004_widen_string_columns.py @@ -0,0 +1,25 @@ +"""widen parser_id brand model columns + +Revision ID: 004 +Revises: 003_add_composite_indexes +Create Date: 2026-04-15 +""" +from alembic import op +import sqlalchemy as sa + +revision = "004_widen_string_columns" +down_revision = "003_add_composite_indexes" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + op.alter_column("cars", "parser_id", type_=sa.String(255), existing_nullable=False) + op.alter_column("cars", "brand", type_=sa.String(100), existing_nullable=False) + op.alter_column("cars", "model", type_=sa.String(500), existing_nullable=False) + + +def downgrade() -> None: + op.alter_column("cars", "parser_id", type_=sa.String(50), existing_nullable=False) + op.alter_column("cars", "brand", type_=sa.String(50), existing_nullable=False) + op.alter_column("cars", "model", type_=sa.String(50), existing_nullable=False) diff --git a/alembic/versions/005_unique_images_constraint.py b/alembic/versions/005_unique_images_constraint.py new file mode 100644 index 0000000..b77fc6e --- /dev/null +++ b/alembic/versions/005_unique_images_constraint.py @@ -0,0 +1,33 @@ +"""add unique constraint on images(car_id, fullres_image) + +Revision ID: 005 +Revises: 004_widen_string_columns +Create Date: 2026-04-16 +""" +from alembic import op + +revision = "005_unique_images_constraint" +down_revision = "004_widen_string_columns" +branch_labels = None +depends_on = None + + +def upgrade() -> None: + # Удаляем дубли через CTE + ROW_NUMBER (быстрее чем NOT IN subquery) + op.execute(""" + WITH dupes AS ( + SELECT id, + ROW_NUMBER() OVER (PARTITION BY car_id, fullres_image ORDER BY id) AS rn + FROM images + ) + DELETE FROM images WHERE id IN (SELECT id FROM dupes WHERE rn > 1) + """) + op.create_unique_constraint( + "uq_images_car_id_fullres", + "images", + ["car_id", "fullres_image"], + ) + + +def downgrade() -> None: + op.drop_constraint("uq_images_car_id_fullres", "images", type_="unique") diff --git a/alembic/versions/006_image_car_id_bigint.py b/alembic/versions/006_image_car_id_bigint.py new file mode 100644 index 0000000..3cc3373 --- /dev/null +++ b/alembic/versions/006_image_car_id_bigint.py @@ -0,0 +1,31 @@ +"""widen images.car_id from integer to bigint + +Revision ID: 006 +Revises: 005_unique_images_constraint +Create Date: 2026-04-16 +""" +import sqlalchemy as sa +from alembic import op + +revision = "006_image_car_id_bigint" +down_revision = "005_unique_images_constraint" + + +def upgrade() -> None: + op.alter_column( + "images", + "car_id", + existing_type=sa.Integer(), + type_=sa.BigInteger(), + existing_nullable=False, + ) + + +def downgrade() -> None: + op.alter_column( + "images", + "car_id", + existing_type=sa.BigInteger(), + type_=sa.Integer(), + existing_nullable=False, + )