update alembic migrations
This commit is contained in:
@@ -3,7 +3,7 @@
|
|||||||
[alembic]
|
[alembic]
|
||||||
script_location = alembic
|
script_location = alembic
|
||||||
prepend_sys_path = .
|
prepend_sys_path = .
|
||||||
sqlalchemy.url = postgresql+psycopg2://iaai:iaai@localhost:5432/iaai_scraper
|
sqlalchemy.url = postgresql+psycopg2://encar:encar@localhost:5434/encar_db
|
||||||
|
|
||||||
[loggers]
|
[loggers]
|
||||||
keys = root,sqlalchemy,alembic
|
keys = root,sqlalchemy,alembic
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
# Alembic env.py — подключение к БД через Settings.
|
# Alembic env.py — подключение к БД через Settings.
|
||||||
|
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
@@ -10,8 +10,8 @@ from sqlalchemy import engine_from_config, pool
|
|||||||
# Добавляем корень проекта в sys.path
|
# Добавляем корень проекта в sys.path
|
||||||
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
|
||||||
|
|
||||||
from iaai_scraper.core.config import Settings
|
from encar_scraper.core.config import Settings
|
||||||
from iaai_scraper.storage.models import Base
|
from encar_scraper.storage.models import Base
|
||||||
|
|
||||||
config = context.config
|
config = context.config
|
||||||
|
|
||||||
|
|||||||
@@ -11,7 +11,7 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
# Таблица cars — аукционные машины с IAAI
|
# Таблица cars — автомобили с Encar
|
||||||
op.create_table(
|
op.create_table(
|
||||||
"cars",
|
"cars",
|
||||||
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
|
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
|
||||||
|
|||||||
@@ -10,12 +10,12 @@ depends_on: Union[str, Sequence[str], None] = None
|
|||||||
|
|
||||||
|
|
||||||
def upgrade() -> None:
|
def upgrade() -> None:
|
||||||
# Partial index для активных машин IAAI (is_sold = FALSE)
|
# Partial index для активных машин Encar (is_sold = FALSE)
|
||||||
# Ускоряет поиск при mark_sold операциях
|
# Ускоряет поиск при mark_sold операциях
|
||||||
op.execute(
|
op.execute(
|
||||||
"CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
|
"CREATE INDEX IF NOT EXISTS ix_cars_active_encar "
|
||||||
"ON cars (origin_url) "
|
"ON cars (origin_url) "
|
||||||
"WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
|
"WHERE is_sold = FALSE AND origin_id LIKE 'encar:%'"
|
||||||
)
|
)
|
||||||
|
|
||||||
# Composite index для проверки дубликатов изображений
|
# Composite index для проверки дубликатов изображений
|
||||||
@@ -35,4 +35,4 @@ def upgrade() -> None:
|
|||||||
def downgrade() -> None:
|
def downgrade() -> None:
|
||||||
op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
|
op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
|
||||||
op.drop_index("ix_images_car_id_fullres", table_name="images")
|
op.drop_index("ix_images_car_id_fullres", table_name="images")
|
||||||
op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")
|
op.execute("DROP INDEX IF EXISTS ix_cars_active_encar")
|
||||||
|
|||||||
25
alembic/versions/004_widen_string_columns.py
Normal file
25
alembic/versions/004_widen_string_columns.py
Normal file
@@ -0,0 +1,25 @@
|
|||||||
|
"""widen parser_id brand model columns
|
||||||
|
|
||||||
|
Revision ID: 004
|
||||||
|
Revises: 003_add_composite_indexes
|
||||||
|
Create Date: 2026-04-15
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
import sqlalchemy as sa
|
||||||
|
|
||||||
|
revision = "004_widen_string_columns"
|
||||||
|
down_revision = "003_add_composite_indexes"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.alter_column("cars", "parser_id", type_=sa.String(255), existing_nullable=False)
|
||||||
|
op.alter_column("cars", "brand", type_=sa.String(100), existing_nullable=False)
|
||||||
|
op.alter_column("cars", "model", type_=sa.String(500), existing_nullable=False)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.alter_column("cars", "parser_id", type_=sa.String(50), existing_nullable=False)
|
||||||
|
op.alter_column("cars", "brand", type_=sa.String(50), existing_nullable=False)
|
||||||
|
op.alter_column("cars", "model", type_=sa.String(50), existing_nullable=False)
|
||||||
33
alembic/versions/005_unique_images_constraint.py
Normal file
33
alembic/versions/005_unique_images_constraint.py
Normal file
@@ -0,0 +1,33 @@
|
|||||||
|
"""add unique constraint on images(car_id, fullres_image)
|
||||||
|
|
||||||
|
Revision ID: 005
|
||||||
|
Revises: 004_widen_string_columns
|
||||||
|
Create Date: 2026-04-16
|
||||||
|
"""
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "005_unique_images_constraint"
|
||||||
|
down_revision = "004_widen_string_columns"
|
||||||
|
branch_labels = None
|
||||||
|
depends_on = None
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
# Удаляем дубли через CTE + ROW_NUMBER (быстрее чем NOT IN subquery)
|
||||||
|
op.execute("""
|
||||||
|
WITH dupes AS (
|
||||||
|
SELECT id,
|
||||||
|
ROW_NUMBER() OVER (PARTITION BY car_id, fullres_image ORDER BY id) AS rn
|
||||||
|
FROM images
|
||||||
|
)
|
||||||
|
DELETE FROM images WHERE id IN (SELECT id FROM dupes WHERE rn > 1)
|
||||||
|
""")
|
||||||
|
op.create_unique_constraint(
|
||||||
|
"uq_images_car_id_fullres",
|
||||||
|
"images",
|
||||||
|
["car_id", "fullres_image"],
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.drop_constraint("uq_images_car_id_fullres", "images", type_="unique")
|
||||||
31
alembic/versions/006_image_car_id_bigint.py
Normal file
31
alembic/versions/006_image_car_id_bigint.py
Normal file
@@ -0,0 +1,31 @@
|
|||||||
|
"""widen images.car_id from integer to bigint
|
||||||
|
|
||||||
|
Revision ID: 006
|
||||||
|
Revises: 005_unique_images_constraint
|
||||||
|
Create Date: 2026-04-16
|
||||||
|
"""
|
||||||
|
import sqlalchemy as sa
|
||||||
|
from alembic import op
|
||||||
|
|
||||||
|
revision = "006_image_car_id_bigint"
|
||||||
|
down_revision = "005_unique_images_constraint"
|
||||||
|
|
||||||
|
|
||||||
|
def upgrade() -> None:
|
||||||
|
op.alter_column(
|
||||||
|
"images",
|
||||||
|
"car_id",
|
||||||
|
existing_type=sa.Integer(),
|
||||||
|
type_=sa.BigInteger(),
|
||||||
|
existing_nullable=False,
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def downgrade() -> None:
|
||||||
|
op.alter_column(
|
||||||
|
"images",
|
||||||
|
"car_id",
|
||||||
|
existing_type=sa.BigInteger(),
|
||||||
|
type_=sa.Integer(),
|
||||||
|
existing_nullable=False,
|
||||||
|
)
|
||||||
Reference in New Issue
Block a user