qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepared for production

Browse Source
This commit is contained in:
Andrey Shershnev
2026-04-23 13:54:53 +03:00
parent 09fecdae31
commit f221aebef0
8 changed files with 159 additions and 118 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
alembic/env.py
View File

@@ -14,6 +14,7 @@ from iaai_scraper.core.config import Settings
from iaai_scraper.storage.models import Base
config = context.config
VERSION_TABLE = "iaai_parser_alembic_version"
# Logging
if config.config_file_name is not None:
@@ -32,6 +33,7 @@ def run_migrations_offline() -> None:
context.configure(
url=url,
target_metadata=target_metadata,
version_table=VERSION_TABLE,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
)
@@ -47,7 +49,11 @@ def run_migrations_online() -> None:
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(connection=connection, target_metadata=target_metadata)
context.configure(
connection=connection,
target_metadata=target_metadata,
version_table=VERSION_TABLE,
)
with context.begin_transaction():
context.run_migrations()

150
alembic/versions/001_initial.py
View File

@@ -1,4 +1,4 @@
# Начальная схема: cars, images, sync_runs
# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs
from typing import Sequence, Union
from alembic import op
@@ -10,70 +10,94 @@ branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _schema_name() -> str | None:
bind = op.get_bind()
return "public" if bind.dialect.name == "postgresql" else None
def _table_exists(table_name: str) -> bool:
inspector = sa.inspect(op.get_bind())
return table_name in inspector.get_table_names(schema=_schema_name())
def _index_exists(table_name: str, index_name: str) -> bool:
if not _table_exists(table_name):
return False
inspector = sa.inspect(op.get_bind())
indexes = inspector.get_indexes(table_name, schema=_schema_name())
return any(index.get("name") == index_name for index in indexes)
def upgrade() -> None:
# Таблица cars — аукционные машины с IAAI
op.create_table(
"cars",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
sa.Column("brand", sa.String(50), nullable=False),
sa.Column("model", sa.String(50), nullable=False),
sa.Column("year", sa.Integer, nullable=True),
sa.Column("price", sa.BigInteger, nullable=True),
sa.Column("currency", sa.String(10), nullable=False, server_default="USD"),
sa.Column("mileage", sa.Integer, nullable=False, server_default="0"),
sa.Column("country", sa.String(10), nullable=False, server_default="NA"),
sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("color", sa.String, nullable=False, server_default="other"),
sa.Column("drive", sa.String(10), nullable=True),
sa.Column("gearbox", sa.String(10), nullable=True),
sa.Column("steering_wheel", sa.String(10), nullable=True),
sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"),
sa.Column("engine_volume", sa.Integer, nullable=True),
sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"),
sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("origin", sa.String(20), nullable=False, server_default="NA"),
sa.Column("origin_url", sa.String, nullable=False),
sa.Column("origin_id", sa.String, nullable=False, unique=True),
sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("evaluation", sa.String, nullable=True),
sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")),
sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("slug", sa.String, nullable=False),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
op.create_index("ix_cars_origin_url", "cars", ["origin_url"])
op.create_index("ix_cars_origin_id", "cars", ["origin_id"], unique=True)
# Таблица iaai_cars — аукционные машины с IAAI
if not _table_exists("iaai_cars"):
op.create_table(
"iaai_cars",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
sa.Column("brand", sa.String(50), nullable=False),
sa.Column("model", sa.String(50), nullable=False),
sa.Column("year", sa.Integer, nullable=True),
sa.Column("price", sa.BigInteger, nullable=True),
sa.Column("currency", sa.String(10), nullable=False, server_default="USD"),
sa.Column("mileage", sa.Integer, nullable=False, server_default="0"),
sa.Column("country", sa.String(10), nullable=False, server_default="NA"),
sa.Column("is_sold", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("color", sa.String, nullable=False, server_default="other"),
sa.Column("drive", sa.String(10), nullable=True),
sa.Column("gearbox", sa.String(10), nullable=True),
sa.Column("steering_wheel", sa.String(10), nullable=True),
sa.Column("body_type", sa.String(20), nullable=False, server_default="OTHER"),
sa.Column("engine_volume", sa.Integer, nullable=True),
sa.Column("selling_type", sa.String(20), nullable=False, server_default="NA"),
sa.Column("one_owner", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("new_car", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("is_hidden", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("origin", sa.String(20), nullable=False, server_default="NA"),
sa.Column("origin_url", sa.String, nullable=False),
sa.Column("origin_id", sa.String, nullable=False, unique=True),
sa.Column("is_damaged", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("evaluation", sa.String, nullable=True),
sa.Column("non_smoking", sa.Boolean, nullable=False, server_default=sa.text("true")),
sa.Column("rental", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("repair_history", sa.Boolean, nullable=False, server_default=sa.text("false")),
sa.Column("slug", sa.String, nullable=False),
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
# Таблица images — изображения машин
op.create_table(
"images",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("fullres_image", sa.String, nullable=False),
sa.Column("preview_image", sa.String, nullable=False),
sa.Column("order_index", sa.Integer, nullable=False),
sa.Column("car_id", sa.Integer, sa.ForeignKey("cars.id", ondelete="CASCADE"), nullable=False),
)
if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"):
op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"])
if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"):
op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True)
# Таблица iaai_images — изображения машин
if not _table_exists("iaai_images"):
op.create_table(
"iaai_images",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("fullres_image", sa.String, nullable=False),
sa.Column("preview_image", sa.String, nullable=False),
sa.Column("order_index", sa.Integer, nullable=False),
sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False),
)
# Таблица iaai_sync_runs — логирование синхронизаций
if not _table_exists("iaai_sync_runs"):
op.create_table(
"iaai_sync_runs",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("status", sa.Text, nullable=False),
sa.Column("lane", sa.Text, nullable=False),
sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"),
sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"),
sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"),
sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"),
sa.Column("error_summary", sa.Text, nullable=True),
)
# Таблица sync_runs — логирование синхронизаций
op.create_table(
"sync_runs",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
sa.Column("status", sa.Text, nullable=False),
sa.Column("lane", sa.Text, nullable=False),
sa.Column("ids_fetched", sa.Integer, nullable=False, server_default="0"),
sa.Column("cars_upserted", sa.Integer, nullable=False, server_default="0"),
sa.Column("cars_failed", sa.Integer, nullable=False, server_default="0"),
sa.Column("images_upserted", sa.Integer, nullable=False, server_default="0"),
sa.Column("error_summary", sa.Text, nullable=True),
)
def downgrade() -> None:
op.drop_table("sync_runs")
op.drop_table("images")
op.drop_table("cars")
# Compatibility-only migration for shared production databases.
pass

23
alembic/versions/002_add_indexes.py
View File

@@ -10,20 +10,15 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.create_index("ix_cars_brand", "cars", ["brand"])
op.create_index("ix_cars_brand_model", "cars", ["brand", "model"])
op.create_index("ix_cars_year", "cars", ["year"])
op.create_index("ix_cars_is_sold", "cars", ["is_sold"])
op.create_index("ix_cars_last_seen_at", "cars", ["last_seen_at"])
op.create_index("ix_images_car_id", "images", ["car_id"])
op.create_index("ix_sync_runs_status", "sync_runs", ["status"])
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)")
def downgrade() -> None:
op.drop_index("ix_sync_runs_status", table_name="sync_runs")
op.drop_index("ix_images_car_id", table_name="images")
op.drop_index("ix_cars_last_seen_at", table_name="cars")
op.drop_index("ix_cars_is_sold", table_name="cars")
op.drop_index("ix_cars_year", table_name="cars")
op.drop_index("ix_cars_brand_model", table_name="cars")
op.drop_index("ix_cars_brand", table_name="cars")
# Compatibility-only migration for shared production databases.
pass

20
alembic/versions/003_add_composite_indexes.py
View File

@@ -13,26 +13,24 @@ def upgrade() -> None:
# Partial index для активных машин IAAI (is_sold = FALSE)
# Ускоряет поиск при mark_sold операциях
op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
"ON cars (origin_url) "
"CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai "
"ON iaai_cars (origin_url) "
"WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
)
# Composite index для проверки дубликатов изображений
op.create_index(
"ix_images_car_id_fullres",
"images",
["car_id", "fullres_image"],
op.execute(
"CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres "
"ON iaai_images (car_id, fullres_image)"
)
# Index для быстрого поиска existing машин по origin_url
op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
"ON cars (origin_url, origin_id)"
"CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id "
"ON iaai_cars (origin_url, origin_id)"
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
op.drop_index("ix_images_car_id_fullres", table_name="images")
op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")
# Compatibility-only migration for shared production databases.
pass