fix docker scraping

improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup
This commit is contained in:
qananasikq
2026-04-13 16:35:08 +03:00
parent b9557922ed
commit a4c93a1df1
30 changed files with 2239 additions and 624 deletions

View File

@@ -0,0 +1,38 @@
# Индексы для масштабированной БД (20k+ записей)
from typing import Sequence, Union
from alembic import op
revision: str = "003_add_composite_indexes"
down_revision: Union[str, None] = "002_add_indexes"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Partial index для активных машин IAAI (is_sold = FALSE)
# Ускоряет поиск при mark_sold операциях
op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_active_iaai "
"ON cars (origin_url) "
"WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
)
# Composite index для проверки дубликатов изображений
op.create_index(
"ix_images_car_id_fullres",
"images",
["car_id", "fullres_image"],
)
# Index для быстрого поиска existing машин по origin_url
op.execute(
"CREATE INDEX IF NOT EXISTS ix_cars_origin_url_id "
"ON cars (origin_url, origin_id)"
)
def downgrade() -> None:
op.execute("DROP INDEX IF EXISTS ix_cars_origin_url_id")
op.drop_index("ix_images_car_id_fullres", table_name="images")
op.execute("DROP INDEX IF EXISTS ix_cars_active_iaai")