qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add mobilede scraper

Browse Source
This commit is contained in:
qananasikq
2026-04-29 19:13:06 +03:00
parent b0bcd16ea2
commit f00819a21c
58 changed files with 15743 additions and 71 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
alembic/env.py
View File

@@ -10,11 +10,11 @@ from sqlalchemy import engine_from_config, pool
# Добавляем корень проекта в sys.path
sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(__file__), "..")))
from iaai_scraper.core.config import Settings
from iaai_scraper.storage.models import Base
from mobilede_scraper.core.config import Settings
from mobilede_scraper.storage.models import Base
config = context.config
VERSION_TABLE = "iaai_parser_alembic_version"
VERSION_TABLE = "MOBILEDE_parser_alembic_version"
# Logging
if config.config_file_name is not None:

30
alembic/versions/001_initial.py
View File

@@ -1,4 +1,4 @@
# Начальная схема: iaai_cars, iaai_images, iaai_sync_runs
# Начальная схема: MOBILEDE_cars, MOBILEDE_images, MOBILEDE_sync_runs
from typing import Sequence, Union
from alembic import op
@@ -29,10 +29,10 @@ def _index_exists(table_name: str, index_name: str) -> bool:
def upgrade() -> None:
# Таблица iaai_cars — аукционные машины с IAAI
if not _table_exists("iaai_cars"):
# Таблица MOBILEDE_cars — аукционные машины с MOBILEDE
if not _table_exists("MOBILEDE_cars"):
op.create_table(
"iaai_cars",
"MOBILEDE_cars",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("parser_id", sa.String(50), nullable=False, unique=True),
sa.Column("brand", sa.String(50), nullable=False),
@@ -65,26 +65,26 @@ def upgrade() -> None:
sa.Column("last_seen_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
)
if not _index_exists("iaai_cars", "ix_iaai_cars_origin_url"):
op.create_index("ix_iaai_cars_origin_url", "iaai_cars", ["origin_url"])
if not _index_exists("iaai_cars", "ix_iaai_cars_origin_id"):
op.create_index("ix_iaai_cars_origin_id", "iaai_cars", ["origin_id"], unique=True)
if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_url"):
op.create_index("ix_MOBILEDE_cars_origin_url", "MOBILEDE_cars", ["origin_url"])
if not _index_exists("MOBILEDE_cars", "ix_MOBILEDE_cars_origin_id"):
op.create_index("ix_MOBILEDE_cars_origin_id", "MOBILEDE_cars", ["origin_id"], unique=True)
# Таблица iaai_images — изображения машин
if not _table_exists("iaai_images"):
# Таблица MOBILEDE_images — изображения машин
if not _table_exists("MOBILEDE_images"):
op.create_table(
"iaai_images",
"MOBILEDE_images",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("fullres_image", sa.String, nullable=False),
sa.Column("preview_image", sa.String, nullable=False),
sa.Column("order_index", sa.Integer, nullable=False),
sa.Column("car_id", sa.Integer, sa.ForeignKey("iaai_cars.id", ondelete="CASCADE"), nullable=False),
sa.Column("car_id", sa.Integer, sa.ForeignKey("MOBILEDE_cars.id", ondelete="CASCADE"), nullable=False),
)
# Таблица iaai_sync_runs — логирование синхронизаций
if not _table_exists("iaai_sync_runs"):
# Таблица MOBILEDE_sync_runs — логирование синхронизаций
if not _table_exists("MOBILEDE_sync_runs"):
op.create_table(
"iaai_sync_runs",
"MOBILEDE_sync_runs",
sa.Column("id", sa.BigInteger, primary_key=True, autoincrement=True),
sa.Column("started_at", sa.DateTime(timezone=True), nullable=False, server_default=sa.func.now()),
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),

14
alembic/versions/002_add_indexes.py
View File

@@ -10,13 +10,13 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand ON iaai_cars (brand)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_brand_model ON iaai_cars (brand, model)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_year ON iaai_cars (year)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_is_sold ON iaai_cars (is_sold)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_at ON iaai_cars (last_seen_at)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON iaai_images (car_id)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON iaai_sync_runs (status)")
op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand" ON "MOBILEDE_cars" (brand)')
op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_brand_model" ON "MOBILEDE_cars" (brand, model)')
op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_year" ON "MOBILEDE_cars" (year)')
op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_is_sold" ON "MOBILEDE_cars" (is_sold)')
op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_last_seen_at" ON "MOBILEDE_cars" (last_seen_at)')
op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id" ON "MOBILEDE_images" (car_id)')
op.execute('CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_sync_runs_status" ON "MOBILEDE_sync_runs" (status)')
def downgrade() -> None:

16
alembic/versions/003_add_composite_indexes.py
View File

@@ -10,24 +10,24 @@ depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
# Partial index для активных машин IAAI (is_sold = FALSE)
# Partial index для активных машин MOBILEDE (is_sold = FALSE)
# Ускоряет поиск при mark_sold операциях
op.execute(
"CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_iaai "
"ON iaai_cars (origin_url) "
"WHERE is_sold = FALSE AND origin_id LIKE 'iaai:%'"
'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_MOBILEDE" '
'ON "MOBILEDE_cars" (origin_url) '
"WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%'"
)
# Composite index для проверки дубликатов изображений
op.execute(
"CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id_fullres "
"ON iaai_images (car_id, fullres_image)"
'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id_fullres" '
'ON "MOBILEDE_images" (car_id, fullres_image)'
)
# Index для быстрого поиска existing машин по origin_url
op.execute(
"CREATE INDEX IF NOT EXISTS ix_iaai_cars_origin_url_id "
"ON iaai_cars (origin_url, origin_id)"
'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_origin_url_id" '
'ON "MOBILEDE_cars" (origin_url, origin_id)'
)