duuuuuuuden/iaai-fast
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fast IAAI implementation

Browse Source
This commit is contained in:
Andrey Shershnev
2026-04-24 00:20:42 +03:00
parent 2b0cffc118
commit 3cdb717789
31 changed files with 4880 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
alembic/versions/0001_ensure_sync_schema.py Normal file
View File

@@ -0,0 +1,165 @@
"""bootstrap standalone iaai schema
Revision ID: 0001_ensure_sync_schema
Revises:
Create Date: 2026-04-08 00:00:00
"""
from typing import Sequence, Union
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "0001_ensure_sync_schema"
down_revision: Union[str, Sequence[str], None] = None
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _create_enum_if_missing(type_name: str, values: tuple[str, ...]) -> None:
values_sql = ", ".join("'" + value.replace("'", "''") + "'" for value in values)
op.execute(
f"""
DO $$
BEGIN
IF NOT EXISTS (
SELECT 1
FROM pg_type t
JOIN pg_namespace n ON n.oid = t.typnamespace
WHERE t.typname = '{type_name}'
AND n.nspname = 'public'
) THEN
CREATE TYPE public.{type_name} AS ENUM ({values_sql});
END IF;
END
$$;
"""
)
def upgrade() -> None:
_create_enum_if_missing("currencyenum", ("JPY", "USD", "EUR", "RUB", "KRW", "AED", "GBP", "CAD"))
_create_enum_if_missing("driveenum", ("FWD", "RWD", "TWO_WD", "FOUR_WD", "2WD", "4WD", "NA"))
_create_enum_if_missing("gearboxenum", ("AT", "CVT", "MT", "EV", "NA"))
_create_enum_if_missing("steeringwheelenum", ("LEFT", "RIGHT", "left", "right", "NA"))
_create_enum_if_missing(
"bodytypeenum",
(
"COUPE",
"SUV",
"HATCHBACK",
"MINIVAN",
"SEDAN",
"NA",
"Station Wagon",
"Pickup",
"Truck",
"Open",
"RV",
"Other",
"STATION_WAGON",
"PICKUP",
"TRUCK",
"OPEN",
"OTHER",
),
)
_create_enum_if_missing("countryenum", ("JP", "KR", "US", "CA", "NA"))
_create_enum_if_missing(
"originenum",
(
"TAU",
"CARSENSOR",
"HANAMARU",
"ENCAR",
"KURUMA_TRADER",
"carsensor",
"encar",
"kuruma_trader",
"asnet",
"kababa",
"ACV",
"COPART",
"copart",
"NA",
"ASNET",
"KABABA",
),
)
_create_enum_if_missing("sellingtypeenum", ("STOCK", "AUCTION", "TENDER", "stock", "auction", "tender", "NA"))
op.execute(
"""
CREATE TABLE IF NOT EXISTS public.cars (
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
parser_id VARCHAR(50) NOT NULL UNIQUE,
brand VARCHAR(50) NOT NULL,
model VARCHAR(50) NOT NULL,
year INTEGER NULL,
price BIGINT NULL,
currency public.currencyenum NOT NULL DEFAULT 'USD',
mileage INTEGER NOT NULL DEFAULT 0,
country public.countryenum NOT NULL DEFAULT 'NA',
is_sold BOOLEAN NOT NULL DEFAULT FALSE,
color VARCHAR NOT NULL DEFAULT 'other',
drive public.driveenum NULL,
gearbox public.gearboxenum NULL,
steering_wheel public.steeringwheelenum NULL,
body_type public.bodytypeenum NOT NULL DEFAULT 'OTHER',
engine_volume INTEGER NULL,
selling_type public.sellingtypeenum NOT NULL DEFAULT 'NA',
one_owner BOOLEAN NOT NULL DEFAULT FALSE,
new_car BOOLEAN NOT NULL DEFAULT FALSE,
is_hidden BOOLEAN NOT NULL DEFAULT FALSE,
origin public.originenum NOT NULL DEFAULT 'NA',
origin_url VARCHAR NOT NULL,
origin_id VARCHAR NOT NULL UNIQUE,
is_damaged BOOLEAN NOT NULL DEFAULT FALSE,
evaluation VARCHAR NULL,
non_smoking BOOLEAN NOT NULL DEFAULT TRUE,
rental BOOLEAN NOT NULL DEFAULT FALSE,
repair_history BOOLEAN NOT NULL DEFAULT FALSE,
slug VARCHAR NOT NULL,
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
"""
)
op.execute(
"""
CREATE TABLE IF NOT EXISTS public.images (
id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
fullres_image VARCHAR NOT NULL,
preview_image VARCHAR NOT NULL,
order_index INTEGER NOT NULL,
car_id INTEGER NOT NULL REFERENCES public.cars(id) ON DELETE CASCADE
)
"""
)
op.execute(
"""
CREATE TABLE IF NOT EXISTS public.sync_runs (
id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
started_at TIMESTAMPTZ NOT NULL,
finished_at TIMESTAMPTZ NULL,
status TEXT NOT NULL,
lane TEXT NOT NULL,
ids_fetched INTEGER NOT NULL DEFAULT 0,
cars_upserted INTEGER NOT NULL DEFAULT 0,
cars_failed INTEGER NOT NULL DEFAULT 0,
images_upserted INTEGER NOT NULL DEFAULT 0,
error_summary TEXT NULL
)
"""
)
op.execute("CREATE INDEX IF NOT EXISTS ix_images_car_id ON public.images (car_id)")
op.execute("CREATE INDEX IF NOT EXISTS ix_sync_runs_started_at ON public.sync_runs (started_at)")
op.execute("CREATE INDEX IF NOT EXISTS ix_sync_runs_status ON public.sync_runs (status)")
def downgrade() -> None:
# Compatibility-only migration for shared production databases.
pass

86
alembic/versions/0002_extend_origin_enum_for_iaai.py Normal file
View File

@@ -0,0 +1,86 @@
"""extend origin enum for iaai
Revision ID: 0002_extend_origin_enum_for_iaai
Revises: 0001_ensure_sync_schema
Create Date: 2026-04-08 00:00:00
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision: str = "0002_extend_origin_enum_for_iaai"
down_revision: Union[str, Sequence[str], None] = "0001_ensure_sync_schema"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def _enum_type_exists(type_name: str) -> bool:
bind = op.get_bind()
if bind.dialect.name != "postgresql":
return False
exists = bind.execute(
sa.text(
"""
SELECT 1
FROM pg_type t
JOIN pg_namespace n ON n.oid = t.typnamespace
WHERE n.nspname = 'public'
AND t.typname = :type_name
AND t.typtype = 'e'
LIMIT 1
"""
),
{"type_name": type_name},
).scalar_one_or_none()
return bool(exists)
def _enum_type_for_column(table_name: str, column_name: str) -> str | None:
bind = op.get_bind()
if bind.dialect.name != "postgresql":
return None
return bind.execute(
sa.text(
"""
SELECT t.typname
FROM pg_attribute a
JOIN pg_class c ON c.oid = a.attrelid
JOIN pg_namespace n ON n.oid = c.relnamespace
JOIN pg_type t ON t.oid = a.atttypid
WHERE n.nspname = 'public'
AND c.relname = :table_name
AND a.attname = :column_name
AND a.attnum > 0
AND NOT a.attisdropped
AND t.typtype = 'e'
LIMIT 1
"""
),
{"table_name": table_name, "column_name": column_name},
).scalar_one_or_none()
def _add_enum_value_if_missing(type_name: str | None, value: str) -> None:
if not type_name:
return
if not _enum_type_exists(type_name):
return
bind = op.get_bind()
if bind.dialect.name != "postgresql":
return
escaped = value.replace("'", "''")
op.execute(f"ALTER TYPE public.{type_name} ADD VALUE IF NOT EXISTS '{escaped}'")
def upgrade() -> None:
origin_type = _enum_type_for_column("cars", "origin") or "originenum"
_add_enum_value_if_missing(origin_type, "IAAI")
def downgrade() -> None:
# Enum value removal is intentionally skipped for compatibility.
pass

100
alembic/versions/0003_create_iaai_source_tables.py Normal file
View File

@@ -0,0 +1,100 @@
"""create iaai-specific storage tables
Revision ID: 0003_create_iaai_source_tables
Revises: 0002_extend_origin_enum_for_iaai
Create Date: 2026-04-09 00:00:00
"""
from typing import Sequence, Union
from alembic import op
# revision identifiers, used by Alembic.
revision: str = "0003_create_iaai_source_tables"
down_revision: Union[str, Sequence[str], None] = "0002_extend_origin_enum_for_iaai"
branch_labels: Union[str, Sequence[str], None] = None
depends_on: Union[str, Sequence[str], None] = None
def upgrade() -> None:
op.execute(
"""
CREATE TABLE IF NOT EXISTS public.iaai_cars (
id INTEGER GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
parser_id VARCHAR(50) NOT NULL UNIQUE,
brand VARCHAR(50) NOT NULL,
model VARCHAR(50) NOT NULL,
year INTEGER NULL,
price BIGINT NULL,
currency public.currencyenum NOT NULL DEFAULT 'USD',
mileage INTEGER NOT NULL DEFAULT 0,
country public.countryenum NOT NULL DEFAULT 'NA',
is_sold BOOLEAN NOT NULL DEFAULT FALSE,
color VARCHAR NOT NULL DEFAULT 'other',
drive public.driveenum NULL,
gearbox public.gearboxenum NULL,
steering_wheel public.steeringwheelenum NULL,
body_type public.bodytypeenum NOT NULL DEFAULT 'OTHER',
engine_volume INTEGER NULL,
selling_type public.sellingtypeenum NOT NULL DEFAULT 'NA',
one_owner BOOLEAN NOT NULL DEFAULT FALSE,
new_car BOOLEAN NOT NULL DEFAULT FALSE,
is_hidden BOOLEAN NOT NULL DEFAULT FALSE,
origin public.originenum NOT NULL DEFAULT 'NA',
origin_url VARCHAR NOT NULL,
origin_id VARCHAR NOT NULL UNIQUE,
is_damaged BOOLEAN NOT NULL DEFAULT FALSE,
evaluation VARCHAR NULL,
non_smoking BOOLEAN NOT NULL DEFAULT TRUE,
rental BOOLEAN NOT NULL DEFAULT FALSE,
repair_history BOOLEAN NOT NULL DEFAULT FALSE,
slug VARCHAR NOT NULL,
last_seen_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
)
"""
)
op.execute(
"""
CREATE TABLE IF NOT EXISTS public.iaai_images (
id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
fullres_image VARCHAR NOT NULL,
preview_image VARCHAR NOT NULL,
order_index INTEGER NOT NULL,
car_id INTEGER NOT NULL REFERENCES public.iaai_cars(id) ON DELETE CASCADE
)
"""
)
op.execute(
"""
CREATE TABLE IF NOT EXISTS public.iaai_sync_runs (
id BIGINT GENERATED BY DEFAULT AS IDENTITY PRIMARY KEY,
started_at TIMESTAMPTZ NOT NULL,
finished_at TIMESTAMPTZ NULL,
status TEXT NOT NULL,
lane TEXT NOT NULL,
ids_fetched INTEGER NOT NULL DEFAULT 0,
cars_upserted INTEGER NOT NULL DEFAULT 0,
cars_failed INTEGER NOT NULL DEFAULT 0,
images_upserted INTEGER NOT NULL DEFAULT 0,
error_summary TEXT NULL
)
"""
)
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_id ON public.iaai_images (car_id)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_images_car_order ON public.iaai_images (car_id, order_index)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_cars_last_seen_id ON public.iaai_cars (last_seen_at DESC, id DESC)")
op.execute(
"CREATE INDEX IF NOT EXISTS ix_iaai_cars_active_feed ON public.iaai_cars "
"(last_seen_at DESC, id DESC) WHERE is_sold = FALSE AND is_hidden = FALSE"
)
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_started_at ON public.iaai_sync_runs (started_at)")
op.execute("CREATE INDEX IF NOT EXISTS ix_iaai_sync_runs_status ON public.iaai_sync_runs (status)")
def downgrade() -> None:
# Compatibility-only migration for shared production databases.
pass