update infra and schema

2026-05-11 12:45:37 +03:00
parent 5049a29ef7
commit 124dfc514f
13 changed files with 124 additions and 81 deletions
--- a/.env.example
+++ b/.env.example
@@ -25,7 +25,6 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000
 CELERY_WORKER_CONCURRENCY=3
 CELERY_WORKER_POOL=prefork
 CELERY_WORKER_MAX_TASKS_PER_CHILD=5
-CELERY_BATCH_SIZE=1000

 # Runtime mode
 MOBILEDE_STARTUP_SYNC_ENABLED=false
@@ -39,8 +38,6 @@ MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1
 # Search settings
 MOBILEDE_FILTERED_SEARCH_URL=
 MOBILEDE_FILTERED_SEARCH_URLS=
-MOBILEDE_STARTUP_MAX_PAGES=100
-MOBILEDE_BEAT_MAX_PAGES=100
 MOBILEDE_REQUEST_DELAY_SECONDS=0.30
 MOBILEDE_CONCURRENT_PAGES=1
 MOBILEDE_CURSOR_ENABLED=true
@@ -93,12 +90,4 @@ SOCKS5_PROXY_PORT=1002
 SOCKS5_PROXY_USER=
 SOCKS5_PROXY_PASS=

-# IAAI
-IAAI_STARTUP_SYNC_ENABLED=true
-IAAI_LOG_LEVEL=INFO
-IAAI_BROWSER_ENGINE=chromium
-IAAI_TOKENS_FILE=/data/tokens.json
-IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json
-IAAI_SELF_HEAL_ENABLED=true
-
 TZ=UTC
--- a/.gitignore
+++ b/.gitignore
@@ -6,6 +6,8 @@ coverage.xml
 !.env.example
 .venv/
 venv/
+*.tgz
+*.tar.gz
 *.pyc
 *.pyo
 *.pyd
--- a/2
+++ b/2
@@ -27,7 +27,7 @@ STOPSIGNAL SIGINT

 USER app

-# РџРѕ СѓРјРѕР»С‡Р°РЅРёСЋ Р·Р°РїСѓСЃРєР°РµРј API.
+# По умолчанию запускаем API.
 ENTRYPOINT ["./entrypoint.sh"]
 CMD ["uvicorn", "mobilede_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"]

--- a/alembic/env.py
+++ b/alembic/env.py
@@ -16,7 +16,7 @@ from mobilede_scraper.storage.models import Base
 config = context.config
 VERSION_TABLE = "MOBILEDE_parser_alembic_version"

-# Logging
+# Логирование
 if config.config_file_name is not None:
    fileConfig(config.config_file_name)

@@ -28,7 +28,7 @@ target_metadata = Base.metadata


 def run_migrations_offline() -> None:
-    # Run migrations in 'offline' mode.
+    # Запуск миграций в офлайн-режиме.
    url = config.get_main_option("sqlalchemy.url")
    context.configure(
        url=url,
@@ -42,7 +42,7 @@ def run_migrations_offline() -> None:


 def run_migrations_online() -> None:
-    # Run migrations in 'online' mode.
+    # Запуск миграций в онлайн-режиме.
    connectable = engine_from_config(
        config.get_section(config.config_ini_section, {}),
        prefix="sqlalchemy.",
--- a/alembic/versions/001_initial.py
+++ b/alembic/versions/001_initial.py
@@ -99,5 +99,5 @@ def upgrade() -> None:


 def downgrade() -> None:
-    # Compatibility-only migration for shared production databases.
+    # Миграция оставлена только для совместимости с общей production-базой.
    pass
--- a/alembic/versions/002_add_indexes.py
+++ b/alembic/versions/002_add_indexes.py
@@ -20,5 +20,5 @@ def upgrade() -> None:


 def downgrade() -> None:
-    # Compatibility-only migration for shared production databases.
+    # Миграция оставлена только для совместимости с общей production-базой.
    pass
--- a/alembic/versions/003_add_composite_indexes.py
+++ b/alembic/versions/003_add_composite_indexes.py
@@ -10,21 +10,21 @@ depends_on: Union[str, Sequence[str], None] = None


 def upgrade() -> None:
-    # Partial index для активных машин MOBILEDE (is_sold = FALSE)
-    # Ускоряет поиск при mark_sold операциях
+    # Частичный индекс для активных машин MOBILEDE (is_sold = FALSE).
+    # Ускоряет поиск при операциях mark_sold.
    op.execute(
        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_MOBILEDE" '
        'ON "MOBILEDE_cars" (origin_url) '
        "WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%'"
    )

-    # Composite index для проверки дубликатов изображений
+    # Составной индекс для проверки дубликатов изображений.
    op.execute(
        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id_fullres" '
        'ON "MOBILEDE_images" (car_id, fullres_image)'
    )

-    # Index для быстрого поиска existing машин по origin_url
+    # Индекс для быстрого поиска существующих машин по origin_url.
    op.execute(
        'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_origin_url_id" '
        'ON "MOBILEDE_cars" (origin_url, origin_id)'
@@ -32,5 +32,5 @@ def upgrade() -> None:


 def downgrade() -> None:
-    # Compatibility-only migration for shared production databases.
+    # Миграция оставлена только для совместимости с общей production-базой.
    pass
--- a/alembic/versions/004_add_ingestion_tables.py
+++ b/alembic/versions/004_add_ingestion_tables.py
@@ -1,4 +1,4 @@
-# Placeholder migration (ingestion tables not yet needed)
+# Заглушка миграции (таблицы ingestion пока не требуются)
 from typing import Sequence, Union

 from alembic import op
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -4,31 +4,17 @@
  CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0}
  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0}
  MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS: ${MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS:-1800}
-  MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-20}
-  MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-40}
+  MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-40}
+  MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-80}
  CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900}
  CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200}
  CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400}
-  IAAI_PROFILE: ${IAAI_PROFILE:-fast}
-  IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}}
-  IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true}
-  IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false}
-  IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false}
-  IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1}
-  IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-2}
-  IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0}
-  IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1}
-  IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1}
-  # 0 = без лимита.
-  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
-  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4}
+  CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-6}
  CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork}
  CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
-  CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000}
-  MOBILEDE_STARTUP_MAX_PAGES: ${MOBILEDE_STARTUP_MAX_PAGES:-100}
-  MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100}
-  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0}
-  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2}
+  MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.45}
+  MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-1}
+  MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800"
  MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-}
  MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-}
  MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true}
@@ -37,45 +23,48 @@
  MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10}
  MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true}
  MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true}
-  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2}
+  MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-5}
+  MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true}
  MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10}
  MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20}
  MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50}
-  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1000}
+  MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-700}
  MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true}
-  MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-false}
-  MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-1000}
-  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40}
+  # Первый проход должен идти по заранее нарезанным leaf-сегментам.
+  # Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально.
+  MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true}
+  MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400}
+  MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-320}
+  MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0}
+  MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-3}
+  MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN: ${MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN:-true}
+  MOBILEDE_FILTERED_URL_FAST_START: ${MOBILEDE_FILTERED_URL_FAST_START:-false}
+  MOBILEDE_REFINE_ADAPTIVE_SEGMENTS: ${MOBILEDE_REFINE_ADAPTIVE_SEGMENTS:-true}
+  MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS: ${MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS:-240}
+  MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES: ${MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES:-3}
+  MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8}
  MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false}
-  MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-false}
+  MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true}
+  MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.70}
+  MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.90}
+  MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.35}
  MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true}
  MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
-  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true}
-  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1}
-  IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
-  IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
-  IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
-  IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8}
-  IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-32}
-  IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true}
-  IAAI_FILTERED_SEARCH_URL: ${IAAI_FILTERED_SEARCH_URL:-}
-  IAAI_FILTERED_SEARCH_URLS: ${IAAI_FILTERED_SEARCH_URLS:-}
-  IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-runtime}
-  IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999}
-  IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000}
-  IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true}
-  IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
-  IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json}
-  IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  IAAI_BROWSER_ENGINE: chromium
+  MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false}
+  MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-10}
+  MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED: ${MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED:-false}
+  MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false}
+  MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10}
+  MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true}
+  MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
  # Автовосстановление worker.
-  IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true}
-  IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
-  IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900}
-  IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
-  IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
+  MOBILEDE_SELF_HEAL_ENABLED: ${MOBILEDE_SELF_HEAL_ENABLED:-true}
+  MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30}
+  MOBILEDE_SELF_HEAL_STALL_SECONDS: ${MOBILEDE_SELF_HEAL_STALL_SECONDS:-900}
+  MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300}
+  MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300}
  # При простое БД перезапускаем worker.
-  IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600}
+  MOBILEDE_DB_IDLE_RESTART_SECONDS: ${MOBILEDE_DB_IDLE_RESTART_SECONDS:-3600}
  TZ: ${TZ:-UTC}

 x-env-file: &env-file
@@ -182,7 +171,6 @@ services:
  worker:
    <<: *worker-service
    restart: unless-stopped
-    init: true
    depends_on:
      migrate:
        condition: service_completed_successfully
@@ -191,9 +179,9 @@ services:
    stop_grace_period: 60s
    command: >
      celery -A mobilede_scraper.worker.celery_app worker
-      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork}
+      --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-8} --pool=${CELERY_WORKER_POOL:-prefork}
      --pidfile=/tmp/celery-worker.pid
-      -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
+      -Q mobilede_sync --max-tasks-per-child=5
    healthcheck:
      # Проверка worker.
      test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"]
--- a/mobilede_scraper/storage/db.py
+++ b/mobilede_scraper/storage/db.py
@@ -4,7 +4,7 @@ from contextlib import contextmanager
 from datetime import datetime, timezone
 from typing import Any, Iterator

-from sqlalchemy import create_engine, delete, or_, select, text, update
+from sqlalchemy import create_engine, delete, func, or_, select, text, update
 from sqlalchemy.dialects.postgresql import insert as pg_insert
 from sqlalchemy.orm import Session, sessionmaker

@@ -808,6 +808,34 @@ class PersistenceService:
 				if row and row[0] and row[1] and row[2]
 			]

+	def get_active_cars_batch_for_image_enrich(
+		self,
+		prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES,
+		*,
+		limit: int = 50,
+		max_existing_images: int = 1,
+	) -> list[tuple[int, str, str, int]]:
+		prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix)
+		with self.session_scope() as session:
+			image_count = func.count(Image.id)
+			result = session.execute(
+				select(Car.id, Car.origin_id, Car.origin_url, image_count.label("image_count"))
+				.outerjoin(Image, Image.car_id == Car.id)
+				.where(
+					_origin_prefix_filter(Car.origin_id, prefixes),
+					Car.is_sold == False,  # noqa: E712
+				)
+				.group_by(Car.id, Car.origin_id, Car.origin_url)
+				.having(image_count <= max(0, int(max_existing_images)))
+				.order_by(Car.last_seen_at.desc())
+				.limit(max(1, int(limit)))
+			)
+			return [
+				(int(row[0]), str(row[1]), str(row[2]), int(row[3] or 0))
+				for row in result
+				if row and row[0] and row[1] and row[2]
+			]
+
 	def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int:
 		if not car_ids:
 			return 0
--- a/mobilede_scraper/storage/enums.py
+++ b/mobilede_scraper/storage/enums.py
@@ -16,7 +16,44 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
-COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "IT", "NA")
+COUNTRY_ENUM_VALUES = (
+	"AT",
+	"BE",
+	"BG",
+	"CA",
+	"CH",
+	"CY",
+	"CZ",
+	"DE",
+	"DK",
+	"EE",
+	"ES",
+	"FI",
+	"FR",
+	"GB",
+	"GR",
+	"HR",
+	"HU",
+	"IE",
+	"IT",
+	"JP",
+	"KR",
+	"LI",
+	"LT",
+	"LU",
+	"LV",
+	"MT",
+	"NA",
+	"NL",
+	"NO",
+	"PL",
+	"PT",
+	"RO",
+	"SE",
+	"SI",
+	"SK",
+	"US",
+)
 ORIGIN_ENUM_VALUES = (
 	"MOBILEDE",
 	"MOBILE_DE",
--- a/mobilede_scraper/storage/models.py
+++ b/mobilede_scraper/storage/models.py
@@ -5,7 +5,6 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship

 from .enums import (
 	BODY_TYPE_ENUM_VALUES,
-	COUNTRY_ENUM_VALUES,
 	CURRENCY_ENUM_VALUES,
 	DRIVE_ENUM_VALUES,
 	GEARBOX_ENUM_VALUES,
@@ -33,7 +32,7 @@ class Car(Base):
 	price: Mapped[int | None] = mapped_column(BigInteger, nullable=True)
 	currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD")
 	mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
-	country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA")
+	country: Mapped[str] = mapped_column(String(10), nullable=False, default="NA")
 	is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True)
 	color: Mapped[str] = mapped_column(String(), nullable=False, default="other")
 	drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True)
--- a/runtime_config.json
+++ b/runtime_config.json
@@ -6,7 +6,7 @@
    "ids_max_pages": null,
    "condition_check_enabled": false,
    "lane": "MOBILEDE_cars",
-    "only_new": false,
+    "only_new": true,
    "limit": null
  },
  "filters": {