diff --git a/.env.example b/.env.example index affe2ed..77802bf 100644 --- a/.env.example +++ b/.env.example @@ -25,7 +25,6 @@ CELERY_BROKER_VISIBILITY_TIMEOUT=90000 CELERY_WORKER_CONCURRENCY=3 CELERY_WORKER_POOL=prefork CELERY_WORKER_MAX_TASKS_PER_CHILD=5 -CELERY_BATCH_SIZE=1000 # Runtime mode MOBILEDE_STARTUP_SYNC_ENABLED=false @@ -39,8 +38,6 @@ MOBILEDE_BOOTSTRAP_CONTINUATION_DELAY_SECONDS=1 # Search settings MOBILEDE_FILTERED_SEARCH_URL= MOBILEDE_FILTERED_SEARCH_URLS= -MOBILEDE_STARTUP_MAX_PAGES=100 -MOBILEDE_BEAT_MAX_PAGES=100 MOBILEDE_REQUEST_DELAY_SECONDS=0.30 MOBILEDE_CONCURRENT_PAGES=1 MOBILEDE_CURSOR_ENABLED=true @@ -93,12 +90,4 @@ SOCKS5_PROXY_PORT=1002 SOCKS5_PROXY_USER= SOCKS5_PROXY_PASS= -# IAAI -IAAI_STARTUP_SYNC_ENABLED=true -IAAI_LOG_LEVEL=INFO -IAAI_BROWSER_ENGINE=chromium -IAAI_TOKENS_FILE=/data/tokens.json -IAAI_RUNTIME_CONFIG_FILE=/app/runtime_config.json -IAAI_SELF_HEAL_ENABLED=true - TZ=UTC diff --git a/.gitignore b/.gitignore index cd3ffec..86557bd 100644 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,8 @@ coverage.xml !.env.example .venv/ venv/ +*.tgz +*.tar.gz *.pyc *.pyo *.pyd diff --git a/Dockerfile b/Dockerfile index 036c5d9..b1e40f1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ STOPSIGNAL SIGINT USER app -# РџРѕ умолчанию запускаем API. +# По умолчанию запускаем API. ENTRYPOINT ["./entrypoint.sh"] CMD ["uvicorn", "mobilede_scraper.api.app:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/alembic/env.py b/alembic/env.py index 3246fd7..496aebb 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -16,7 +16,7 @@ from mobilede_scraper.storage.models import Base config = context.config VERSION_TABLE = "MOBILEDE_parser_alembic_version" -# Logging +# Логирование if config.config_file_name is not None: fileConfig(config.config_file_name) @@ -28,7 +28,7 @@ target_metadata = Base.metadata def run_migrations_offline() -> None: - # Run migrations in 'offline' mode. + # Запуск миграций в офлайн-режиме. url = config.get_main_option("sqlalchemy.url") context.configure( url=url, @@ -42,7 +42,7 @@ def run_migrations_offline() -> None: def run_migrations_online() -> None: - # Run migrations in 'online' mode. + # Запуск миграций в онлайн-режиме. connectable = engine_from_config( config.get_section(config.config_ini_section, {}), prefix="sqlalchemy.", diff --git a/alembic/versions/001_initial.py b/alembic/versions/001_initial.py index 7276934..bc42385 100644 --- a/alembic/versions/001_initial.py +++ b/alembic/versions/001_initial.py @@ -99,5 +99,5 @@ def upgrade() -> None: def downgrade() -> None: - # Compatibility-only migration for shared production databases. + # Миграция оставлена только для совместимости с общей production-базой. pass diff --git a/alembic/versions/002_add_indexes.py b/alembic/versions/002_add_indexes.py index da39c54..fcbd6e8 100644 --- a/alembic/versions/002_add_indexes.py +++ b/alembic/versions/002_add_indexes.py @@ -20,5 +20,5 @@ def upgrade() -> None: def downgrade() -> None: - # Compatibility-only migration for shared production databases. + # Миграция оставлена только для совместимости с общей production-базой. pass diff --git a/alembic/versions/003_add_composite_indexes.py b/alembic/versions/003_add_composite_indexes.py index 5bcb91e..0797138 100644 --- a/alembic/versions/003_add_composite_indexes.py +++ b/alembic/versions/003_add_composite_indexes.py @@ -10,21 +10,21 @@ depends_on: Union[str, Sequence[str], None] = None def upgrade() -> None: - # Partial index для активных машин MOBILEDE (is_sold = FALSE) - # Ускоряет поиск при mark_sold операциях + # Частичный индекс для активных машин MOBILEDE (is_sold = FALSE). + # Ускоряет поиск при операциях mark_sold. op.execute( 'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_active_MOBILEDE" ' 'ON "MOBILEDE_cars" (origin_url) ' "WHERE is_sold = FALSE AND origin_id LIKE 'mobilede:%'" ) - # Composite index для проверки дубликатов изображений + # Составной индекс для проверки дубликатов изображений. op.execute( 'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_images_car_id_fullres" ' 'ON "MOBILEDE_images" (car_id, fullres_image)' ) - # Index для быстрого поиска existing машин по origin_url + # Индекс для быстрого поиска существующих машин по origin_url. op.execute( 'CREATE INDEX IF NOT EXISTS "ix_MOBILEDE_cars_origin_url_id" ' 'ON "MOBILEDE_cars" (origin_url, origin_id)' @@ -32,5 +32,5 @@ def upgrade() -> None: def downgrade() -> None: - # Compatibility-only migration for shared production databases. + # Миграция оставлена только для совместимости с общей production-базой. pass diff --git a/alembic/versions/004_add_ingestion_tables.py b/alembic/versions/004_add_ingestion_tables.py index 3ce4466..a204e00 100644 --- a/alembic/versions/004_add_ingestion_tables.py +++ b/alembic/versions/004_add_ingestion_tables.py @@ -1,4 +1,4 @@ -# Placeholder migration (ingestion tables not yet needed) +# Заглушка миграции (таблицы ingestion пока не требуются) from typing import Sequence, Union from alembic import op diff --git a/docker-compose.yml b/docker-compose.yml index d36dd80..45682b7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -4,31 +4,17 @@ CELERY_BROKER_URL: ${CELERY_BROKER_URL:-redis://redis:6379/0} CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/0} MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS: ${MOBILEDE_DATABASE_POOL_RECYCLE_SECONDS:-1800} - MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-20} - MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-40} + MOBILEDE_DATABASE_POOL_SIZE: ${MOBILEDE_DATABASE_POOL_SIZE:-40} + MOBILEDE_DATABASE_MAX_OVERFLOW: ${MOBILEDE_DATABASE_MAX_OVERFLOW:-80} CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-900} CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-1200} CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-2400} - IAAI_PROFILE: ${IAAI_PROFILE:-fast} - IAAI_SCRAPING_PROFILE: ${IAAI_SCRAPING_PROFILE:-${IAAI_PROFILE:-fast}} - IAAI_HTTP_FIRST: ${IAAI_HTTP_FIRST:-true} - IAAI_BROWSER_FALLBACK_ENABLED: ${IAAI_BROWSER_FALLBACK_ENABLED:-false} - IAAI_ANONYMOUS_BOOTSTRAP_ENABLED: ${IAAI_ANONYMOUS_BOOTSTRAP_ENABLED:-false} - IAAI_CHALLENGE_REFRESH_ATTEMPTS: ${IAAI_CHALLENGE_REFRESH_ATTEMPTS:-1} - IAAI_LISTING_POST_ATTEMPTS: ${IAAI_LISTING_POST_ATTEMPTS:-2} - IAAI_REQUEST_JITTER_MAX_S: ${IAAI_REQUEST_JITTER_MAX_S:-0} - IAAI_DETAIL_RETRIES: ${IAAI_DETAIL_RETRIES:-1} - IAAI_LISTING_RETRIES: ${IAAI_LISTING_RETRIES:-1} - # 0 = без лимита. - CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} - CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-4} + CELERY_WORKER_CONCURRENCY: ${CELERY_WORKER_CONCURRENCY:-6} CELERY_WORKER_POOL: ${CELERY_WORKER_POOL:-prefork} CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} - CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-1000} - MOBILEDE_STARTUP_MAX_PAGES: ${MOBILEDE_STARTUP_MAX_PAGES:-100} - MOBILEDE_BEAT_MAX_PAGES: ${MOBILEDE_BEAT_MAX_PAGES:-100} - MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0} - MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-2} + MOBILEDE_REQUEST_DELAY_SECONDS: ${MOBILEDE_REQUEST_DELAY_SECONDS:-0.45} + MOBILEDE_CONCURRENT_PAGES: ${MOBILEDE_CONCURRENT_PAGES:-1} + MOBILEDE_ANTIBOT_BACKOFF_SECONDS: "1800" MOBILEDE_FILTERED_SEARCH_URL: ${MOBILEDE_FILTERED_SEARCH_URL:-} MOBILEDE_FILTERED_SEARCH_URLS: ${MOBILEDE_FILTERED_SEARCH_URLS:-} MOBILEDE_CURSOR_ENABLED: ${MOBILEDE_CURSOR_ENABLED:-true} @@ -37,45 +23,48 @@ MOBILEDE_PROGRESS_LOG_EVERY_PAGES: ${MOBILEDE_PROGRESS_LOG_EVERY_PAGES:-10} MOBILEDE_SKIP_EMPTY_WINDOW: ${MOBILEDE_SKIP_EMPTY_WINDOW:-true} MOBILEDE_ROTATE_RUNTIME_SEGMENTS: ${MOBILEDE_ROTATE_RUNTIME_SEGMENTS:-true} - MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-2} + MOBILEDE_RUNTIME_INITIAL_TASKS: ${MOBILEDE_RUNTIME_INITIAL_TASKS:-5} + MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP: ${MOBILEDE_SKIP_LATE_OVERFLOW_CHILDREN_DURING_BOOTSTRAP:-true} MOBILEDE_SEGMENT_PAGE_WINDOW: ${MOBILEDE_SEGMENT_PAGE_WINDOW:-10} MOBILEDE_RESULTS_PER_PAGE: ${MOBILEDE_RESULTS_PER_PAGE:-20} MOBILEDE_MAX_PAGE_NUMBER: ${MOBILEDE_MAX_PAGE_NUMBER:-50} - MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-1000} + MOBILEDE_SEGMENT_TARGET_RESULTS: ${MOBILEDE_SEGMENT_TARGET_RESULTS:-700} MOBILEDE_COMPACT_SEGMENTS: ${MOBILEDE_COMPACT_SEGMENTS:-true} - MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-false} - MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-1000} - MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-40} + # Первый проход должен идти по заранее нарезанным leaf-сегментам. + # Значения заданы жёстко, чтобы старые переменные хоста/.env не отключали preplan локально. + MOBILEDE_PREPLAN_SEGMENT_PROBES: ${MOBILEDE_PREPLAN_SEGMENT_PROBES:-true} + MOBILEDE_PREPLAN_MAX_SEGMENTS: ${MOBILEDE_PREPLAN_MAX_SEGMENTS:-2400} + MOBILEDE_PREPLAN_MAX_PROBES: ${MOBILEDE_PREPLAN_MAX_PROBES:-320} + MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO: ${MOBILEDE_PREPLAN_SPLIT_THRESHOLD_RATIO:-1.0} + MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH: ${MOBILEDE_PREPLAN_MAX_SPLIT_DEPTH:-3} + MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN: ${MOBILEDE_FILTERED_URL_ADAPTIVE_PLAN:-true} + MOBILEDE_FILTERED_URL_FAST_START: ${MOBILEDE_FILTERED_URL_FAST_START:-false} + MOBILEDE_REFINE_ADAPTIVE_SEGMENTS: ${MOBILEDE_REFINE_ADAPTIVE_SEGMENTS:-true} + MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS: ${MOBILEDE_REFINE_ADAPTIVE_MAX_SECONDS:-240} + MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES: ${MOBILEDE_REFINE_ADAPTIVE_MAX_PASSES:-3} + MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS: ${MOBILEDE_PLAN_PROBE_TIMEOUT_SECONDS:-8} MOBILEDE_DYNAMIC_SEGMENT_PROBES: ${MOBILEDE_DYNAMIC_SEGMENT_PROBES:-false} - MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-false} + MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE: ${MOBILEDE_SPLIT_SEGMENTS_BY_MILEAGE:-true} + MOBILEDE_SEGMENT_TARGET_MIN_RATIO: ${MOBILEDE_SEGMENT_TARGET_MIN_RATIO:-0.70} + MOBILEDE_SEGMENT_TARGET_MAX_RATIO: ${MOBILEDE_SEGMENT_TARGET_MAX_RATIO:-0.90} + MOBILEDE_SEGMENT_TINY_RATIO: ${MOBILEDE_SEGMENT_TINY_RATIO:-0.35} MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS: ${MOBILEDE_SKIP_EMPTY_DYNAMIC_SEGMENTS:-true} MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true} - MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true} - MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1} - IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true} - IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0} - IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9} - IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-8} - IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-32} - IAAI_BLOCK_RESOURCES: ${IAAI_BLOCK_RESOURCES:-true} - IAAI_FILTERED_SEARCH_URL: ${IAAI_FILTERED_SEARCH_URL:-} - IAAI_FILTERED_SEARCH_URLS: ${IAAI_FILTERED_SEARCH_URLS:-} - IAAI_LISTING_SEGMENTS: ${IAAI_LISTING_SEGMENTS:-runtime} - IAAI_MAX_PAGES_PER_RUN: ${IAAI_MAX_PAGES_PER_RUN:-9999} - IAAI_MAX_VEHICLES_PER_RUN: ${IAAI_MAX_VEHICLES_PER_RUN:-50000} - IAAI_ALWAYS_FULL_SCAN: ${IAAI_ALWAYS_FULL_SCAN:-true} - IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true} - IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/home/app/tokens.json} - IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} - IAAI_BROWSER_ENGINE: chromium + MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-false} + MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-10} + MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED: ${MOBILEDE_SELECTIVE_DETAIL_ENRICH_ENABLED:-false} + MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED: ${MOBILEDE_DETAIL_ENRICH_IMAGES_ENABLED:-false} + MOBILEDE_DETAIL_TIMEOUT_SECONDS: ${MOBILEDE_DETAIL_TIMEOUT_SECONDS:-10} + MOBILEDE_STARTUP_SYNC_ENABLED: ${MOBILEDE_STARTUP_SYNC_ENABLED:-true} + MOBILEDE_RUNTIME_CONFIG_FILE: ${MOBILEDE_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} # Автовосстановление worker. - IAAI_SELF_HEAL_ENABLED: ${IAAI_SELF_HEAL_ENABLED:-true} - IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${IAAI_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} - IAAI_SELF_HEAL_STALL_SECONDS: ${IAAI_SELF_HEAL_STALL_SECONDS:-900} - IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS: ${IAAI_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} - IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${IAAI_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} + MOBILEDE_SELF_HEAL_ENABLED: ${MOBILEDE_SELF_HEAL_ENABLED:-true} + MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS: ${MOBILEDE_SELF_HEAL_CHECK_INTERVAL_SECONDS:-30} + MOBILEDE_SELF_HEAL_STALL_SECONDS: ${MOBILEDE_SELF_HEAL_STALL_SECONDS:-900} + MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS: ${MOBILEDE_SELF_HEAL_STARTUP_GRACE_SECONDS:-300} + MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS: ${MOBILEDE_SELF_HEAL_RESTART_COOLDOWN_SECONDS:-300} # При простое БД перезапускаем worker. - IAAI_DB_IDLE_RESTART_SECONDS: ${IAAI_DB_IDLE_RESTART_SECONDS:-3600} + MOBILEDE_DB_IDLE_RESTART_SECONDS: ${MOBILEDE_DB_IDLE_RESTART_SECONDS:-3600} TZ: ${TZ:-UTC} x-env-file: &env-file @@ -182,7 +171,6 @@ services: worker: <<: *worker-service restart: unless-stopped - init: true depends_on: migrate: condition: service_completed_successfully @@ -191,9 +179,9 @@ services: stop_grace_period: 60s command: > celery -A mobilede_scraper.worker.celery_app worker - --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-4} --pool=${CELERY_WORKER_POOL:-prefork} + --loglevel=info --concurrency=${CELERY_WORKER_CONCURRENCY:-8} --pool=${CELERY_WORKER_POOL:-prefork} --pidfile=/tmp/celery-worker.pid - -Q mobilede_sync,iaai_sync --max-tasks-per-child=${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} + -Q mobilede_sync --max-tasks-per-child=5 healthcheck: # Проверка worker. test: ["CMD-SHELL", "test -f /tmp/celery-worker.pid && kill -0 $(cat /tmp/celery-worker.pid) && pgrep -f 'mobilede_scraper.worker.self_heal' >/dev/null"] diff --git a/mobilede_scraper/storage/db.py b/mobilede_scraper/storage/db.py index dfcef83..3a5c0f3 100644 --- a/mobilede_scraper/storage/db.py +++ b/mobilede_scraper/storage/db.py @@ -4,7 +4,7 @@ from contextlib import contextmanager from datetime import datetime, timezone from typing import Any, Iterator -from sqlalchemy import create_engine, delete, or_, select, text, update +from sqlalchemy import create_engine, delete, func, or_, select, text, update from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.orm import Session, sessionmaker @@ -808,6 +808,34 @@ class PersistenceService: if row and row[0] and row[1] and row[2] ] + def get_active_cars_batch_for_image_enrich( + self, + prefix: str | tuple[str, ...] = MOBILEDE_ORIGIN_PREFIXES, + *, + limit: int = 50, + max_existing_images: int = 1, + ) -> list[tuple[int, str, str, int]]: + prefixes = (prefix,) if isinstance(prefix, str) else tuple(prefix) + with self.session_scope() as session: + image_count = func.count(Image.id) + result = session.execute( + select(Car.id, Car.origin_id, Car.origin_url, image_count.label("image_count")) + .outerjoin(Image, Image.car_id == Car.id) + .where( + _origin_prefix_filter(Car.origin_id, prefixes), + Car.is_sold == False, # noqa: E712 + ) + .group_by(Car.id, Car.origin_id, Car.origin_url) + .having(image_count <= max(0, int(max_existing_images))) + .order_by(Car.last_seen_at.desc()) + .limit(max(1, int(limit))) + ) + return [ + (int(row[0]), str(row[1]), str(row[2]), int(row[3] or 0)) + for row in result + if row and row[0] and row[1] and row[2] + ] + def mark_cars_sold_by_ids(self, car_ids: list[int], *, sold_at: datetime | None = None) -> int: if not car_ids: return 0 diff --git a/mobilede_scraper/storage/enums.py b/mobilede_scraper/storage/enums.py index c0cece2..4357d2b 100644 --- a/mobilede_scraper/storage/enums.py +++ b/mobilede_scraper/storage/enums.py @@ -16,7 +16,44 @@ BODY_TYPE_ENUM_VALUES = ( "OTHER", "NA", ) -COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "IT", "NA") +COUNTRY_ENUM_VALUES = ( + "AT", + "BE", + "BG", + "CA", + "CH", + "CY", + "CZ", + "DE", + "DK", + "EE", + "ES", + "FI", + "FR", + "GB", + "GR", + "HR", + "HU", + "IE", + "IT", + "JP", + "KR", + "LI", + "LT", + "LU", + "LV", + "MT", + "NA", + "NL", + "NO", + "PL", + "PT", + "RO", + "SE", + "SI", + "SK", + "US", +) ORIGIN_ENUM_VALUES = ( "MOBILEDE", "MOBILE_DE", diff --git a/mobilede_scraper/storage/models.py b/mobilede_scraper/storage/models.py index 4728abc..1a4a142 100644 --- a/mobilede_scraper/storage/models.py +++ b/mobilede_scraper/storage/models.py @@ -5,7 +5,6 @@ from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship from .enums import ( BODY_TYPE_ENUM_VALUES, - COUNTRY_ENUM_VALUES, CURRENCY_ENUM_VALUES, DRIVE_ENUM_VALUES, GEARBOX_ENUM_VALUES, @@ -33,7 +32,7 @@ class Car(Base): price: Mapped[int | None] = mapped_column(BigInteger, nullable=True) currency: Mapped[str] = mapped_column(Enum(*CURRENCY_ENUM_VALUES, name="currencyenum", native_enum=False, create_constraint=False), nullable=False, default="USD") mileage: Mapped[int] = mapped_column(Integer, nullable=False, default=0) - country: Mapped[str] = mapped_column(Enum(*COUNTRY_ENUM_VALUES, name="countryenum", native_enum=False, create_constraint=False), nullable=False, default="NA") + country: Mapped[str] = mapped_column(String(10), nullable=False, default="NA") is_sold: Mapped[bool] = mapped_column(Boolean, nullable=False, default=False, index=True) color: Mapped[str] = mapped_column(String(), nullable=False, default="other") drive: Mapped[str | None] = mapped_column(Enum(*DRIVE_ENUM_VALUES, name="driveenum", native_enum=False, create_constraint=False), nullable=True) diff --git a/runtime_config.json b/runtime_config.json index 565cecd..e3aa61c 100644 --- a/runtime_config.json +++ b/runtime_config.json @@ -6,7 +6,7 @@ "ids_max_pages": null, "condition_check_enabled": false, "lane": "MOBILEDE_cars", - "only_new": false, + "only_new": true, "limit": null }, "filters": {