fix docker scraping
improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
This commit is contained in:
qananasikq
@@ -84,18 +84,18 @@ class CaptureConfig:
|
||||
@dataclass(slots=True)
|
||||
class HumanPaceConfig:
|
||||
enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
|
||||
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 2.5)
|
||||
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 4.5)
|
||||
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 2.0)
|
||||
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 4.0)
|
||||
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.5)
|
||||
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 1.5)
|
||||
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.3)
|
||||
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 1.0)
|
||||
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.5)
|
||||
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 1.5)
|
||||
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 3.0)
|
||||
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 6.0)
|
||||
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
|
||||
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
|
||||
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
|
||||
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
|
||||
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
|
||||
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
|
||||
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
|
||||
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
|
||||
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
|
||||
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
|
||||
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
|
||||
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
|
||||
|
||||
|
||||
# Конфиг сбора листинга (URL, лимиты страниц и машин)
|
||||
@@ -103,11 +103,14 @@ class HumanPaceConfig:
|
||||
@dataclass(slots=True)
|
||||
class ListingConfig:
|
||||
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
|
||||
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 5)
|
||||
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 100)
|
||||
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 200)
|
||||
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
|
||||
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
|
||||
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
|
||||
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
|
||||
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
|
||||
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
|
||||
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
|
||||
early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
|
||||
|
||||
|
||||
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
|
||||
@@ -135,13 +138,16 @@ class RedisConfig:
|
||||
class CeleryConfig:
|
||||
broker_url: str = _env_str("CELERY_BROKER_URL", "")
|
||||
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
|
||||
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 600)
|
||||
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 900)
|
||||
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 1)
|
||||
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 20)
|
||||
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
|
||||
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
|
||||
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
|
||||
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
|
||||
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
|
||||
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
|
||||
beat_sync_limit: int = _env_int("CELERY_BEAT_SYNC_LIMIT", 26)
|
||||
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
|
||||
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
|
||||
parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
|
||||
block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
|
||||
|
||||
|
||||
# --- Конфиг прокси (server, username, password) ---
|
||||
@@ -167,38 +173,22 @@ class ProxyConfig:
|
||||
return result
|
||||
|
||||
|
||||
# --- Конфиг сессии браузера (куки, storage_state) ---
|
||||
|
||||
@dataclass(slots=True)
|
||||
class SessionConfig:
|
||||
storage_state_path: str | None = _env_optional_str("IAAI_STORAGE_STATE_PATH")
|
||||
max_age_days: int = _env_int("IAAI_SESSION_MAX_AGE_DAYS", 30)
|
||||
save_on_exit: bool = _env_bool("IAAI_SESSION_SAVE_ON_EXIT", True)
|
||||
login_email: str | None = _env_optional_str("IAAI_LOGIN_EMAIL")
|
||||
login_password: str | None = _env_optional_str("IAAI_LOGIN_PASSWORD")
|
||||
login_url: str = _env_str("IAAI_LOGIN_URL", "https://www.iaai.com/Login")
|
||||
|
||||
@property
|
||||
def enabled(self) -> bool:
|
||||
return bool(self.storage_state_path)
|
||||
|
||||
@property
|
||||
def has_credentials(self) -> bool:
|
||||
return bool(self.login_email and self.login_password)
|
||||
|
||||
|
||||
# --- Главный объект настроек: собирает все блоки конфигурации ---
|
||||
|
||||
@dataclass(slots=True)
|
||||
class Settings:
|
||||
home_url: str = "https://www.iaai.com/"
|
||||
default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
|
||||
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 800)
|
||||
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
|
||||
fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
|
||||
fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
|
||||
fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
|
||||
max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
|
||||
retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
|
||||
retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
|
||||
retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
|
||||
headless: bool = _env_bool("IAAI_HEADLESS", True)
|
||||
browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
|
||||
log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
|
||||
log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
|
||||
enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
|
||||
@@ -215,7 +205,14 @@ class Settings:
|
||||
redis: RedisConfig = field(default_factory=RedisConfig)
|
||||
celery: CeleryConfig = field(default_factory=CeleryConfig)
|
||||
proxy: ProxyConfig = field(default_factory=ProxyConfig)
|
||||
session: SessionConfig = field(default_factory=SessionConfig)
|
||||
|
||||
@property
|
||||
def parallel_tabs(self) -> int:
|
||||
return self.celery.parallel_tabs
|
||||
|
||||
@property
|
||||
def block_resources(self) -> bool:
|
||||
return self.celery.block_resources
|
||||
|
||||
# Глобальный синглтон — используется по умолчанию во всех модулях.
|
||||
settings = Settings()
|
||||
|
||||
Reference in New Issue
Block a user