improve runtime sync flow

mobilede_scraper/core/config.py

@@ -51,7 +51,6 @@ def _env_float(name: str, default: float) -> float:
 
 @dataclass(slots=True)
 class ListingConfig:
-    cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.mobile.de/Vehiclelisting/Cars")
     filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
     filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
 
@@ -112,51 +111,6 @@ class CeleryConfig:
     worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
     broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
     beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
-    beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
-    batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
-    parallel_tabs: int = _env_int("MOBILEDE_PARALLEL_TABS", 8)
-    fetch_concurrency: int = _env_int("MOBILEDE_FETCH_CONCURRENCY", _env_int("MOBILEDE_PARALLEL_TABS", 8))
-    parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
-    block_resources: bool = _env_bool("MOBILEDE_BLOCK_RESOURCES", True)
-
-
-@dataclass(slots=True)
-class ScrapingProfileConfig:
-    name: str = _env_str("MOBILEDE_SCRAPING_PROFILE", _env_str("MOBILEDE_PROFILE", "stable")).strip().lower()
-    http_first: bool = _env_bool("MOBILEDE_HTTP_FIRST", True)
-    browser_fallback_enabled: bool = _env_bool("MOBILEDE_BROWSER_FALLBACK_ENABLED", True)
-    anonymous_bootstrap_enabled: bool = _env_bool("MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED", True)
-    verbose_http_logs: bool = _env_bool("MOBILEDE_VERBOSE_HTTP_LOGS", False)
-    verbose_progress_logs: bool = _env_bool("MOBILEDE_VERBOSE_PROGRESS_LOGS", False)
-    challenge_refresh_enabled: bool = _env_bool("MOBILEDE_CHALLENGE_REFRESH_ENABLED", True)
-    challenge_refresh_attempts: int = _env_int("MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS", 3)
-    listing_post_attempts: int = _env_int("MOBILEDE_LISTING_POST_ATTEMPTS", 4)
-    request_jitter_max_s: float = _env_float("MOBILEDE_REQUEST_JITTER_MAX_S", 0.15)
-    detail_retries: int | None = _env_int("MOBILEDE_DETAIL_RETRIES", -1)
-    listing_retries: int | None = _env_int("MOBILEDE_LISTING_RETRIES", -1)
-
-    def __post_init__(self) -> None:
-        if self.name == "fast":
-            if "MOBILEDE_BROWSER_FALLBACK_ENABLED" not in os.environ:
-                self.browser_fallback_enabled = False
-            if "MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
-                self.anonymous_bootstrap_enabled = False
-            if "MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
-                self.challenge_refresh_attempts = 1
-            if "MOBILEDE_LISTING_POST_ATTEMPTS" not in os.environ:
-                self.listing_post_attempts = 2
-            if "MOBILEDE_REQUEST_JITTER_MAX_S" not in os.environ:
-                self.request_jitter_max_s = 0.0
-            if "MOBILEDE_DETAIL_RETRIES" not in os.environ:
-                self.detail_retries = 1
-            if "MOBILEDE_LISTING_RETRIES" not in os.environ:
-                self.listing_retries = 1
-        else:
-            if self.detail_retries is not None and self.detail_retries < 0:
-                self.detail_retries = None
-            if self.listing_retries is not None and self.listing_retries < 0:
-                self.listing_retries = None
-
 
 # Конфиг прокси.
 
@@ -170,16 +124,6 @@ class ProxyConfig:
     def enabled(self) -> bool:
         return bool(self.server)
 
-    def to_playwright_dict(self) -> dict[str, str] | None:
-        if not self.server:
-            return None
-        result: dict[str, str] = {"server": self.server}
-        if self.username:
-            result["username"] = self.username
-        if self.password:
-            result["password"] = self.password
-        return result
-
     def to_requests_proxy_url(self) -> str | None:
         if not self.server:
             return None
@@ -211,44 +155,13 @@ class ProxyConfig:
 
 @dataclass(slots=True)
 class Settings:
-    home_url: str = "https://www.mobile.de/"
-    default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000)
-    network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400)
-    fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000)
-    fast_path_max_attempts: int = _env_int("MOBILEDE_FAST_PATH_MAX_ATTEMPTS", 1)
-    fallback_navigation_timeout_ms: int = _env_int("MOBILEDE_FALLBACK_NAV_TIMEOUT_MS", 15000)
-    max_retries: int = _env_int("MOBILEDE_MAX_RETRIES", 3)
-    retry_delay_seconds: float = _env_float("MOBILEDE_RETRY_DELAY_SECONDS", 2.5)
-    retry_backoff_multiplier: float = _env_float("MOBILEDE_RETRY_BACKOFF_MULTIPLIER", 2.0)
-    retry_jitter_seconds: float = _env_float("MOBILEDE_RETRY_JITTER_SECONDS", 0.25)
     headless: bool = _env_bool("MOBILEDE_HEADLESS", True)
-    browser_engine: str = _env_str("MOBILEDE_BROWSER_ENGINE", "auto")
     log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
-    log_file: str | None = _env_optional_str("MOBILEDE_LOG_FILE")
-    enable_trace_id_logs: bool = _env_bool("MOBILEDE_ENABLE_TRACE_ID_LOGS", True)
-    sync_only_new: bool = _env_bool("MOBILEDE_SYNC_ONLY_NEW", False)
-    raw_output_json: str | None = _env_optional_str("MOBILEDE_RAW_OUTPUT_JSON")
-    tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE")
     runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
-    scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60)
     listing: ListingConfig = field(default_factory=ListingConfig)
     database: DatabaseConfig = field(default_factory=DatabaseConfig)
     redis: RedisConfig = field(default_factory=RedisConfig)
     celery: CeleryConfig = field(default_factory=CeleryConfig)
-    proxy: ProxyConfig = field(default_factory=ProxyConfig)
-    scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
-
-    @property
-    def parallel_tabs(self) -> int:
-        return self.celery.parallel_tabs
-
-    @property
-    def fetch_concurrency(self) -> int:
-        return self.celery.fetch_concurrency
-
-    @property
-    def block_resources(self) -> bool:
-        return self.celery.block_resources
 
 # Глобальные настройки.
 settings = Settings()

mobilede_scraper/core/logs.py

@@ -2,12 +2,12 @@ import logging
 import sys
 from contextvars import ContextVar
 
-# Храним trace_id текущего потока/корутины.
+# Trace ID текущего потока или корутины.
 TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
 
 
 class TraceIdFilter(logging.Filter):
-    # Добавляет trace_id в каждую запись лога для сквозной трассировки.
+    # Добавляет trace_id в запись лога.
     def filter(self, record: logging.LogRecord) -> bool:
         record.trace_id = TRACE_ID.get()
         return True
@@ -18,7 +18,7 @@ def set_trace_id(trace_id: str) -> None:
 
 
 def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
-    # stderr — Docker и Celery prefork корректно его подхватывают.
+    # Пишем в stderr для Docker и Celery.
     handlers: list[logging.Handler] = [logging.StreamHandler(sys.stderr)]
     if log_file:
         handlers.append(logging.FileHandler(log_file, encoding="utf-8"))
@@ -28,7 +28,7 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
         handler.setLevel(getattr(logging, level.upper(), logging.INFO))
     root = logging.getLogger()
     root.setLevel(getattr(logging, level.upper(), logging.INFO))
-    # Убираем старые хендлеры, чтобы не дублировать после fork.
+    # Убираем старые хендлеры после fork.
     for old_handler in list(root.handlers):
         try:
             old_handler.close()