qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

improve runtime sync flow

Browse Source
This commit is contained in:
qananasikq
2026-05-11 12:45:51 +03:00
parent 124dfc514f
commit d9d073641b
13 changed files with 1224 additions and 356 deletions
Download Patch File Download Diff File Expand all files Collapse all files

87
mobilede_scraper/core/config.py
View File

@@ -51,7 +51,6 @@ def _env_float(name: str, default: float) -> float:
@dataclass(slots=True)
class ListingConfig:
cars_url: str = _env_str("MOBILEDE_CARS_LISTING_URL", "https://www.mobile.de/Vehiclelisting/Cars")
filtered_search_url: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URL")
filtered_search_urls_raw: str | None = _env_optional_str("MOBILEDE_FILTERED_SEARCH_URLS")
@@ -112,51 +111,6 @@ class CeleryConfig:
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("MOBILEDE_PARALLEL_TABS", 8)
fetch_concurrency: int = _env_int("MOBILEDE_FETCH_CONCURRENCY", _env_int("MOBILEDE_PARALLEL_TABS", 8))
parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
block_resources: bool = _env_bool("MOBILEDE_BLOCK_RESOURCES", True)
@dataclass(slots=True)
class ScrapingProfileConfig:
name: str = _env_str("MOBILEDE_SCRAPING_PROFILE", _env_str("MOBILEDE_PROFILE", "stable")).strip().lower()
http_first: bool = _env_bool("MOBILEDE_HTTP_FIRST", True)
browser_fallback_enabled: bool = _env_bool("MOBILEDE_BROWSER_FALLBACK_ENABLED", True)
anonymous_bootstrap_enabled: bool = _env_bool("MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED", True)
verbose_http_logs: bool = _env_bool("MOBILEDE_VERBOSE_HTTP_LOGS", False)
verbose_progress_logs: bool = _env_bool("MOBILEDE_VERBOSE_PROGRESS_LOGS", False)
challenge_refresh_enabled: bool = _env_bool("MOBILEDE_CHALLENGE_REFRESH_ENABLED", True)
challenge_refresh_attempts: int = _env_int("MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS", 3)
listing_post_attempts: int = _env_int("MOBILEDE_LISTING_POST_ATTEMPTS", 4)
request_jitter_max_s: float = _env_float("MOBILEDE_REQUEST_JITTER_MAX_S", 0.15)
detail_retries: int | None = _env_int("MOBILEDE_DETAIL_RETRIES", -1)
listing_retries: int | None = _env_int("MOBILEDE_LISTING_RETRIES", -1)
def __post_init__(self) -> None:
if self.name == "fast":
if "MOBILEDE_BROWSER_FALLBACK_ENABLED" not in os.environ:
self.browser_fallback_enabled = False
if "MOBILEDE_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
self.anonymous_bootstrap_enabled = False
if "MOBILEDE_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
self.challenge_refresh_attempts = 1
if "MOBILEDE_LISTING_POST_ATTEMPTS" not in os.environ:
self.listing_post_attempts = 2
if "MOBILEDE_REQUEST_JITTER_MAX_S" not in os.environ:
self.request_jitter_max_s = 0.0
if "MOBILEDE_DETAIL_RETRIES" not in os.environ:
self.detail_retries = 1
if "MOBILEDE_LISTING_RETRIES" not in os.environ:
self.listing_retries = 1
else:
if self.detail_retries is not None and self.detail_retries < 0:
self.detail_retries = None
if self.listing_retries is not None and self.listing_retries < 0:
self.listing_retries = None
# Конфиг прокси.
@@ -170,16 +124,6 @@ class ProxyConfig:
def enabled(self) -> bool:
return bool(self.server)
def to_playwright_dict(self) -> dict[str, str] | None:
if not self.server:
return None
result: dict[str, str] = {"server": self.server}
if self.username:
result["username"] = self.username
if self.password:
result["password"] = self.password
return result
def to_requests_proxy_url(self) -> str | None:
if not self.server:
return None
@@ -211,44 +155,13 @@ class ProxyConfig:
@dataclass(slots=True)
class Settings:
home_url: str = "https://www.mobile.de/"
default_timeout_ms: int = _env_int("MOBILEDE_TIMEOUT_MS", 45000)
network_settle_ms: int = _env_int("MOBILEDE_NETWORK_SETTLE_MS", 400)
fast_path_timeout_ms: int = _env_int("MOBILEDE_FAST_PATH_TIMEOUT_MS", 15000)
fast_path_max_attempts: int = _env_int("MOBILEDE_FAST_PATH_MAX_ATTEMPTS", 1)
fallback_navigation_timeout_ms: int = _env_int("MOBILEDE_FALLBACK_NAV_TIMEOUT_MS", 15000)
max_retries: int = _env_int("MOBILEDE_MAX_RETRIES", 3)
retry_delay_seconds: float = _env_float("MOBILEDE_RETRY_DELAY_SECONDS", 2.5)
retry_backoff_multiplier: float = _env_float("MOBILEDE_RETRY_BACKOFF_MULTIPLIER", 2.0)
retry_jitter_seconds: float = _env_float("MOBILEDE_RETRY_JITTER_SECONDS", 0.25)
headless: bool = _env_bool("MOBILEDE_HEADLESS", True)
browser_engine: str = _env_str("MOBILEDE_BROWSER_ENGINE", "auto")
log_level: str = _env_str("MOBILEDE_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("MOBILEDE_LOG_FILE")
enable_trace_id_logs: bool = _env_bool("MOBILEDE_ENABLE_TRACE_ID_LOGS", True)
sync_only_new: bool = _env_bool("MOBILEDE_SYNC_ONLY_NEW", False)
raw_output_json: str | None = _env_optional_str("MOBILEDE_RAW_OUTPUT_JSON")
tokens_file: str | None = _env_path_str("MOBILEDE_TOKENS_FILE")
runtime_config_file: str | None = _env_path_str("MOBILEDE_RUNTIME_CONFIG_FILE")
scheduler_interval_minutes: int = _env_int("MOBILEDE_SCHEDULER_INTERVAL_MINUTES", 60)
listing: ListingConfig = field(default_factory=ListingConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
@property
def parallel_tabs(self) -> int:
return self.celery.parallel_tabs
@property
def fetch_concurrency(self) -> int:
return self.celery.fetch_concurrency
@property
def block_resources(self) -> bool:
return self.celery.block_resources
# Глобальные настройки.
settings = Settings()