qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fast parser

Browse Source
This commit is contained in:
qananasikq
2026-04-24 12:39:05 +03:00
parent 3a3222c7dc
commit fc1bea562a
19 changed files with 2482 additions and 92 deletions
Download Patch File Download Diff File Expand all files Collapse all files

97
iaai_scraper/core/config.py
View File

@@ -113,9 +113,11 @@ class ListingConfig:
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
# Сегментация листинга по брендам для обхода лимита пагинации IAAI (~22 600 машин).
# JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...] или "auto" для авто-списка.
# JSON-массив объектов: [{"make":"TOYOTA"},{"make":"FORD"},...], "auto" для авто-списка
# или "runtime" для построения по runtime_config.filters.brands.
# Пустая строка = без сегментации (backward compatible).
listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
fast_segment_year_splits: bool = _env_bool("IAAI_FAST_SEGMENT_YEAR_SPLITS", True)
# Список брендов IAAI для автоматической сегментации.
@@ -149,6 +151,43 @@ _YEAR_SPLITS: tuple[tuple[int, int], ...] = (
)
def build_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
"""Строит сегменты по переданному списку брендов.
Крупные бренды режутся по годовым диапазонам так же, как в ``auto``.
"""
segments: list[dict[str, str | int | None]] = []
seen: set[str] = set()
for raw_make in makes:
make = str(raw_make or "").strip().upper()
if not make or make in seen:
continue
seen.add(make)
if make in _LARGE_MAKES:
for yr_min, yr_max in _YEAR_SPLITS:
segments.append({"make": make, "year_min": yr_min, "year_max": yr_max})
else:
segments.append({"make": make, "year_min": None, "year_max": None})
return segments
def build_fast_listing_segments_for_makes(makes: tuple[str, ...] | list[str]) -> list[dict[str, str | int | None]]:
"""Строит HTTP-first сегменты без UI-фильтров годов.
Это ближе к iaai-fast: один бренд = один hidden-payload HTTP обход.
Годовые split-сегменты требуют браузерный UI-фильтр и ломают стабильность fast-профиля.
"""
segments: list[dict[str, str | int | None]] = []
seen: set[str] = set()
for raw_make in makes:
make = str(raw_make or "").strip().upper()
if not make or make in seen:
continue
seen.add(make)
segments.append({"make": make, "year_min": None, "year_max": None})
return segments
def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
"""Парсит IAAI_LISTING_SEGMENTS в список сегментов.
@@ -160,14 +199,7 @@ def parse_listing_segments(raw: str) -> list[dict[str, str | int | None]]:
if not raw:
return []
if raw.lower() == "auto":
segments: list[dict[str, str | int | None]] = []
for m in IAAI_DEFAULT_MAKES:
if m in _LARGE_MAKES:
for yr_min, yr_max in _YEAR_SPLITS:
segments.append({"make": m, "year_min": yr_min, "year_max": yr_max})
else:
segments.append({"make": m, "year_min": None, "year_max": None})
return segments
return build_listing_segments_for_makes(list(IAAI_DEFAULT_MAKES))
try:
data = json.loads(raw)
except (json.JSONDecodeError, ValueError):
@@ -216,7 +248,7 @@ class DiscoveryConfig:
mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 500)
always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", True)
always_full_scan: bool = _env_bool("IAAI_ALWAYS_FULL_SCAN", False)
# --- Конфиг Celery (лимиты задач, concurrency, beat-расписание) ---
@@ -229,16 +261,54 @@ class CeleryConfig:
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_pool: str = _env_str("CELERY_WORKER_POOL", "prefork")
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
fetch_concurrency: int = _env_int("IAAI_FETCH_CONCURRENCY", _env_int("IAAI_PARALLEL_TABS", 8))
parallel_segments: bool = _env_bool("CELERY_PARALLEL_SEGMENTS", False)
block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
@dataclass(slots=True)
class ScrapingProfileConfig:
name: str = _env_str("IAAI_SCRAPING_PROFILE", _env_str("IAAI_PROFILE", "stable")).strip().lower()
http_first: bool = _env_bool("IAAI_HTTP_FIRST", True)
browser_fallback_enabled: bool = _env_bool("IAAI_BROWSER_FALLBACK_ENABLED", True)
anonymous_bootstrap_enabled: bool = _env_bool("IAAI_ANONYMOUS_BOOTSTRAP_ENABLED", True)
challenge_refresh_enabled: bool = _env_bool("IAAI_CHALLENGE_REFRESH_ENABLED", True)
challenge_refresh_attempts: int = _env_int("IAAI_CHALLENGE_REFRESH_ATTEMPTS", 3)
listing_post_attempts: int = _env_int("IAAI_LISTING_POST_ATTEMPTS", 4)
request_jitter_max_s: float = _env_float("IAAI_REQUEST_JITTER_MAX_S", 0.15)
detail_retries: int | None = _env_int("IAAI_DETAIL_RETRIES", -1)
listing_retries: int | None = _env_int("IAAI_LISTING_RETRIES", -1)
def __post_init__(self) -> None:
if self.name == "fast":
if "IAAI_BROWSER_FALLBACK_ENABLED" not in os.environ:
self.browser_fallback_enabled = False
if "IAAI_ANONYMOUS_BOOTSTRAP_ENABLED" not in os.environ:
self.anonymous_bootstrap_enabled = False
if "IAAI_CHALLENGE_REFRESH_ATTEMPTS" not in os.environ:
self.challenge_refresh_attempts = 1
if "IAAI_LISTING_POST_ATTEMPTS" not in os.environ:
self.listing_post_attempts = 2
if "IAAI_REQUEST_JITTER_MAX_S" not in os.environ:
self.request_jitter_max_s = 0.0
if "IAAI_DETAIL_RETRIES" not in os.environ:
self.detail_retries = 1
if "IAAI_LISTING_RETRIES" not in os.environ:
self.listing_retries = 1
else:
if self.detail_retries is not None and self.detail_retries < 0:
self.detail_retries = None
if self.listing_retries is not None and self.listing_retries < 0:
self.listing_retries = None
# --- Конфиг прокси (server, username, password) ---
@dataclass(slots=True)
@@ -269,7 +339,7 @@ class Settings:
home_url: str = "https://www.iaai.com/"
default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 15000)
fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
@@ -295,11 +365,16 @@ class Settings:
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
scraping_profile: ScrapingProfileConfig = field(default_factory=ScrapingProfileConfig)
@property
def parallel_tabs(self) -> int:
return self.celery.parallel_tabs
@property
def fetch_concurrency(self) -> int:
return self.celery.fetch_concurrency
@property
def block_resources(self) -> bool:
return self.celery.block_resources

6
iaai_scraper/core/logs.py
View File

@@ -29,9 +29,15 @@ def setup_logging(level: str = "INFO", log_file: str | None = None) -> None:
root = logging.getLogger()
root.setLevel(getattr(logging, level.upper(), logging.INFO))
# Убираем старые хендлеры, чтобы не дублировать после fork.
for old_handler in list(root.handlers):
try:
old_handler.close()
except Exception:
pass
root.handlers.clear()
for handler in handlers:
root.addHandler(handler)
root.propagate = False
fmt = logging.Formatter(
"%(asctime)s | %(levelname)s | %(name)s | trace=%(trace_id)s | %(message)s"
)