qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

Browse Source
This commit is contained in:
qananasikq
2026-04-20 16:27:42 +03:00
parent 55203d33ea
commit 65d5f8e1eb
23 changed files with 3436 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

20
iaai_scraper/core/config.py
View File

@@ -106,6 +106,7 @@ class ListingConfig:
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
resume_max_nav_pages: int = _env_int("IAAI_LISTING_RESUME_MAX_NAV_PAGES", 90)
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
@@ -118,6 +119,22 @@ class ListingConfig:
listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
@dataclass(slots=True)
class DiscoveryConfig:
mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 3000)
sitemap_timeout_seconds: int = _env_int("IAAI_SITEMAP_TIMEOUT_SECONDS", 30)
sitemap_retry_attempts: int = _env_int("IAAI_SITEMAP_RETRY_ATTEMPTS", 4)
sitemap_retry_backoff_seconds: float = _env_float("IAAI_SITEMAP_RETRY_BACKOFF_SECONDS", 1.25)
sitemap_direct_probe_limit: int = _env_int("IAAI_SITEMAP_DIRECT_PROBE_LIMIT", 12)
sitemap_direct_probe_stop_after_misses: int = _env_int(
"IAAI_SITEMAP_DIRECT_PROBE_STOP_AFTER_MISSES",
3,
)
sitemap_use_curl_cffi: bool = _env_bool("IAAI_SITEMAP_USE_CURL_CFFI", True)
# Список брендов IAAI для автоматической сегментации.
# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
IAAI_DEFAULT_MAKES: tuple[str, ...] = (
@@ -217,6 +234,7 @@ class CeleryConfig:
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
@@ -224,6 +242,7 @@ class CeleryConfig:
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
parallel_segments: bool = _env_bool("IAAI_PARALLEL_SEGMENTS", False)
block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
@@ -278,6 +297,7 @@ class Settings:
capture: CaptureConfig = field(default_factory=CaptureConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)