qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix docker scraping

Browse Source 
improve batch sync

add postgres upsert

fix sync locking

improve listing sync

speed up scraper

clean up project

prepare for github

update docker setup
This commit is contained in:
qananasikq
2026-04-13 16:35:08 +03:00
parent b9557922ed
commit a4c93a1df1
30 changed files with 2239 additions and 624 deletions
Download Patch File Download Diff File Expand all files Collapse all files
iaai_scraper/core/config.py
+39 -42
View File
@@ -84,18 +84,18 @@ class CaptureConfig:
@dataclass(slots=True)
class HumanPaceConfig:
enabled: bool = _env_bool("IAAI_HUMAN_PACE_ENABLED", True)
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 2.5)
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 4.5)
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 2.0)
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 4.0)
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.5)
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 1.5)
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.3)
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 1.0)
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.5)
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 1.5)
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 3.0)
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 6.0)
after_listing_open_min_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MIN_S", 0.5)
after_listing_open_max_s: float = _env_float("IAAI_AFTER_LISTING_OPEN_MAX_S", 1.2)
after_filter_action_min_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MIN_S", 0.5)
after_filter_action_max_s: float = _env_float("IAAI_AFTER_FILTER_ACTION_MAX_S", 1.2)
before_vehicle_open_min_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MIN_S", 0.1)
before_vehicle_open_max_s: float = _env_float("IAAI_BEFORE_VEHICLE_OPEN_MAX_S", 0.3)
after_vehicle_open_min_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MIN_S", 0.05)
after_vehicle_open_max_s: float = _env_float("IAAI_AFTER_VEHICLE_OPEN_MAX_S", 0.15)
between_vehicles_min_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MIN_S", 0.05)
between_vehicles_max_s: float = _env_float("IAAI_BETWEEN_VEHICLES_MAX_S", 0.15)
after_page_change_min_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MIN_S", 0.8)
after_page_change_max_s: float = _env_float("IAAI_AFTER_PAGE_CHANGE_MAX_S", 1.8)
# Конфиг сбора листинга (URL, лимиты страниц и машин)
@@ -103,11 +103,14 @@ class HumanPaceConfig:
@dataclass(slots=True)
class ListingConfig:
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 5)
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 100)
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 200)
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
# Порог раннего останова: если доля уже известных машин на странице >= этого значения,
# прекращаем листать — все новые машины уже найдены. 0 = отключено.
early_stop_threshold: float = _env_float("IAAI_EARLY_STOP_THRESHOLD", 0.8)
# - Конфиг PostgreSQL (URL, пул соединений, pool_recycle)
@@ -135,13 +138,16 @@ class RedisConfig:
class CeleryConfig:
broker_url: str = _env_str("CELERY_BROKER_URL", "")
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 600)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 900)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 1)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 20)
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
beat_sync_interval_minutes: int = _env_int("CELERY_BEAT_SYNC_INTERVAL_MINUTES", 60)
beat_sync_limit: int = _env_int("CELERY_BEAT_SYNC_LIMIT", 26)
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
# --- Конфиг прокси (server, username, password) ---
@@ -167,38 +173,22 @@ class ProxyConfig:
return result
# --- Конфиг сессии браузера (куки, storage_state) ---
@dataclass(slots=True)
class SessionConfig:
storage_state_path: str | None = _env_optional_str("IAAI_STORAGE_STATE_PATH")
max_age_days: int = _env_int("IAAI_SESSION_MAX_AGE_DAYS", 30)
save_on_exit: bool = _env_bool("IAAI_SESSION_SAVE_ON_EXIT", True)
login_email: str | None = _env_optional_str("IAAI_LOGIN_EMAIL")
login_password: str | None = _env_optional_str("IAAI_LOGIN_PASSWORD")
login_url: str = _env_str("IAAI_LOGIN_URL", "https://www.iaai.com/Login")
@property
def enabled(self) -> bool:
return bool(self.storage_state_path)
@property
def has_credentials(self) -> bool:
return bool(self.login_email and self.login_password)
# --- Главный объект настроек: собирает все блоки конфигурации ---
@dataclass(slots=True)
class Settings:
home_url: str = "https://www.iaai.com/"
default_timeout_ms: int = _env_int("IAAI_TIMEOUT_MS", 45000)
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 800)
network_settle_ms: int = _env_int("IAAI_NETWORK_SETTLE_MS", 400)
fast_path_timeout_ms: int = _env_int("IAAI_FAST_PATH_TIMEOUT_MS", 5000)
fast_path_max_attempts: int = _env_int("IAAI_FAST_PATH_MAX_ATTEMPTS", 1)
fallback_navigation_timeout_ms: int = _env_int("IAAI_FALLBACK_NAV_TIMEOUT_MS", 15000)
max_retries: int = _env_int("IAAI_MAX_RETRIES", 3)
retry_delay_seconds: float = _env_float("IAAI_RETRY_DELAY_SECONDS", 2.5)
retry_backoff_multiplier: float = _env_float("IAAI_RETRY_BACKOFF_MULTIPLIER", 2.0)
retry_jitter_seconds: float = _env_float("IAAI_RETRY_JITTER_SECONDS", 0.25)
headless: bool = _env_bool("IAAI_HEADLESS", True)
browser_engine: str = _env_str("IAAI_BROWSER_ENGINE", "auto")
log_level: str = _env_str("IAAI_LOG_LEVEL", "INFO")
log_file: str | None = _env_optional_str("IAAI_LOG_FILE")
enable_trace_id_logs: bool = _env_bool("IAAI_ENABLE_TRACE_ID_LOGS", True)
@@ -215,7 +205,14 @@ class Settings:
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)
proxy: ProxyConfig = field(default_factory=ProxyConfig)
session: SessionConfig = field(default_factory=SessionConfig)
@property
def parallel_tabs(self) -> int:
return self.celery.parallel_tabs
@property
def block_resources(self) -> bool:
return self.celery.block_resources
# Глобальный синглтон — используется по умолчанию во всех модулях.
settings = Settings()
iaai_scraper/core/exceptions.py
+3 -3
View File
@@ -1,10 +1,10 @@
class ScraperError(Exception):
"""Base scraper exception."""
"""Базовое исключение скрапера."""
class AntiBotDetectedError(ScraperError):
"""Raised when the target website appears to block automation."""
"""Вызывается, когда сайт блокирует автоматизацию."""
class SiteStructureChangedError(ScraperError):
"""Raised when the page shape changed and required data is missing."""
"""Вызывается, когда структура страницы изменилась и данных не хватает."""
iaai_scraper/core/logs.py
+1 -1
View File
@@ -2,7 +2,7 @@ import logging
import sys
from contextvars import ContextVar
# ContextVar хранит trace_id текущего потока/корутины.
# Храним trace_id текущего потока/корутины.
TRACE_ID: ContextVar[str] = ContextVar("trace_id", default="-")
iaai_scraper/core/utils.py
+35 -2
View File
@@ -1,7 +1,7 @@
import json
import re
from pathlib import Path
from typing import Any, Iterable
from typing import Any, Callable, Iterable
def save_to_json(data: Any, filename: str | Path) -> None:
@@ -17,7 +17,7 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
return None
# Regex для VIN, lot, price.
# Регулярные выражения для VIN, lot и price.
VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
LOT_RE = re.compile(r"\b(\d{7,10})\b")
PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
@@ -37,3 +37,36 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
for item in obj:
found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
return found
def deep_find_all_keys(
payloads: list,
field_map: dict[str, set[str]],
max_depth: int = 64,
) -> dict[str, list]:
"""Извлекает все нужные поля за один проход по JSON."""
# Готовим обратную карту: нормализованный ключ -> имя поля.
reverse: dict[str, str] = {}
for field_name, keys in field_map.items():
for k in keys:
reverse[k.lower()] = field_name
result: dict[str, list] = {f: [] for f in field_map}
def _recurse(obj: Any, depth: int) -> None:
if depth >= max_depth:
return
if isinstance(obj, dict):
for k, v in obj.items():
field = reverse.get(k.lower())
if field is not None:
result[field].append(v)
_recurse(v, depth + 1)
elif isinstance(obj, list):
for item in obj:
_recurse(item, depth + 1)
for payload in payloads:
_recurse(payload, 0)
return result