Remove flaresolverr and optimize refresh upserts

docker-compose.yml

@@ -52,12 +52,6 @@
   MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
   MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true}
   MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1}
-  MOBILEDE_FLARESOLVERR_ENABLED: ${MOBILEDE_FLARESOLVERR_ENABLED:-false}
-  MOBILEDE_FLARESOLVERR_URL: ${MOBILEDE_FLARESOLVERR_URL:-http://flaresolverr:8191/v1}
-  MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS: ${MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS:-120}
-  MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS: ${MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS:-60000}
-  MOBILEDE_FLARESOLVERR_SESSION: ${MOBILEDE_FLARESOLVERR_SESSION:-}
-  MOBILEDE_FLARESOLVERR_STATUSES: ${MOBILEDE_FLARESOLVERR_STATUSES:-403,429,503}
   IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
   IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
   IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
@@ -150,23 +144,6 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  flaresolverr:
-    image: ghcr.io/flaresolverr/flaresolverr:latest
-    container_name: mobilede-flaresolverr
-    restart: unless-stopped
-    environment:
-      LOG_LEVEL: ${FLARESOLVERR_LOG_LEVEL:-info}
-      LOG_HTML: ${FLARESOLVERR_LOG_HTML:-false}
-      CAPTCHA_SOLVER: ${FLARESOLVERR_CAPTCHA_SOLVER:-none}
-      TZ: ${TZ:-UTC}
-    ports:
-      - "127.0.0.1:${MOBILEDE_HOST_FLARESOLVERR_PORT:-8191}:8191"
-    logging:
-      driver: json-file
-      options:
-        max-size: "10m"
-        max-file: "5"
-
   # Миграции.
   migrate:
     <<: *app-service

mobilede_scraper/core/exceptions.py

@@ -1,14 +0,0 @@
-class ScraperError(Exception):
-    """Базовое исключение скрапера."""
-
-
-class AntiBotDetectedError(ScraperError):
-    """Вызывается, когда сайт блокирует автоматизацию."""
-
-
-class SiteStructureChangedError(ScraperError):
-    """Вызывается, когда структура страницы изменилась и данных не хватает."""
-
-
-class ListingResumeError(ScraperError):
-    """Вызывается, когда resume по checkpoint больше недостижим."""

mobilede_scraper/core/retry.py

@@ -1,53 +0,0 @@
-import logging
-import random
-import time
-from collections.abc import Callable
-from functools import wraps
-from typing import Any
-
-from playwright.sync_api import Error, TimeoutError as PlaywrightTimeoutError
-
-from .exceptions import AntiBotDetectedError
-
-logger = logging.getLogger("MOBILEDE_scraper.retry")
-
-# Типы исключений, при которых retry имеет смысл.
-RETRYABLE_EXCEPTIONS = (
-    PlaywrightTimeoutError,
-    Error,
-    ConnectionError,
-    OSError,
-    TimeoutError,
-    AntiBotDetectedError,
-)
-
-
-def retryable(
-    max_attempts: int,
-    delay_seconds: float = 2.5,
-    backoff_multiplier: float = 2.0,
-    jitter_seconds: float = 0.0,
-) -> Callable[[Callable[..., Any]], Callable[..., Any]]:
-    def decorator(func: Callable[..., Any]) -> Callable[..., Any]:
-        @wraps(func)
-        def wrapper(*args: Any, **kwargs: Any) -> Any:
-            last_error: Exception | None = None
-            for attempt in range(1, max_attempts + 1):
-                try:
-                    return func(*args, **kwargs)
-                except RETRYABLE_EXCEPTIONS as exc:
-                    last_error = exc
-                    logger.warning("%s failed on attempt %s/%s: %s", func.__name__, attempt, max_attempts, exc)
-                    if attempt < max_attempts:
-                        sleep_for = delay_seconds * (backoff_multiplier ** (attempt - 1))
-                        if jitter_seconds > 0:
-                            sleep_for += random.uniform(0, jitter_seconds)
-                        logger.debug("Retrying %s in %.2fs", func.__name__, sleep_for)
-                        time.sleep(sleep_for)
-            if last_error is not None:
-                raise last_error
-            raise RuntimeError("Retry wrapper failed without a captured exception")
-
-        return wrapper
-
-    return decorator

mobilede_scraper/core/utils.py

@@ -1,7 +1,6 @@
-import json
-import re
+import json
 from pathlib import Path
-from typing import Any, Callable, Iterable
+from typing import Any
 
 
 def save_to_json(data: Any, filename: str | Path) -> None:
@@ -10,21 +9,7 @@ def save_to_json(data: Any, filename: str | Path) -> None:
     path.write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
 
 
-def first_non_empty(values: Iterable[Any]) -> Any | None:
-    for value in values:
-        if value not in (None, "", [], {}, ()):
-            return value
-    return None
-
-
-# Регулярные выражения для VIN, lot и price.
-VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
-LOT_RE = re.compile(r"\b(\d{7,10})\b")
-PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
-
-
 def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
-    # Рекурсивно ищет значения по набору ключей в произвольном JSON-дереве.
     found = []
     if _depth >= max_depth:
         return found
@@ -37,36 +22,3 @@ def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int =
         for item in obj:
             found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
     return found
-
-
-def deep_find_all_keys(
-    payloads: list,
-    field_map: dict[str, set[str]],
-    max_depth: int = 64,
-) -> dict[str, list]:
-    """Извлекает все нужные поля за один проход по JSON."""
-    # Готовим обратную карту: нормализованный ключ -> имя поля.
-    reverse: dict[str, str] = {}
-    for field_name, keys in field_map.items():
-        for k in keys:
-            reverse[k.lower()] = field_name
-
-    result: dict[str, list] = {f: [] for f in field_map}
-
-    def _recurse(obj: Any, depth: int) -> None:
-        if depth >= max_depth:
-            return
-        if isinstance(obj, dict):
-            for k, v in obj.items():
-                field = reverse.get(k.lower())
-                if field is not None:
-                    result[field].append(v)
-                _recurse(v, depth + 1)
-        elif isinstance(obj, list):
-            for item in obj:
-                _recurse(item, depth + 1)
-
-    for payload in payloads:
-        _recurse(payload, 0)
-
-    return result

mobilede_scraper/mobile_de/mapper.py

@@ -248,6 +248,8 @@ class MobileDeMapper:
             return "US"
         if text in {"CA", "CANADA"}:
             return "CA"
+        if text in {"IT", "ITALY", "ITALIA"}:
+            return "IT"
         if text in {"JP", "JAPAN"}:
             return "JP"
         if text in {"KR", "KOREA", "SOUTH KOREA"}:

mobilede_scraper/storage/db.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from contextlib import contextmanager
 from datetime import datetime, timezone
 from typing import Any, Iterator
@@ -12,6 +13,7 @@ from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord
 
 logger = logging.getLogger("MOBILEDE_scraper.db")
+MOBILEDE_SKIP_IMAGES_FOR_UPDATED = os.getenv("MOBILEDE_SKIP_IMAGES_FOR_UPDATED", "1").strip().lower() in {"1", "true", "yes", "on"}
 
 
 CAR_DB_FIELDS = {
@@ -264,17 +266,19 @@ class PersistenceService:
 				images = [image.model_dump(mode="python") for image in record.images]
 				car_by_id = existing_by_id.get(record.origin_id)
 				car_by_url = existing_by_url.get(record.origin_url)
-				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+				entry: dict[str, object] = {"record": record, "images": images, "car_id": None, "action": "inserted"}
 
 				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
 					for key, value in payload.items():
 						setattr(car_by_url, key, value)
 					car_by_url.last_seen_at = record.last_seen_at
 					entry["car_id"] = int(car_by_url.id)
+					entry["action"] = "updated"
 					updated += 1
 				else:
 					upsert_payloads.append(payload)
 					if car_by_id is not None:
+						entry["action"] = "updated"
 						updated += 1
 					else:
 						inserted += 1
@@ -306,7 +310,10 @@ class PersistenceService:
 
 			for entry in entries:
 				car_id = int(entry["car_id"])
+				action = str(entry.get("action") or "updated")
 				images = entry["images"]
+				if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and action == "updated":
+					continue
 				new_image_urls = {
 					str(img.get("fullres_image", ""))
 					for img in images

mobilede_scraper/storage/enums.py

@@ -16,7 +16,7 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
-COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "NA")
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "IT", "NA")
 ORIGIN_ENUM_VALUES = (
 	"MOBILEDE",
 	"MOBILE_DE",