Remove flaresolverr and optimize refresh upserts

docker-compose.yml

@@ -52,12 +52,6 @@
   MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true}
   MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true}
   MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1}
-  MOBILEDE_FLARESOLVERR_ENABLED: ${MOBILEDE_FLARESOLVERR_ENABLED:-false}
-  MOBILEDE_FLARESOLVERR_URL: ${MOBILEDE_FLARESOLVERR_URL:-http://flaresolverr:8191/v1}
-  MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS: ${MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS:-120}
-  MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS: ${MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS:-60000}
-  MOBILEDE_FLARESOLVERR_SESSION: ${MOBILEDE_FLARESOLVERR_SESSION:-}
-  MOBILEDE_FLARESOLVERR_STATUSES: ${MOBILEDE_FLARESOLVERR_STATUSES:-403,429,503}
   IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true}
   IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0}
   IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9}
@@ -150,23 +144,6 @@ services:
         max-size: "10m"
         max-file: "5"
 
-  flaresolverr:
-    image: ghcr.io/flaresolverr/flaresolverr:latest
-    container_name: mobilede-flaresolverr
-    restart: unless-stopped
-    environment:
-      LOG_LEVEL: ${FLARESOLVERR_LOG_LEVEL:-info}
-      LOG_HTML: ${FLARESOLVERR_LOG_HTML:-false}
-      CAPTCHA_SOLVER: ${FLARESOLVERR_CAPTCHA_SOLVER:-none}
-      TZ: ${TZ:-UTC}
-    ports:
-      - "127.0.0.1:${MOBILEDE_HOST_FLARESOLVERR_PORT:-8191}:8191"
-    logging:
-      driver: json-file
-      options:
-        max-size: "10m"
-        max-file: "5"
-
   # Миграции.
   migrate:
     <<: *app-service

mobilede_scraper/mobile_de/mapper.py

@@ -248,6 +248,8 @@ class MobileDeMapper:
             return "US"
         if text in {"CA", "CANADA"}:
             return "CA"
+        if text in {"IT", "ITALY", "ITALIA"}:
+            return "IT"
         if text in {"JP", "JAPAN"}:
             return "JP"
         if text in {"KR", "KOREA", "SOUTH KOREA"}:

mobilede_scraper/storage/db.py

@@ -1,4 +1,5 @@
 import logging
+import os
 from contextlib import contextmanager
 from datetime import datetime, timezone
 from typing import Any, Iterator
@@ -12,6 +13,7 @@ from .models import Base, Car, Image, SyncRun
 from .schemas import CarRecord
 
 logger = logging.getLogger("MOBILEDE_scraper.db")
+MOBILEDE_SKIP_IMAGES_FOR_UPDATED = os.getenv("MOBILEDE_SKIP_IMAGES_FOR_UPDATED", "1").strip().lower() in {"1", "true", "yes", "on"}
 
 
 CAR_DB_FIELDS = {
@@ -264,17 +266,19 @@ class PersistenceService:
 				images = [image.model_dump(mode="python") for image in record.images]
 				car_by_id = existing_by_id.get(record.origin_id)
 				car_by_url = existing_by_url.get(record.origin_url)
-				entry: dict[str, object] = {"record": record, "images": images, "car_id": None}
+				entry: dict[str, object] = {"record": record, "images": images, "car_id": None, "action": "inserted"}
 
 				if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None:
 					for key, value in payload.items():
 						setattr(car_by_url, key, value)
 					car_by_url.last_seen_at = record.last_seen_at
 					entry["car_id"] = int(car_by_url.id)
+					entry["action"] = "updated"
 					updated += 1
 				else:
 					upsert_payloads.append(payload)
 					if car_by_id is not None:
+						entry["action"] = "updated"
 						updated += 1
 					else:
 						inserted += 1
@@ -306,7 +310,10 @@ class PersistenceService:
 
 			for entry in entries:
 				car_id = int(entry["car_id"])
+				action = str(entry.get("action") or "updated")
 				images = entry["images"]
+				if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and action == "updated":
+					continue
 				new_image_urls = {
 					str(img.get("fullres_image", ""))
 					for img in images

mobilede_scraper/storage/enums.py

@@ -16,7 +16,7 @@ BODY_TYPE_ENUM_VALUES = (
 	"OTHER",
 	"NA",
 )
-COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "NA")
+COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "IT", "NA")
 ORIGIN_ENUM_VALUES = (
 	"MOBILEDE",
 	"MOBILE_DE",