From 47ee4311c7a2f353cc538d9c0fba747ce0f98657 Mon Sep 17 00:00:00 2001 From: qananasikq Date: Mon, 4 May 2026 19:02:40 +0300 Subject: [PATCH] Remove flaresolverr and optimize refresh upserts --- docker-compose.yml | 23 ----------------------- mobilede_scraper/mobile_de/mapper.py | 2 ++ mobilede_scraper/storage/db.py | 9 ++++++++- mobilede_scraper/storage/enums.py | 2 +- 4 files changed, 11 insertions(+), 25 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index daf7c6e..e9499dd 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -52,12 +52,6 @@ MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED: ${MOBILEDE_BOOTSTRAP_FULL_SCAN_ENABLED:-true} MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP: ${MOBILEDE_INCREMENTAL_AFTER_BOOTSTRAP:-true} MOBILEDE_INCREMENTAL_PAGE_WINDOW: ${MOBILEDE_INCREMENTAL_PAGE_WINDOW:-1} - MOBILEDE_FLARESOLVERR_ENABLED: ${MOBILEDE_FLARESOLVERR_ENABLED:-false} - MOBILEDE_FLARESOLVERR_URL: ${MOBILEDE_FLARESOLVERR_URL:-http://flaresolverr:8191/v1} - MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS: ${MOBILEDE_FLARESOLVERR_TIMEOUT_SECONDS:-120} - MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS: ${MOBILEDE_FLARESOLVERR_MAX_TIMEOUT_MS:-60000} - MOBILEDE_FLARESOLVERR_SESSION: ${MOBILEDE_FLARESOLVERR_SESSION:-} - MOBILEDE_FLARESOLVERR_STATUSES: ${MOBILEDE_FLARESOLVERR_STATUSES:-403,429,503} IAAI_STARTUP_SYNC_ENABLED: ${IAAI_STARTUP_SYNC_ENABLED:-true} IAAI_INTER_BATCH_DELAY_SECONDS: ${IAAI_INTER_BATCH_DELAY_SECONDS:-0} IAAI_FAIL_RATE_THRESHOLD: ${IAAI_FAIL_RATE_THRESHOLD:-0.9} @@ -150,23 +144,6 @@ services: max-size: "10m" max-file: "5" - flaresolverr: - image: ghcr.io/flaresolverr/flaresolverr:latest - container_name: mobilede-flaresolverr - restart: unless-stopped - environment: - LOG_LEVEL: ${FLARESOLVERR_LOG_LEVEL:-info} - LOG_HTML: ${FLARESOLVERR_LOG_HTML:-false} - CAPTCHA_SOLVER: ${FLARESOLVERR_CAPTCHA_SOLVER:-none} - TZ: ${TZ:-UTC} - ports: - - "127.0.0.1:${MOBILEDE_HOST_FLARESOLVERR_PORT:-8191}:8191" - logging: - driver: json-file - options: - max-size: "10m" - max-file: "5" - # Миграции. migrate: <<: *app-service diff --git a/mobilede_scraper/mobile_de/mapper.py b/mobilede_scraper/mobile_de/mapper.py index b8a2a6f..93a0855 100644 --- a/mobilede_scraper/mobile_de/mapper.py +++ b/mobilede_scraper/mobile_de/mapper.py @@ -248,6 +248,8 @@ class MobileDeMapper: return "US" if text in {"CA", "CANADA"}: return "CA" + if text in {"IT", "ITALY", "ITALIA"}: + return "IT" if text in {"JP", "JAPAN"}: return "JP" if text in {"KR", "KOREA", "SOUTH KOREA"}: diff --git a/mobilede_scraper/storage/db.py b/mobilede_scraper/storage/db.py index b8bb009..926b97a 100644 --- a/mobilede_scraper/storage/db.py +++ b/mobilede_scraper/storage/db.py @@ -1,4 +1,5 @@ import logging +import os from contextlib import contextmanager from datetime import datetime, timezone from typing import Any, Iterator @@ -12,6 +13,7 @@ from .models import Base, Car, Image, SyncRun from .schemas import CarRecord logger = logging.getLogger("MOBILEDE_scraper.db") +MOBILEDE_SKIP_IMAGES_FOR_UPDATED = os.getenv("MOBILEDE_SKIP_IMAGES_FOR_UPDATED", "1").strip().lower() in {"1", "true", "yes", "on"} CAR_DB_FIELDS = { @@ -264,17 +266,19 @@ class PersistenceService: images = [image.model_dump(mode="python") for image in record.images] car_by_id = existing_by_id.get(record.origin_id) car_by_url = existing_by_url.get(record.origin_url) - entry: dict[str, object] = {"record": record, "images": images, "car_id": None} + entry: dict[str, object] = {"record": record, "images": images, "car_id": None, "action": "inserted"} if car_by_url is not None and car_by_url.origin_id != record.origin_id and car_by_id is None: for key, value in payload.items(): setattr(car_by_url, key, value) car_by_url.last_seen_at = record.last_seen_at entry["car_id"] = int(car_by_url.id) + entry["action"] = "updated" updated += 1 else: upsert_payloads.append(payload) if car_by_id is not None: + entry["action"] = "updated" updated += 1 else: inserted += 1 @@ -306,7 +310,10 @@ class PersistenceService: for entry in entries: car_id = int(entry["car_id"]) + action = str(entry.get("action") or "updated") images = entry["images"] + if MOBILEDE_SKIP_IMAGES_FOR_UPDATED and action == "updated": + continue new_image_urls = { str(img.get("fullres_image", "")) for img in images diff --git a/mobilede_scraper/storage/enums.py b/mobilede_scraper/storage/enums.py index c7a1af9..c0cece2 100644 --- a/mobilede_scraper/storage/enums.py +++ b/mobilede_scraper/storage/enums.py @@ -16,7 +16,7 @@ BODY_TYPE_ENUM_VALUES = ( "OTHER", "NA", ) -COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "NA") +COUNTRY_ENUM_VALUES = ("JP", "KR", "US", "CA", "DE", "IT", "NA") ORIGIN_ENUM_VALUES = ( "MOBILEDE", "MOBILE_DE",