119 lines
4.8 KiB
Python
119 lines
4.8 KiB
Python
import logging
|
||
from datetime import datetime, timezone
|
||
from typing import Optional
|
||
|
||
from sqlalchemy.orm import Session
|
||
|
||
from .core.config import settings
|
||
from .scraper import IAAIScraper
|
||
from .storage.db import get_db_session
|
||
from .storage.models import VehicleCandidate, VehicleRawSnapshot
|
||
|
||
logger = logging.getLogger("iaai_scraper.fetch_service")
|
||
|
||
|
||
class FetchService:
|
||
"""Сервис для захвата сырых данных автомобилей (HTTP/browser fallback)."""
|
||
|
||
def __init__(self):
|
||
self.db: Session = get_db_session()
|
||
self.scraper = IAAIScraper()
|
||
|
||
def fetch_vehicle_data(self, candidate: VehicleCandidate) -> Optional[VehicleRawSnapshot]:
|
||
"""
|
||
Захватывает данные для кандидата автомобиля.
|
||
|
||
Args:
|
||
candidate: Кандидат для обработки
|
||
|
||
Returns:
|
||
VehicleRawSnapshot если захват успешен, None если неудача
|
||
"""
|
||
logger.info(f"Fetching data for candidate {candidate.id}: {candidate.url}")
|
||
|
||
try:
|
||
# Сначала пытаемся HTTP fast-path
|
||
raw_data = self._try_http_capture(candidate.url)
|
||
if raw_data:
|
||
return self._save_snapshot(candidate.id, "http", True, raw_data)
|
||
|
||
# Если HTTP не сработал, используем browser fallback
|
||
logger.info(f"HTTP failed for {candidate.url}, trying browser fallback")
|
||
raw_data = self._try_browser_capture(candidate.url)
|
||
if raw_data:
|
||
return self._save_snapshot(candidate.id, "browser", True, raw_data)
|
||
|
||
# Оба метода failed
|
||
logger.warning(f"Both HTTP and browser capture failed for {candidate.url}")
|
||
self._save_snapshot(candidate.id, "browser", False, None, "Both capture methods failed")
|
||
return None
|
||
|
||
except Exception as e:
|
||
error_msg = f"Unexpected error during capture: {str(e)}"
|
||
logger.error(f"Error fetching {candidate.url}: {error_msg}")
|
||
self._save_snapshot(candidate.id, "unknown", False, None, error_msg)
|
||
return None
|
||
|
||
def _try_http_capture(self, url: str) -> Optional[str]:
|
||
"""Пытается захватить данные через HTTP."""
|
||
try:
|
||
# Используем существующий метод из scraper
|
||
# Но нам нужно инициализировать scraper с сессией
|
||
# Пока заглушка - интегрируем позже
|
||
# Для теста вернем None, чтобы использовать browser
|
||
return None
|
||
except Exception as e:
|
||
logger.debug(f"HTTP capture failed for {url}: {e}")
|
||
return None
|
||
|
||
def _try_browser_capture(self, url: str) -> Optional[str]:
|
||
"""Пытается захватить данные через browser."""
|
||
try:
|
||
# Используем scrape_vehicle_detail из scraper
|
||
with IAAIScraper() as scraper:
|
||
result = scraper.scrape_vehicle_detail(url)
|
||
# Возвращаем HTML или JSON данные
|
||
return result.get("raw_html") or json.dumps(result)
|
||
except Exception as e:
|
||
logger.debug(f"Browser capture failed for {url}: {e}")
|
||
return None
|
||
|
||
def _save_snapshot(self, candidate_id: int, method: str, success: bool,
|
||
raw_data: Optional[str], error_message: Optional[str] = None) -> VehicleRawSnapshot:
|
||
"""Сохраняет snapshot в базу данных."""
|
||
snapshot = VehicleRawSnapshot(
|
||
candidate_id=candidate_id,
|
||
method=method,
|
||
success=success,
|
||
raw_data=raw_data,
|
||
error_message=error_message
|
||
)
|
||
self.db.add(snapshot)
|
||
self.db.commit()
|
||
self.db.refresh(snapshot)
|
||
return snapshot
|
||
|
||
def process_pending_candidates(self, limit: int = 10) -> int:
|
||
"""
|
||
Обрабатывает ожидающих кандидатов.
|
||
|
||
Returns:
|
||
Количество успешно обработанных кандидатов
|
||
"""
|
||
from .discovery_service import DiscoveryService
|
||
discovery = DiscoveryService()
|
||
|
||
candidates = discovery.get_pending_candidates(limit)
|
||
processed_count = 0
|
||
|
||
for candidate in candidates:
|
||
discovery.mark_candidate_processing(candidate.id)
|
||
|
||
snapshot = self.fetch_vehicle_data(candidate)
|
||
if snapshot and snapshot.success:
|
||
discovery.mark_candidate_processed(candidate.id)
|
||
processed_count += 1
|
||
else:
|
||
discovery.mark_candidate_failed(candidate.id)
|
||
|
||
return processed_count |