import logging import time from datetime import datetime, timezone, timedelta from .core.config import settings from .discovery_service import DiscoveryService from .fetch_service import FetchService from .enrichment_service import EnrichmentService logger = logging.getLogger("iaai_scraper.scheduler_service") class SchedulerService: """Сервис для планирования и координации ingestion pipeline.""" def __init__(self): self.discovery = DiscoveryService() self.fetch = FetchService() self.enrichment = EnrichmentService() def run_full_pipeline(self): """Запускает полный цикл ingestion: discovery -> fetch -> enrichment.""" logger.info("Starting full ingestion pipeline") try: # 1. Discovery phase logger.info("Phase 1: Discovery") discovered_count = self.discovery.discover_new_vehicles() logger.info(f"Discovered {discovered_count} new candidates") # 2. Fetch phase logger.info("Phase 2: Fetch") fetched_count = self.fetch.process_pending_candidates(limit=50) logger.info(f"Successfully fetched {fetched_count} candidates") # 3. Enrichment phase logger.info("Phase 3: Enrichment") enriched_count = self.enrichment.process_unparsed_snapshots(limit=50) logger.info(f"Successfully enriched {enriched_count} snapshots") logger.info("Ingestion pipeline completed") except Exception as e: logger.error(f"Pipeline failed: {e}") raise def run_continuous_pipeline(self, interval_minutes: int = 30): """Запускает непрерывный цикл ingestion с интервалом.""" logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals") while True: try: self.run_full_pipeline() except Exception as e: logger.error(f"Pipeline iteration failed: {e}") logger.info(f"Sleeping for {interval_minutes} minutes") time.sleep(interval_minutes * 60) def run_targeted_enrichment(self): """Запускает только enrichment для существующих snapshots.""" logger.info("Running targeted enrichment") enriched_count = self.enrichment.process_unparsed_snapshots(limit=100) logger.info(f"Enriched {enriched_count} snapshots") def cleanup_old_data(self, days_to_keep: int = 30): """Очищает старые данные (опционально).""" # TODO: implement if needed pass