Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

iaai_scraper/scheduler_service.py

@@ -0,0 +1,69 @@
+import logging
+import time
+from datetime import datetime, timezone, timedelta
+
+from .core.config import settings
+from .discovery_service import DiscoveryService
+from .fetch_service import FetchService
+from .enrichment_service import EnrichmentService
+
+logger = logging.getLogger("iaai_scraper.scheduler_service")
+
+
+class SchedulerService:
+    """Сервис для планирования и координации ingestion pipeline."""
+
+    def __init__(self):
+        self.discovery = DiscoveryService()
+        self.fetch = FetchService()
+        self.enrichment = EnrichmentService()
+
+    def run_full_pipeline(self):
+        """Запускает полный цикл ingestion: discovery -> fetch -> enrichment."""
+        logger.info("Starting full ingestion pipeline")
+
+        try:
+            # 1. Discovery phase
+            logger.info("Phase 1: Discovery")
+            discovered_count = self.discovery.discover_new_vehicles()
+            logger.info(f"Discovered {discovered_count} new candidates")
+
+            # 2. Fetch phase
+            logger.info("Phase 2: Fetch")
+            fetched_count = self.fetch.process_pending_candidates(limit=50)
+            logger.info(f"Successfully fetched {fetched_count} candidates")
+
+            # 3. Enrichment phase
+            logger.info("Phase 3: Enrichment")
+            enriched_count = self.enrichment.process_unparsed_snapshots(limit=50)
+            logger.info(f"Successfully enriched {enriched_count} snapshots")
+
+            logger.info("Ingestion pipeline completed")
+
+        except Exception as e:
+            logger.error(f"Pipeline failed: {e}")
+            raise
+
+    def run_continuous_pipeline(self, interval_minutes: int = 30):
+        """Запускает непрерывный цикл ingestion с интервалом."""
+        logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals")
+
+        while True:
+            try:
+                self.run_full_pipeline()
+            except Exception as e:
+                logger.error(f"Pipeline iteration failed: {e}")
+
+            logger.info(f"Sleeping for {interval_minutes} minutes")
+            time.sleep(interval_minutes * 60)
+
+    def run_targeted_enrichment(self):
+        """Запускает только enrichment для существующих snapshots."""
+        logger.info("Running targeted enrichment")
+        enriched_count = self.enrichment.process_unparsed_snapshots(limit=100)
+        logger.info(f"Enriched {enriched_count} snapshots")
+
+    def cleanup_old_data(self, days_to_keep: int = 30):
+        """Очищает старые данные (опционально)."""
+        # TODO: implement if needed
+        pass