69 lines
2.7 KiB
Python
69 lines
2.7 KiB
Python
import logging
|
||
import time
|
||
from datetime import datetime, timezone, timedelta
|
||
|
||
from .core.config import settings
|
||
from .discovery_service import DiscoveryService
|
||
from .fetch_service import FetchService
|
||
from .enrichment_service import EnrichmentService
|
||
|
||
logger = logging.getLogger("iaai_scraper.scheduler_service")
|
||
|
||
|
||
class SchedulerService:
|
||
"""Сервис для планирования и координации ingestion pipeline."""
|
||
|
||
def __init__(self):
|
||
self.discovery = DiscoveryService()
|
||
self.fetch = FetchService()
|
||
self.enrichment = EnrichmentService()
|
||
|
||
def run_full_pipeline(self):
|
||
"""Запускает полный цикл ingestion: discovery -> fetch -> enrichment."""
|
||
logger.info("Starting full ingestion pipeline")
|
||
|
||
try:
|
||
# 1. Discovery phase
|
||
logger.info("Phase 1: Discovery")
|
||
discovered_count = self.discovery.discover_new_vehicles()
|
||
logger.info(f"Discovered {discovered_count} new candidates")
|
||
|
||
# 2. Fetch phase
|
||
logger.info("Phase 2: Fetch")
|
||
fetched_count = self.fetch.process_pending_candidates(limit=50)
|
||
logger.info(f"Successfully fetched {fetched_count} candidates")
|
||
|
||
# 3. Enrichment phase
|
||
logger.info("Phase 3: Enrichment")
|
||
enriched_count = self.enrichment.process_unparsed_snapshots(limit=50)
|
||
logger.info(f"Successfully enriched {enriched_count} snapshots")
|
||
|
||
logger.info("Ingestion pipeline completed")
|
||
|
||
except Exception as e:
|
||
logger.error(f"Pipeline failed: {e}")
|
||
raise
|
||
|
||
def run_continuous_pipeline(self, interval_minutes: int = 30):
|
||
"""Запускает непрерывный цикл ingestion с интервалом."""
|
||
logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals")
|
||
|
||
while True:
|
||
try:
|
||
self.run_full_pipeline()
|
||
except Exception as e:
|
||
logger.error(f"Pipeline iteration failed: {e}")
|
||
|
||
logger.info(f"Sleeping for {interval_minutes} minutes")
|
||
time.sleep(interval_minutes * 60)
|
||
|
||
def run_targeted_enrichment(self):
|
||
"""Запускает только enrichment для существующих snapshots."""
|
||
logger.info("Running targeted enrichment")
|
||
enriched_count = self.enrichment.process_unparsed_snapshots(limit=100)
|
||
logger.info(f"Enriched {enriched_count} snapshots")
|
||
|
||
def cleanup_old_data(self, days_to_keep: int = 30):
|
||
"""Очищает старые данные (опционально)."""
|
||
# TODO: implement if needed
|
||
pass |