Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

2026-04-20 16:27:42 +03:00
parent 55203d33ea
commit 65d5f8e1eb
23 changed files with 3436 additions and 518 deletions
--- a/iaai_scraper/discovery/init.py
+++ b/iaai_scraper/discovery/init.py
@@ -0,0 +1,17 @@
+from .sitemap import (
+    DEFAULT_SITEMAP_INDEX_URL,
+    SitemapDiscoveryError,
+    SitemapBlockedError,
+    SitemapDiscoveryResult,
+    discover_vehicle_urls_from_sitemap,
+    discover_vehicle_urls_from_sitemap_with_stats,
+)
+
+__all__ = [
+    "DEFAULT_SITEMAP_INDEX_URL",
+    "SitemapBlockedError",
+    "SitemapDiscoveryError",
+    "SitemapDiscoveryResult",
+    "discover_vehicle_urls_from_sitemap",
+    "discover_vehicle_urls_from_sitemap_with_stats",
+]