127 lines
5.5 KiB
Python
127 lines
5.5 KiB
Python
import argparse
|
|
from pathlib import Path
|
|
|
|
from .core.config import Settings
|
|
from .core.utils import save_to_json
|
|
from .scraper import IAAIScraper
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(description="IAAI scraper CLI")
|
|
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
|
|
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
default_output_dir = Path("artifacts/json")
|
|
|
|
subparsers.add_parser("init-db", help="Create DB tables")
|
|
|
|
listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
|
|
listing_parser.add_argument("--make", default=None)
|
|
listing_parser.add_argument("--model", default=None)
|
|
listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
|
|
|
|
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
|
|
scrape_parser.add_argument("vehicle_url")
|
|
scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
|
|
|
|
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
|
|
sync_vehicle_parser.add_argument("vehicle_url")
|
|
sync_vehicle_parser.add_argument("--lane", default="iaai")
|
|
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
|
|
|
|
sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
|
|
sync_listing_parser.add_argument("--make", default=None)
|
|
sync_listing_parser.add_argument("--model", default=None)
|
|
sync_listing_parser.add_argument("--lane", default="iaai_cars")
|
|
sync_listing_parser.add_argument("--limit", type=int, default=None)
|
|
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
|
|
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
|
|
|
|
# Новые команды для ingestion pipeline
|
|
subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap")
|
|
|
|
fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates")
|
|
fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process")
|
|
|
|
enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots")
|
|
enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process")
|
|
|
|
subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)")
|
|
|
|
return parser
|
|
|
|
|
|
def main() -> None:
|
|
parser = build_parser()
|
|
args = parser.parse_args()
|
|
|
|
runtime_settings: Settings | None = None
|
|
if args.headless is not None or args.debug:
|
|
runtime_settings = Settings()
|
|
if args.headless is not None:
|
|
runtime_settings.headless = args.headless == "true"
|
|
if args.debug:
|
|
runtime_settings.log_level = "DEBUG"
|
|
|
|
with IAAIScraper(runtime_settings) as scraper:
|
|
if args.command == "init-db":
|
|
data = scraper.init_db()
|
|
print(f"DB initialized: {data}")
|
|
return
|
|
elif args.command == "collect-listing":
|
|
data = scraper.collect_listing(make=args.make, model=args.model)
|
|
elif args.command == "scrape-vehicle":
|
|
data = scraper.scrape_vehicle_detail(args.vehicle_url)
|
|
elif args.command == "sync-vehicle":
|
|
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
|
|
elif args.command == "sync-listing":
|
|
only_new = None if args.only_new is None else args.only_new == "true"
|
|
data = scraper.sync_listing(
|
|
make=args.make,
|
|
model=args.model,
|
|
lane=args.lane,
|
|
limit=args.limit,
|
|
only_new=only_new,
|
|
)
|
|
elif args.command == "discover-vehicles":
|
|
from .discovery_service import DiscoveryService
|
|
discovery = DiscoveryService()
|
|
count = discovery.discover_new_vehicles()
|
|
print(f"Discovered {count} new vehicle candidates")
|
|
return
|
|
elif args.command == "fetch-pending":
|
|
from .fetch_service import FetchService
|
|
fetch = FetchService()
|
|
count = fetch.process_pending_candidates(limit=args.limit)
|
|
print(f"Successfully fetched {count} candidates")
|
|
return
|
|
elif args.command == "enrich-snapshots":
|
|
from .enrichment_service import EnrichmentService
|
|
enrichment = EnrichmentService()
|
|
count = enrichment.process_unparsed_snapshots(limit=args.limit)
|
|
print(f"Successfully enriched {count} snapshots")
|
|
return
|
|
elif args.command == "run-pipeline":
|
|
from .scheduler_service import SchedulerService
|
|
scheduler = SchedulerService()
|
|
scheduler.run_full_pipeline()
|
|
print("Pipeline completed")
|
|
return
|
|
else:
|
|
only_new = None if args.only_new is None else args.only_new == "true"
|
|
data = scraper.sync_listing(
|
|
make=args.make,
|
|
model=args.model,
|
|
lane=args.lane,
|
|
limit=args.limit,
|
|
only_new=only_new,
|
|
)
|
|
|
|
save_to_json(data, Path(args.output))
|
|
print(f"Saved to {Path(args.output).resolve()}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|