105 lines
4.7 KiB
Python
105 lines
4.7 KiB
Python
import argparse
|
|
from pathlib import Path
|
|
|
|
from .core.utils import save_to_json
|
|
from .scraper import IAAIScraper
|
|
|
|
|
|
def build_parser() -> argparse.ArgumentParser:
|
|
parser = argparse.ArgumentParser(description="Public IAAI scraper")
|
|
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
|
|
init_db_parser = subparsers.add_parser("init-db", help="Create local DB tables")
|
|
init_db_parser.add_argument("--output", default="iaai_db_init.json", help="Path to output JSON")
|
|
|
|
listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from Vehiclelisting/Cars")
|
|
listing_parser.add_argument("--make", default=None, help="Optional make filter")
|
|
listing_parser.add_argument("--model", default=None, help="Optional model filter")
|
|
listing_parser.add_argument("--output", default="iaai_listing_links.json", help="Path to output JSON")
|
|
|
|
open_parser = subparsers.add_parser(
|
|
"open-vehicle",
|
|
help="Open a vehicle page in gentle mode and save only DOM-based hints",
|
|
)
|
|
open_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
|
|
open_parser.add_argument("--output", default="iaai_vehicle_opened.json", help="Path to output JSON")
|
|
|
|
scrape_parser = subparsers.add_parser(
|
|
"scrape-vehicle",
|
|
help="Open a vehicle page and capture a limited set of likely useful JSON responses",
|
|
)
|
|
scrape_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
|
|
scrape_parser.add_argument("--output", default="iaai_vehicle_detail.json", help="Path to output JSON")
|
|
|
|
export_parser = subparsers.add_parser(
|
|
"export-db-json",
|
|
help="Scrape a vehicle page and save only the DB-ready car record JSON",
|
|
)
|
|
export_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
|
|
export_parser.add_argument("--output", default="iaai_vehicle_db_record.json", help="Path to output JSON")
|
|
|
|
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape one vehicle and upsert it into the DB")
|
|
sync_vehicle_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
|
|
sync_vehicle_parser.add_argument("--lane", default="iaai", help="Logical lane name for sync_runs")
|
|
sync_vehicle_parser.add_argument("--output", default="iaai_sync_vehicle.json", help="Path to output JSON")
|
|
|
|
sync_listing_parser = subparsers.add_parser(
|
|
"sync-listing",
|
|
help="Collect vehicle URLs from the Cars listing and sync them sequentially",
|
|
)
|
|
sync_listing_parser.add_argument("--make", default=None, help="Optional make filter")
|
|
sync_listing_parser.add_argument("--model", default=None, help="Optional model filter")
|
|
sync_listing_parser.add_argument("--lane", default="iaai_cars", help="Logical lane name for sync_runs")
|
|
sync_listing_parser.add_argument("--output", default="iaai_sync_listing.json", help="Path to output JSON")
|
|
|
|
daemon_parser = subparsers.add_parser(
|
|
"run-daemon",
|
|
help="Run the scraper in a loop, syncing vehicles every N minutes (default 60)",
|
|
)
|
|
daemon_parser.add_argument(
|
|
"--interval", type=int, default=None,
|
|
help="Override interval in minutes (default from IAAI_SCHEDULER_INTERVAL_MINUTES or 60)",
|
|
)
|
|
|
|
return parser
|
|
|
|
|
|
def main() -> None:
|
|
parser = build_parser()
|
|
args = parser.parse_args()
|
|
|
|
if args.command == "run-daemon":
|
|
# daemon: бесконечный цикл
|
|
from .core.config import Settings
|
|
runtime_settings = Settings()
|
|
if args.interval is not None:
|
|
runtime_settings.scheduler_interval_minutes = args.interval
|
|
with IAAIScraper(runtime_settings) as scraper:
|
|
scraper.persistence.create_tables()
|
|
scraper.run_scheduled()
|
|
return
|
|
|
|
# одноразовый запуск
|
|
with IAAIScraper() as scraper:
|
|
if args.command == "init-db":
|
|
data = scraper.init_db()
|
|
elif args.command == "collect-listing":
|
|
data = scraper.collect_listing(make=args.make, model=args.model)
|
|
elif args.command == "open-vehicle":
|
|
data = scraper.open_vehicle_page(args.vehicle_url)
|
|
elif args.command == "scrape-vehicle":
|
|
data = scraper.scrape_vehicle_detail(args.vehicle_url)
|
|
elif args.command == "export-db-json":
|
|
data = scraper.scrape_vehicle_detail(args.vehicle_url).get("db_record", {})
|
|
elif args.command == "sync-vehicle":
|
|
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
|
|
else:
|
|
data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane)
|
|
|
|
save_to_json(data, Path(args.output))
|
|
print(f"Saved result to {Path(args.output).resolve()}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|