qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: harden sync flow, proxy bridge, and data normalization

Browse Source
This commit is contained in:
qananasikq
2026-04-08 18:08:16 +03:00
parent 447a88feed
commit 6821404d79
17 changed files with 683 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
iaai_scraper/cli.py
View File

@@ -13,39 +13,41 @@ def build_parser() -> argparse.ArgumentParser:
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
subparsers = parser.add_subparsers(dest="command", required=True)
default_output_dir = Path("artifacts/json")
init_db_parser = subparsers.add_parser("init-db", help="Create local DB tables")
init_db_parser.add_argument("--output", default="iaai_db_init.json", help="Path to output JSON")
init_db_parser.add_argument("--output", default=str(default_output_dir / "iaai_db_init.json"), help="Path to output JSON")
listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from Vehiclelisting/Cars")
listing_parser.add_argument("--make", default=None, help="Optional make filter")
listing_parser.add_argument("--model", default=None, help="Optional model filter")
listing_parser.add_argument("--output", default="iaai_listing_links.json", help="Path to output JSON")
listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"), help="Path to output JSON")
open_parser = subparsers.add_parser(
"open-vehicle",
help="Open a vehicle page in gentle mode and save only DOM-based hints",
)
open_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
open_parser.add_argument("--output", default="iaai_vehicle_opened.json", help="Path to output JSON")
open_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_opened.json"), help="Path to output JSON")
scrape_parser = subparsers.add_parser(
"scrape-vehicle",
help="Open a vehicle page and capture a limited set of likely useful JSON responses",
)
scrape_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
scrape_parser.add_argument("--output", default="iaai_vehicle_detail.json", help="Path to output JSON")
scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"), help="Path to output JSON")
export_parser = subparsers.add_parser(
"export-db-json",
help="Scrape a vehicle page and save only the DB-ready car record JSON",
)
export_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
export_parser.add_argument("--output", default="iaai_vehicle_db_record.json", help="Path to output JSON")
export_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_db_record.json"), help="Path to output JSON")
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape one vehicle and upsert it into the DB")
sync_vehicle_parser.add_argument("vehicle_url", help="IAAI vehicle detail URL")
sync_vehicle_parser.add_argument("--lane", default="iaai", help="Logical lane name for sync_runs")
sync_vehicle_parser.add_argument("--output", default="iaai_sync_vehicle.json", help="Path to output JSON")
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"), help="Path to output JSON")
sync_listing_parser = subparsers.add_parser(
"sync-listing",
@@ -55,7 +57,13 @@ def build_parser() -> argparse.ArgumentParser:
sync_listing_parser.add_argument("--model", default=None, help="Optional model filter")
sync_listing_parser.add_argument("--lane", default="iaai_cars", help="Logical lane name for sync_runs")
sync_listing_parser.add_argument("--limit", type=int, default=None, help="Limit number of vehicles to sync")
sync_listing_parser.add_argument("--output", default="iaai_sync_listing.json", help="Path to output JSON")
sync_listing_parser.add_argument(
"--only-new",
choices=["true", "false"],
default=None,
help="Process only new vehicles (default from IAAI_SYNC_ONLY_NEW)",
)
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"), help="Path to output JSON")
daemon_parser = subparsers.add_parser(
"run-daemon",
@@ -108,7 +116,14 @@ def main() -> None:
elif args.command == "sync-vehicle":
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
else:
data = scraper.sync_listing(make=args.make, model=args.model, lane=args.lane, limit=args.limit)
only_new = None if args.only_new is None else args.only_new == "true"
data = scraper.sync_listing(
make=args.make,
model=args.model,
lane=args.lane,
limit=args.limit,
only_new=only_new,
)
save_to_json(data, Path(args.output))
print(f"Saved result to {Path(args.output).resolve()}")