169 lines
8.2 KiB
Python
169 lines
8.2 KiB
Python
import argparse
|
||
from pathlib import Path
|
||
|
||
from .core.config import Settings
|
||
from .core.utils import save_to_json
|
||
from .mobile_de import MobileDeClient, MobileDeScraper
|
||
from .scraper import IAAIScraper
|
||
|
||
|
||
def build_parser() -> argparse.ArgumentParser:
|
||
parser = argparse.ArgumentParser(description="mobile.de scraper CLI")
|
||
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
|
||
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
|
||
subparsers = parser.add_subparsers(dest="command", required=True)
|
||
|
||
default_output_dir = Path("artifacts/json")
|
||
|
||
subparsers.add_parser("init-db", help="Create DB tables")
|
||
|
||
listing_parser = subparsers.add_parser("collect-listing", help="Deprecated IAAI command: collect vehicle URLs from listing page")
|
||
listing_parser.add_argument("--make", default=None)
|
||
listing_parser.add_argument("--model", default=None)
|
||
listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
|
||
|
||
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Deprecated IAAI command: scrape a vehicle detail page")
|
||
scrape_parser.add_argument("vehicle_url")
|
||
scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
|
||
|
||
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Deprecated IAAI command: scrape + upsert one vehicle")
|
||
sync_vehicle_parser.add_argument("vehicle_url")
|
||
sync_vehicle_parser.add_argument("--lane", default="iaai")
|
||
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
|
||
|
||
sync_listing_parser = subparsers.add_parser("sync-listing", help="Deprecated IAAI command: collect listing + sync all vehicles")
|
||
sync_listing_parser.add_argument("--make", default=None)
|
||
sync_listing_parser.add_argument("--model", default=None)
|
||
sync_listing_parser.add_argument("--lane", default="iaai_cars")
|
||
sync_listing_parser.add_argument("--limit", type=int, default=None)
|
||
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
|
||
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
|
||
|
||
mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages")
|
||
mobile_search_parser.add_argument("--page", type=int, default=1)
|
||
mobile_search_parser.add_argument("--max-pages", type=int, default=1)
|
||
mobile_search_parser.add_argument("--delay", type=float, default=0.7)
|
||
mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
|
||
mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500")
|
||
mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id")
|
||
mobile_search_parser.add_argument("--price-min", default=None)
|
||
mobile_search_parser.add_argument("--price-max", default=None)
|
||
mobile_search_parser.add_argument("--year-min", default=None)
|
||
mobile_search_parser.add_argument("--year-max", default=None)
|
||
mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json"))
|
||
|
||
mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page")
|
||
mobile_detail_parser.add_argument("listing_id")
|
||
mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json"))
|
||
|
||
mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages")
|
||
mobile_sync_search_parser.add_argument("--page", type=int, default=1)
|
||
mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1)
|
||
mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7)
|
||
mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars")
|
||
mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
|
||
mobile_sync_search_parser.add_argument("--make-id", default=None)
|
||
mobile_sync_search_parser.add_argument("--model-id", default=None)
|
||
mobile_sync_search_parser.add_argument("--price-min", default=None)
|
||
mobile_sync_search_parser.add_argument("--price-max", default=None)
|
||
mobile_sync_search_parser.add_argument("--year-min", default=None)
|
||
mobile_sync_search_parser.add_argument("--year-max", default=None)
|
||
mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json"))
|
||
|
||
mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page")
|
||
mobile_sync_detail_parser.add_argument("listing_id")
|
||
mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars")
|
||
mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json"))
|
||
|
||
return parser
|
||
|
||
|
||
def main() -> None:
|
||
parser = build_parser()
|
||
args = parser.parse_args()
|
||
|
||
runtime_settings: Settings | None = None
|
||
if args.headless is not None or args.debug:
|
||
runtime_settings = Settings()
|
||
if args.headless is not None:
|
||
runtime_settings.headless = args.headless == "true"
|
||
if args.debug:
|
||
runtime_settings.log_level = "DEBUG"
|
||
|
||
if args.command in {"search", "mobilede-search"}:
|
||
scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
|
||
data = scraper.collect_search(
|
||
start_page=args.page,
|
||
max_pages=args.max_pages,
|
||
search_url=args.search_url,
|
||
make_id=args.make_id,
|
||
model_id=args.model_id,
|
||
price_min=args.price_min,
|
||
price_max=args.price_max,
|
||
year_min=args.year_min,
|
||
year_max=args.year_max,
|
||
)
|
||
save_to_json(data, Path(args.output))
|
||
print(f"Saved to {Path(args.output).resolve()}")
|
||
return
|
||
|
||
if args.command in {"detail", "mobilede-detail"}:
|
||
scraper = MobileDeScraper()
|
||
data = scraper.collect_detail(args.listing_id)
|
||
save_to_json(data, Path(args.output))
|
||
print(f"Saved to {Path(args.output).resolve()}")
|
||
return
|
||
|
||
if args.command == "sync-search":
|
||
scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
|
||
data = scraper.sync_search(
|
||
start_page=args.page,
|
||
max_pages=args.max_pages,
|
||
lane=args.lane,
|
||
search_url=args.search_url,
|
||
make_id=args.make_id,
|
||
model_id=args.model_id,
|
||
price_min=args.price_min,
|
||
price_max=args.price_max,
|
||
year_min=args.year_min,
|
||
year_max=args.year_max,
|
||
)
|
||
save_to_json(data, Path(args.output))
|
||
print(f"Saved to {Path(args.output).resolve()}")
|
||
return
|
||
|
||
if args.command == "sync-detail":
|
||
scraper = MobileDeScraper()
|
||
data = scraper.sync_detail(args.listing_id, lane=args.lane)
|
||
save_to_json(data, Path(args.output))
|
||
print(f"Saved to {Path(args.output).resolve()}")
|
||
return
|
||
|
||
with IAAIScraper(runtime_settings) as scraper:
|
||
if args.command == "init-db":
|
||
data = scraper.init_db()
|
||
print(f"DB initialized: {data}")
|
||
return
|
||
elif args.command == "collect-listing":
|
||
data = scraper.collect_listing(make=args.make, model=args.model)
|
||
elif args.command == "scrape-vehicle":
|
||
data = scraper.scrape_vehicle_detail(args.vehicle_url)
|
||
elif args.command == "sync-vehicle":
|
||
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
|
||
else:
|
||
only_new = None if args.only_new is None else args.only_new == "true"
|
||
data = scraper.sync_listing(
|
||
make=args.make,
|
||
model=args.model,
|
||
lane=args.lane,
|
||
limit=args.limit,
|
||
only_new=only_new,
|
||
)
|
||
|
||
save_to_json(data, Path(args.output))
|
||
print(f"Saved to {Path(args.output).resolve()}")
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|