qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepare mobile de parser release

Browse Source
This commit is contained in:
qananasikq
2026-04-27 21:10:17 +03:00
parent 31f87a9bdf
commit b86bcd04b8
21 changed files with 2835 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
iaai_scraper/cli.py
View File

@@ -3,11 +3,12 @@ from pathlib import Path
from .core.config import Settings
from .core.utils import save_to_json
from .mobile_de import MobileDeClient, MobileDeScraper
from .scraper import IAAIScraper
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="IAAI scraper CLI")
parser = argparse.ArgumentParser(description="mobile.de scraper CLI")
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
subparsers = parser.add_subparsers(dest="command", required=True)
@@ -16,21 +17,21 @@ def build_parser() -> argparse.ArgumentParser:
subparsers.add_parser("init-db", help="Create DB tables")
listing_parser = subparsers.add_parser("collect-listing", help="Collect vehicle URLs from listing page")
listing_parser = subparsers.add_parser("collect-listing", help="Deprecated IAAI command: collect vehicle URLs from listing page")
listing_parser.add_argument("--make", default=None)
listing_parser.add_argument("--model", default=None)
listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Scrape a vehicle detail page")
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Deprecated IAAI command: scrape a vehicle detail page")
scrape_parser.add_argument("vehicle_url")
scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Scrape + upsert one vehicle")
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Deprecated IAAI command: scrape + upsert one vehicle")
sync_vehicle_parser.add_argument("vehicle_url")
sync_vehicle_parser.add_argument("--lane", default="iaai")
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
sync_listing_parser = subparsers.add_parser("sync-listing", help="Collect listing + sync all vehicles")
sync_listing_parser = subparsers.add_parser("sync-listing", help="Deprecated IAAI command: collect listing + sync all vehicles")
sync_listing_parser.add_argument("--make", default=None)
sync_listing_parser.add_argument("--model", default=None)
sync_listing_parser.add_argument("--lane", default="iaai_cars")
@@ -38,6 +39,42 @@ def build_parser() -> argparse.ArgumentParser:
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages")
mobile_search_parser.add_argument("--page", type=int, default=1)
mobile_search_parser.add_argument("--max-pages", type=int, default=1)
mobile_search_parser.add_argument("--delay", type=float, default=0.7)
mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500")
mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id")
mobile_search_parser.add_argument("--price-min", default=None)
mobile_search_parser.add_argument("--price-max", default=None)
mobile_search_parser.add_argument("--year-min", default=None)
mobile_search_parser.add_argument("--year-max", default=None)
mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json"))
mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page")
mobile_detail_parser.add_argument("listing_id")
mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json"))
mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages")
mobile_sync_search_parser.add_argument("--page", type=int, default=1)
mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1)
mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7)
mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars")
mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
mobile_sync_search_parser.add_argument("--make-id", default=None)
mobile_sync_search_parser.add_argument("--model-id", default=None)
mobile_sync_search_parser.add_argument("--price-min", default=None)
mobile_sync_search_parser.add_argument("--price-max", default=None)
mobile_sync_search_parser.add_argument("--year-min", default=None)
mobile_sync_search_parser.add_argument("--year-max", default=None)
mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json"))
mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page")
mobile_sync_detail_parser.add_argument("listing_id")
mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars")
mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json"))
return parser
@@ -53,6 +90,55 @@ def main() -> None:
if args.debug:
runtime_settings.log_level = "DEBUG"
if args.command in {"search", "mobilede-search"}:
scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
data = scraper.collect_search(
start_page=args.page,
max_pages=args.max_pages,
search_url=args.search_url,
make_id=args.make_id,
model_id=args.model_id,
price_min=args.price_min,
price_max=args.price_max,
year_min=args.year_min,
year_max=args.year_max,
)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
return
if args.command in {"detail", "mobilede-detail"}:
scraper = MobileDeScraper()
data = scraper.collect_detail(args.listing_id)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
return
if args.command == "sync-search":
scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
data = scraper.sync_search(
start_page=args.page,
max_pages=args.max_pages,
lane=args.lane,
search_url=args.search_url,
make_id=args.make_id,
model_id=args.model_id,
price_min=args.price_min,
price_max=args.price_max,
year_min=args.year_min,
year_max=args.year_max,
)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
return
if args.command == "sync-detail":
scraper = MobileDeScraper()
data = scraper.sync_detail(args.listing_id, lane=args.lane)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
return
with IAAIScraper(runtime_settings) as scraper:
if args.command == "init-db":
data = scraper.init_db()