qananasikq/mobile.de
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Prepare mobile de parser preview

Browse Source
This commit is contained in:
qananasikq
2026-04-27 12:08:24 +03:00
commit 1eb7c8735a
76 changed files with 17855 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

168
iaai_scraper/cli.py Normal file
View File

@@ -0,0 +1,168 @@
import argparse
from pathlib import Path
from .core.config import Settings
from .core.utils import save_to_json
from .mobile_de import MobileDeClient, MobileDeScraper
from .scraper import IAAIScraper
def build_parser() -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(description="mobile.de scraper CLI")
parser.add_argument("--headless", choices=["true", "false"], default=None, help="Override headless mode")
parser.add_argument("--debug", action="store_true", help="Enable DEBUG logging")
subparsers = parser.add_subparsers(dest="command", required=True)
default_output_dir = Path("artifacts/json")
subparsers.add_parser("init-db", help="Create DB tables")
listing_parser = subparsers.add_parser("collect-listing", help="Deprecated IAAI command: collect vehicle URLs from listing page")
listing_parser.add_argument("--make", default=None)
listing_parser.add_argument("--model", default=None)
listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_listing_links.json"))
scrape_parser = subparsers.add_parser("scrape-vehicle", help="Deprecated IAAI command: scrape a vehicle detail page")
scrape_parser.add_argument("vehicle_url")
scrape_parser.add_argument("--output", default=str(default_output_dir / "iaai_vehicle_detail.json"))
sync_vehicle_parser = subparsers.add_parser("sync-vehicle", help="Deprecated IAAI command: scrape + upsert one vehicle")
sync_vehicle_parser.add_argument("vehicle_url")
sync_vehicle_parser.add_argument("--lane", default="iaai")
sync_vehicle_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_vehicle.json"))
sync_listing_parser = subparsers.add_parser("sync-listing", help="Deprecated IAAI command: collect listing + sync all vehicles")
sync_listing_parser.add_argument("--make", default=None)
sync_listing_parser.add_argument("--model", default=None)
sync_listing_parser.add_argument("--lane", default="iaai_cars")
sync_listing_parser.add_argument("--limit", type=int, default=None)
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
mobile_search_parser = subparsers.add_parser("search", aliases=["mobilede-search"], help="Collect mobile.de search result pages")
mobile_search_parser.add_argument("--page", type=int, default=1)
mobile_search_parser.add_argument("--max-pages", type=int, default=1)
mobile_search_parser.add_argument("--delay", type=float, default=0.7)
mobile_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
mobile_search_parser.add_argument("--make-id", default=None, help="mobile.de make id, for example BMW=3500")
mobile_search_parser.add_argument("--model-id", default=None, help="mobile.de model id")
mobile_search_parser.add_argument("--price-min", default=None)
mobile_search_parser.add_argument("--price-max", default=None)
mobile_search_parser.add_argument("--year-min", default=None)
mobile_search_parser.add_argument("--year-max", default=None)
mobile_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_listing_links.json"))
mobile_detail_parser = subparsers.add_parser("detail", aliases=["mobilede-detail"], help="Collect one mobile.de detail page")
mobile_detail_parser.add_argument("listing_id")
mobile_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_vehicle_detail.json"))
mobile_sync_search_parser = subparsers.add_parser("sync-search", help="Collect and upsert mobile.de search result pages")
mobile_sync_search_parser.add_argument("--page", type=int, default=1)
mobile_sync_search_parser.add_argument("--max-pages", type=int, default=1)
mobile_sync_search_parser.add_argument("--delay", type=float, default=0.7)
mobile_sync_search_parser.add_argument("--lane", default="mobile_de_cars")
mobile_sync_search_parser.add_argument("--search-url", default=None, help="готовая mobile.de ссылка с фильтрами")
mobile_sync_search_parser.add_argument("--make-id", default=None)
mobile_sync_search_parser.add_argument("--model-id", default=None)
mobile_sync_search_parser.add_argument("--price-min", default=None)
mobile_sync_search_parser.add_argument("--price-max", default=None)
mobile_sync_search_parser.add_argument("--year-min", default=None)
mobile_sync_search_parser.add_argument("--year-max", default=None)
mobile_sync_search_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_search.json"))
mobile_sync_detail_parser = subparsers.add_parser("sync-detail", help="Collect and upsert one mobile.de detail page")
mobile_sync_detail_parser.add_argument("listing_id")
mobile_sync_detail_parser.add_argument("--lane", default="mobile_de_cars")
mobile_sync_detail_parser.add_argument("--output", default=str(default_output_dir / "mobilede_sync_detail.json"))
return parser
def main() -> None:
parser = build_parser()
args = parser.parse_args()
runtime_settings: Settings | None = None
if args.headless is not None or args.debug:
runtime_settings = Settings()
if args.headless is not None:
runtime_settings.headless = args.headless == "true"
if args.debug:
runtime_settings.log_level = "DEBUG"
if args.command in {"search", "mobilede-search"}:
scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
data = scraper.collect_search(
start_page=args.page,
max_pages=args.max_pages,
search_url=args.search_url,
make_id=args.make_id,
model_id=args.model_id,
price_min=args.price_min,
price_max=args.price_max,
year_min=args.year_min,
year_max=args.year_max,
)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
return
if args.command in {"detail", "mobilede-detail"}:
scraper = MobileDeScraper()
data = scraper.collect_detail(args.listing_id)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
return
if args.command == "sync-search":
scraper = MobileDeScraper(MobileDeClient(delay_seconds=args.delay))
data = scraper.sync_search(
start_page=args.page,
max_pages=args.max_pages,
lane=args.lane,
search_url=args.search_url,
make_id=args.make_id,
model_id=args.model_id,
price_min=args.price_min,
price_max=args.price_max,
year_min=args.year_min,
year_max=args.year_max,
)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
return
if args.command == "sync-detail":
scraper = MobileDeScraper()
data = scraper.sync_detail(args.listing_id, lane=args.lane)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
return
with IAAIScraper(runtime_settings) as scraper:
if args.command == "init-db":
data = scraper.init_db()
print(f"DB initialized: {data}")
return
elif args.command == "collect-listing":
data = scraper.collect_listing(make=args.make, model=args.model)
elif args.command == "scrape-vehicle":
data = scraper.scrape_vehicle_detail(args.vehicle_url)
elif args.command == "sync-vehicle":
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
else:
only_new = None if args.only_new is None else args.only_new == "true"
data = scraper.sync_listing(
make=args.make,
model=args.model,
lane=args.lane,
limit=args.limit,
only_new=only_new,
)
save_to_json(data, Path(args.output))
print(f"Saved to {Path(args.output).resolve()}")
if __name__ == "__main__":
main()