IAAI scraper

iaai_scraper/core/utils.py

@@ -0,0 +1,50 @@
+import json
+import random
+import re
+import time
+from pathlib import Path
+from typing import Any, Iterable
+
+
+def save_to_json(data: Any, filename: str | Path) -> None:
+    Path(filename).write_text(json.dumps(data, ensure_ascii=False, indent=2), encoding="utf-8")
+
+
+def short_sleep(a: float = 0.10, b: float = 0.35) -> None:
+    time.sleep(random.uniform(a, b))
+
+
+def mask_email(email: str) -> str:
+    if "@" not in email:
+        return "***"
+    local, domain = email.split("@", 1)
+    safe_local = local[:2] + "***" if len(local) > 2 else local[:1] + "*"
+    return f"{safe_local}@{domain}"
+
+
+def first_non_empty(values: Iterable[Any]) -> Any | None:
+    for value in values:
+        if value not in (None, "", [], {}, ()):
+            return value
+    return None
+
+
+# VIN, lot, price regex
+VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
+LOT_RE = re.compile(r"\b(\d{7,10})\b")
+PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")
+
+
+def deep_find_key(obj, target_keys: set[str], max_depth: int = 64, _depth: int = 0) -> list:
+    found = []
+    if _depth >= max_depth:
+        return found
+    if isinstance(obj, dict):
+        for key, value in obj.items():
+            if key.lower() in target_keys:
+                found.append(value)
+            found.extend(deep_find_key(value, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    elif isinstance(obj, list):
+        for item in obj:
+            found.extend(deep_find_key(item, target_keys, max_depth=max_depth, _depth=_depth + 1))
+    return found