480 lines
15 KiB
Python
480 lines
15 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import logging
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from typing import Any
|
|
from urllib.parse import parse_qs, urlencode, urlparse
|
|
from urllib.request import Request, urlopen
|
|
|
|
|
|
logger = logging.getLogger("dubizzle_scraper.discovery.algolia")
|
|
|
|
ALGOLIA_HARD_PAGE_LIMIT = 100
|
|
ALGOLIA_MAX_HITS_PER_QUERY = ALGOLIA_HARD_PAGE_LIMIT * 100
|
|
ALGOLIA_ID_RANGE_START = 0
|
|
ALGOLIA_ID_RANGE_END = 20_000_000
|
|
ALGOLIA_ID_RANGE_MIN_WINDOW = 100
|
|
|
|
|
|
class AlgoliaDiscoveryError(RuntimeError):
|
|
pass
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class AlgoliaDiscoveryPageStat:
|
|
page_number: int
|
|
hits_count: int
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class AlgoliaDiscoveryResult:
|
|
vehicle_urls: list[str] = field(default_factory=list)
|
|
hit_records: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
origin_ids_by_url: dict[str, str] = field(default_factory=dict)
|
|
pages: list[dict[str, int]] = field(default_factory=list)
|
|
early_stopped: bool = False
|
|
truncated_by_time_budget: bool = False
|
|
total_hits: int = 0
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _AlgoliaShard:
|
|
category_slug: str
|
|
id_min: int | None = None
|
|
id_max: int | None = None
|
|
year_min: int | None = None
|
|
year_max: int | None = None
|
|
|
|
def filter_expr(self) -> str:
|
|
parts = [f"(category_v2.slug_paths:{self.category_slug})"]
|
|
if self.year_min is not None:
|
|
parts.append(f"year>={int(self.year_min)}")
|
|
if self.year_max is not None:
|
|
parts.append(f"year<={int(self.year_max)}")
|
|
if self.id_min is not None:
|
|
parts.append(f"id>={int(self.id_min)}")
|
|
if self.id_max is not None:
|
|
parts.append(f"id<={int(self.id_max)}")
|
|
return " AND ".join(parts)
|
|
|
|
def label(self) -> str:
|
|
label = self.category_slug
|
|
if self.year_min is not None or self.year_max is not None:
|
|
label += f"[year:{self.year_min}-{self.year_max}]"
|
|
if self.id_min is None and self.id_max is None:
|
|
return label
|
|
return f"{label}[id:{self.id_min}-{self.id_max}]"
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class _AlgoliaHttpClient:
|
|
endpoint: str
|
|
app_id: str
|
|
api_key: str
|
|
user_agent: str
|
|
index_name: str
|
|
hits_per_page: int
|
|
|
|
def request(self, *, page: int, filters: str) -> dict[str, Any]:
|
|
params = urlencode(
|
|
{
|
|
"query": "",
|
|
"page": page,
|
|
"hitsPerPage": self.hits_per_page,
|
|
"filters": filters,
|
|
}
|
|
)
|
|
payload = {"requests": [{"indexName": self.index_name, "params": params}]}
|
|
request = Request(
|
|
self.endpoint,
|
|
data=json.dumps(payload).encode("utf-8"),
|
|
headers={
|
|
"content-type": "application/json",
|
|
"x-algolia-application-id": self.app_id,
|
|
"x-algolia-api-key": self.api_key,
|
|
"accept": "application/json",
|
|
"user-agent": self.user_agent,
|
|
},
|
|
method="POST",
|
|
)
|
|
with urlopen(request, timeout=30) as response:
|
|
body = response.read().decode("utf-8", errors="ignore")
|
|
parsed = json.loads(body)
|
|
results = parsed.get("results") or []
|
|
return results[0] if results and isinstance(results[0], dict) else {}
|
|
|
|
|
|
def _build_origin_id_from_hit(hit: dict[str, Any]) -> str | None:
|
|
for key in ("id", "objectID", "uuid"):
|
|
value = hit.get(key)
|
|
if value is None:
|
|
continue
|
|
text = str(value).strip()
|
|
if text:
|
|
return f"dubizzle:{text}"
|
|
permalink = str(hit.get("permalink") or "").strip()
|
|
if permalink:
|
|
tail = permalink.rstrip("/").split("/")[-1]
|
|
if tail:
|
|
return f"dubizzle:{tail}"
|
|
return None
|
|
|
|
|
|
def _build_vehicle_url_from_hit(hit: dict[str, Any]) -> str | None:
|
|
permalink = str(hit.get("permalink") or "").strip()
|
|
if permalink:
|
|
if permalink.startswith("http://") or permalink.startswith("https://"):
|
|
return permalink
|
|
if permalink.startswith("/"):
|
|
return f"https://www.dubizzle.com{permalink}"
|
|
return f"https://www.dubizzle.com/{permalink.lstrip('/')}"
|
|
|
|
short = str(hit.get("short_url") or "").strip()
|
|
if short:
|
|
if short.startswith("http://") or short.startswith("https://"):
|
|
return short
|
|
return f"https://dubizzle.com/s/{short.strip('/')}"
|
|
|
|
hit_id = hit.get("id") or hit.get("objectID")
|
|
if hit_id is not None:
|
|
return f"https://www.dubizzle.com/motors/used-cars/ad-{hit_id}/"
|
|
return None
|
|
|
|
|
|
def _extract_make_from_listing_url(listing_url: str | None) -> str | None:
|
|
if not listing_url:
|
|
return None
|
|
try:
|
|
parsed = urlparse(listing_url)
|
|
params = parse_qs(parsed.query)
|
|
candidate = (params.get("Make") or params.get("make") or [None])[0]
|
|
if candidate:
|
|
return str(candidate).strip()
|
|
parts = [p for p in parsed.path.split("/") if p]
|
|
if len(parts) >= 3 and parts[0].lower() == "motors" and parts[1].lower() == "used-cars":
|
|
slug = parts[2].strip().lower()
|
|
if slug and slug != "s":
|
|
return slug.replace("-", " ")
|
|
except Exception:
|
|
return None
|
|
return None
|
|
|
|
|
|
def _make_slug(make: str | None) -> str | None:
|
|
if not make:
|
|
return None
|
|
slug = make.strip().lower().replace(" ", "-")
|
|
return slug or None
|
|
|
|
|
|
def _hit_matches_filters(
|
|
hit: dict[str, Any],
|
|
*,
|
|
make: str | None,
|
|
model: str | None,
|
|
year_min: int | None,
|
|
year_max: int | None,
|
|
) -> bool:
|
|
if make:
|
|
hit_make = str(hit.get("make") or "").strip().lower()
|
|
if hit_make != make.strip().lower():
|
|
return False
|
|
if model:
|
|
hit_model = str(hit.get("model") or "").strip().lower()
|
|
if hit_model != model.strip().lower():
|
|
return False
|
|
|
|
hit_year_raw = hit.get("year")
|
|
hit_year: int | None = None
|
|
if hit_year_raw is not None:
|
|
try:
|
|
hit_year = int(hit_year_raw)
|
|
except Exception:
|
|
hit_year = None
|
|
|
|
if year_min is not None and (hit_year is None or hit_year < year_min):
|
|
return False
|
|
if year_max is not None and (hit_year is None or hit_year > year_max):
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def _probe_total_hits(client: _AlgoliaHttpClient, shard: _AlgoliaShard) -> int:
|
|
first = client.request(page=0, filters=shard.filter_expr())
|
|
return int(first.get("nbHits") or 0)
|
|
|
|
|
|
def _split_id_range(shard: _AlgoliaShard) -> tuple[_AlgoliaShard, _AlgoliaShard] | None:
|
|
lo = shard.id_min if shard.id_min is not None else ALGOLIA_ID_RANGE_START
|
|
hi = shard.id_max if shard.id_max is not None else ALGOLIA_ID_RANGE_END
|
|
if hi - lo <= ALGOLIA_ID_RANGE_MIN_WINDOW:
|
|
return None
|
|
mid = (lo + hi) // 2
|
|
return (
|
|
_AlgoliaShard(
|
|
category_slug=shard.category_slug,
|
|
id_min=lo,
|
|
id_max=mid,
|
|
year_min=shard.year_min,
|
|
year_max=shard.year_max,
|
|
),
|
|
_AlgoliaShard(
|
|
category_slug=shard.category_slug,
|
|
id_min=mid + 1,
|
|
id_max=hi,
|
|
year_min=shard.year_min,
|
|
year_max=shard.year_max,
|
|
),
|
|
)
|
|
|
|
|
|
def _expand_shards(client: _AlgoliaHttpClient, initial_shard: _AlgoliaShard, max_duration_seconds: float | None, started_at: float) -> tuple[list[_AlgoliaShard], bool, int]:
|
|
queue: list[_AlgoliaShard] = [initial_shard]
|
|
ready: list[_AlgoliaShard] = []
|
|
truncated = False
|
|
total_hits = 0
|
|
|
|
while queue:
|
|
if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
|
|
truncated = True
|
|
break
|
|
shard = queue.pop(0)
|
|
shard_total = _probe_total_hits(client, shard)
|
|
if shard is initial_shard:
|
|
total_hits = shard_total
|
|
if shard_total == 0:
|
|
continue
|
|
if shard_total <= ALGOLIA_MAX_HITS_PER_QUERY:
|
|
ready.append(shard)
|
|
continue
|
|
split = _split_id_range(shard)
|
|
if split is None:
|
|
logger.warning(
|
|
"Shard %s still exceeds limit=%d but id-window is too small to split further (hits=%d)",
|
|
shard.label(),
|
|
ALGOLIA_MAX_HITS_PER_QUERY,
|
|
shard_total,
|
|
)
|
|
ready.append(shard)
|
|
continue
|
|
logger.warning(
|
|
"Splitting Algolia shard %s with hits=%d into %s and %s",
|
|
shard.label(),
|
|
shard_total,
|
|
split[0].label(),
|
|
split[1].label(),
|
|
)
|
|
queue.insert(0, split[1])
|
|
queue.insert(0, split[0])
|
|
|
|
return ready, truncated, total_hits
|
|
|
|
|
|
def _fetch_shard_hits(
|
|
*,
|
|
client: _AlgoliaHttpClient,
|
|
shard: _AlgoliaShard,
|
|
make: str | None,
|
|
model: str | None,
|
|
year_min: int | None,
|
|
year_max: int | None,
|
|
known_origin_ids: set[str] | None,
|
|
threshold: float,
|
|
max_duration_seconds: float | None,
|
|
started_at: float,
|
|
) -> tuple[list[str], dict[str, dict[str, Any]], dict[str, str], list[dict[str, int]], bool, bool]:
|
|
urls: list[str] = []
|
|
hit_records: dict[str, dict[str, Any]] = {}
|
|
origin_ids_by_url: dict[str, str] = {}
|
|
pages: list[dict[str, int]] = []
|
|
early_stopped = False
|
|
truncated_by_time_budget = False
|
|
page = 0
|
|
nb_pages = None
|
|
|
|
while True:
|
|
if max_duration_seconds is not None and (time.perf_counter() - started_at) > max_duration_seconds:
|
|
truncated_by_time_budget = True
|
|
break
|
|
|
|
try:
|
|
first = client.request(page=page, filters=shard.filter_expr())
|
|
except Exception as exc:
|
|
raise AlgoliaDiscoveryError(
|
|
f"Algolia request failed for shard={shard.label()} page={page}: {exc}"
|
|
) from exc
|
|
|
|
hits = first.get("hits") or []
|
|
if not isinstance(hits, list):
|
|
hits = []
|
|
|
|
if page == 0:
|
|
nb_pages = min(int(first.get("nbPages") or 0), ALGOLIA_HARD_PAGE_LIMIT)
|
|
|
|
pages.append({"page_number": page + 1, "links_found": len(hits), "shard": shard.label()})
|
|
if not hits:
|
|
break
|
|
|
|
page_known = 0
|
|
page_new = 0
|
|
for raw_hit in hits:
|
|
if not isinstance(raw_hit, dict):
|
|
continue
|
|
if not _hit_matches_filters(
|
|
raw_hit,
|
|
make=make,
|
|
model=model,
|
|
year_min=year_min,
|
|
year_max=year_max,
|
|
):
|
|
continue
|
|
|
|
url = _build_vehicle_url_from_hit(raw_hit)
|
|
if not url:
|
|
continue
|
|
origin_id = _build_origin_id_from_hit(raw_hit)
|
|
if origin_id and known_origin_ids is not None and origin_id in known_origin_ids:
|
|
page_known += 1
|
|
else:
|
|
page_new += 1
|
|
|
|
if url in hit_records:
|
|
continue
|
|
urls.append(url)
|
|
hit_records[url] = raw_hit
|
|
if origin_id:
|
|
origin_ids_by_url[url] = origin_id
|
|
|
|
if known_origin_ids is not None and threshold > 0 and (page_known + page_new) > 0:
|
|
ratio = page_known / (page_known + page_new)
|
|
if ratio >= threshold and page_new == 0:
|
|
early_stopped = True
|
|
break
|
|
|
|
page += 1
|
|
if nb_pages is not None and page >= nb_pages:
|
|
break
|
|
|
|
return urls, hit_records, origin_ids_by_url, pages, early_stopped, truncated_by_time_budget
|
|
|
|
|
|
def discover_vehicle_hits_from_algolia(
|
|
*,
|
|
settings,
|
|
make: str | None = None,
|
|
model: str | None = None,
|
|
limit: int | None = None,
|
|
year_min: int | None = None,
|
|
year_max: int | None = None,
|
|
listing_url: str | None = None,
|
|
known_origin_ids: set[str] | None = None,
|
|
max_duration_seconds: float | None = None,
|
|
) -> AlgoliaDiscoveryResult:
|
|
app_id = settings.algolia.application_id.strip()
|
|
api_key = settings.algolia.api_key.strip()
|
|
index_name = settings.algolia.index_name.strip()
|
|
if not app_id or not api_key or not index_name:
|
|
raise AlgoliaDiscoveryError("Algolia credentials/index are not configured")
|
|
|
|
effective_make = make or _extract_make_from_listing_url(listing_url)
|
|
hits_per_page = max(1, min(100, int(settings.algolia.hits_per_page)))
|
|
base_url = settings.algolia.base_url.strip().rstrip("/")
|
|
if not base_url:
|
|
base_url = f"https://{app_id}-dsn.algolia.net"
|
|
|
|
category_slug = settings.algolia.category_slug.strip() or "motors/used-cars"
|
|
make_slug = _make_slug(effective_make)
|
|
if make_slug:
|
|
category_slug = f"{category_slug}/{make_slug}"
|
|
|
|
client = _AlgoliaHttpClient(
|
|
endpoint=f"{base_url}/1/indexes/*/queries",
|
|
app_id=app_id,
|
|
api_key=api_key,
|
|
user_agent=settings.fingerprint.user_agent,
|
|
index_name=index_name,
|
|
hits_per_page=hits_per_page,
|
|
)
|
|
threshold = float(settings.listing.early_stop_threshold)
|
|
started_at = time.perf_counter()
|
|
|
|
initial_shard = _AlgoliaShard(
|
|
category_slug=category_slug,
|
|
id_min=ALGOLIA_ID_RANGE_START,
|
|
id_max=ALGOLIA_ID_RANGE_END,
|
|
year_min=year_min,
|
|
year_max=year_max,
|
|
)
|
|
shards, shard_build_truncated, total_hits = _expand_shards(
|
|
client,
|
|
initial_shard,
|
|
max_duration_seconds,
|
|
started_at,
|
|
)
|
|
|
|
collected_urls: list[str] = []
|
|
hits_by_url: dict[str, dict[str, Any]] = {}
|
|
origin_ids_by_url: dict[str, str] = {}
|
|
pages: list[dict[str, int]] = []
|
|
early_stopped = False
|
|
truncated_by_time_budget = shard_build_truncated
|
|
|
|
logger.warning(
|
|
"Algolia shard plan ready: total_hits=%d shards=%d category=%s",
|
|
total_hits,
|
|
len(shards),
|
|
category_slug,
|
|
)
|
|
|
|
for shard in shards:
|
|
shard_urls, shard_hits, shard_origin_ids, shard_pages, shard_early_stop, shard_truncated = _fetch_shard_hits(
|
|
client=client,
|
|
shard=shard,
|
|
make=effective_make,
|
|
model=model,
|
|
year_min=year_min,
|
|
year_max=year_max,
|
|
known_origin_ids=known_origin_ids,
|
|
threshold=threshold,
|
|
max_duration_seconds=max_duration_seconds,
|
|
started_at=started_at,
|
|
)
|
|
pages.extend(shard_pages)
|
|
early_stopped = early_stopped or shard_early_stop
|
|
truncated_by_time_budget = truncated_by_time_budget or shard_truncated
|
|
for url in shard_urls:
|
|
if url in hits_by_url:
|
|
continue
|
|
collected_urls.append(url)
|
|
hits_by_url[url] = shard_hits[url]
|
|
if url in shard_origin_ids:
|
|
origin_ids_by_url[url] = shard_origin_ids[url]
|
|
if limit is not None and limit > 0 and len(collected_urls) >= limit:
|
|
break
|
|
if truncated_by_time_budget:
|
|
break
|
|
|
|
logger.info(
|
|
"Algolia discovery done: urls=%d total_hits=%d pages=%d shards=%d",
|
|
len(collected_urls),
|
|
total_hits,
|
|
len(pages),
|
|
len(shards),
|
|
)
|
|
|
|
if limit is not None and limit > 0 and len(collected_urls) > limit:
|
|
collected_urls = collected_urls[:limit]
|
|
|
|
return AlgoliaDiscoveryResult(
|
|
vehicle_urls=collected_urls,
|
|
hit_records={url: hits_by_url[url] for url in collected_urls if url in hits_by_url},
|
|
origin_ids_by_url={url: origin_ids_by_url[url] for url in collected_urls if url in origin_ids_by_url},
|
|
pages=pages,
|
|
early_stopped=early_stopped,
|
|
truncated_by_time_budget=truncated_by_time_budget,
|
|
total_hits=total_hits,
|
|
)
|