qananasikq/dubizzle
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add checkpoint logic and tests

Browse Source
This commit is contained in:
qananasikq
2026-04-15 11:10:32 +03:00
parent d9111be2d2
commit acbb045b6e
8 changed files with 432 additions and 175 deletions
Download Patch File Download Diff File Expand all files Collapse all files

122
iaai_scraper/browser/listing.py
View File

@@ -77,6 +77,62 @@ class ListingCollector:
self.settings = settings
self.pacer = pacer
@staticmethod
def _get_current_page_number(page: Page) -> int | None:
try:
value = page.evaluate(
"""
() => {
const controls = Array.from(document.querySelectorAll('a,button,[role="button"],span,div'));
const current = controls.find((el) => {
const text = (el.textContent || '').trim();
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
});
if (current) {
return parseInt((current.textContent || '').trim(), 10);
}
const match = (document.body?.innerText || '').match(/\b(\d+)\s+of\s+\d+\+?/i);
return match ? parseInt(match[1], 10) : null;
}
"""
)
return int(value) if value is not None else None
except Exception:
return None
@staticmethod
def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=12000,
)
return True
except Exception:
pass
if expected_page_number is not None:
current_page = ListingCollector._get_current_page_number(page)
if current_page == expected_page_number:
return True
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
current_page = ListingCollector._get_current_page_number(page)
if expected_page_number is not None and current_page == expected_page_number:
return True
return not old_first_href
def open_cars_listing(self, page: Page) -> None:
logger.info("Opening cars listing page: %s", self.settings.listing.cars_url)
url = self.settings.listing.cars_url
@@ -248,7 +304,7 @@ class ListingCollector:
return found
def go_to_next_page(self, page: Page) -> bool:
def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
# Запоминаем первую ссылку текущей страницы для определения смены контента.
old_first_href = ""
try:
@@ -276,30 +332,9 @@ class ListingCollector:
except Exception:
continue
# Ждём смены контента (AJAX пагинация): первая VehicleDetail-ссылка должна измениться.
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=8000,
)
except Exception:
pass
else:
try:
page.wait_for_load_state("domcontentloaded", timeout=15000)
except Exception:
pass
try:
page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
except Exception:
pass
self.pacer.after_page_change()
return True
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
# + стрелка с иконкой, без явного текста Next.
@@ -357,24 +392,9 @@ class ListingCollector:
"""
))
if clicked:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=12000,
)
except Exception:
pass
else:
try:
page.wait_for_load_state("domcontentloaded", timeout=12000)
except Exception:
pass
self.pacer.after_page_change()
return True
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("Numeric/icon pagination fallback failed: %s", exc)
@@ -403,19 +423,9 @@ class ListingCollector:
"""
))
if clicked:
if old_first_href:
try:
page.wait_for_function(
f"""() => {{
const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
return a && a.getAttribute('href') !== '{old_first_href}';
}}""",
timeout=8000,
)
except Exception:
pass
self.pacer.after_page_change()
return True
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("JS next-page fallback failed: %s", exc)

201
iaai_scraper/scraper.py
View File

@@ -9,7 +9,7 @@ import html as html_module
from concurrent.futures import ThreadPoolExecutor, as_completed
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from typing import Any, Callable
from urllib.parse import urlsplit, urlunsplit
from urllib.request import Request, urlopen
@@ -29,7 +29,7 @@ from .core.utils import save_to_json
from .parsing.mapper import CarMapper
from .parsing.parser import VehicleParser
from .storage.db import PersistenceService
from .browser.listing import ListingCollector
from .browser.listing import ListingCollector, ListingPageResult
from .storage.schemas import CarRecord
logger = logging.getLogger("iaai_scraper.scraper")
@@ -258,6 +258,70 @@ class IAAIScraper:
new_urls = [url for url in vehicle_urls if url not in known_urls]
return new_urls, skipped
def _extract_page_urls(
self,
page_result: ListingPageResult,
all_raw_urls: list[str],
seen_urls: set[str],
all_listing_origin_urls: set[str] | None = None,
) -> list[str]:
page_urls: list[str] = []
for item in page_result.vehicle_links:
normalized = self._normalize_vehicle_url(item.href)
all_raw_urls.append(item.href)
if normalized and all_listing_origin_urls is not None:
all_listing_origin_urls.add(normalized)
if normalized and normalized not in seen_urls:
seen_urls.add(normalized)
page_urls.append(normalized)
return page_urls
def _recover_empty_listing_page(
self,
page: Page,
*,
page_number: int,
all_raw_urls: list[str],
seen_urls: set[str],
all_listing_origin_urls: set[str] | None = None,
) -> tuple[ListingPageResult, list[str]]:
if page_number == 1:
logger.warning("Page 1 returned 0 links — retrying listing open...")
time.sleep(3)
self.listing_collector.open_cars_listing(page)
page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls)
logger.warning(
"Page %d returned 0 links — retrying current page before stopping pagination",
page_number,
)
try:
page.reload(wait_until="domcontentloaded", timeout=30_000)
except Exception as exc:
logger.debug("Page %d reload failed during empty-page recovery: %s", page_number, exc)
page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
return page_result, self._extract_page_urls(page_result, all_raw_urls, seen_urls, all_listing_origin_urls)
def _reopen_listing_and_resume(
self,
*,
target_page_number: int,
make: str | None,
model: str | None,
) -> Page:
page = self._get_page_with_warmup()
self.listing_collector.open_cars_listing(page)
self.listing_collector.apply_filters(page, make=make, model=model)
for expected_page in range(2, target_page_number + 1):
if not self.listing_collector.go_to_next_page(page, expected_page_number=expected_page):
page.close()
raise RuntimeError(f"Failed to resume listing at page {target_page_number}")
logger.warning("Listing resumed at page %d after recovery", target_page_number)
return page
def _collect_listing_iterative(
self,
*,
@@ -301,28 +365,25 @@ class IAAIScraper:
})
# Собираем URL со страницы.
page_urls: list[str] = []
for item in page_result.vehicle_links:
normalized = self._normalize_vehicle_url(item.href)
all_raw_urls.append(item.href)
if normalized not in seen:
seen.add(normalized)
page_urls.append(normalized)
page_urls = self._extract_page_urls(page_result, all_raw_urls, seen)
if not page_urls:
# Страница 1 пустая — скорее всего transient network issue.
# Пробуем перезагрузить листинг ещё раз.
if page_number == 1:
logger.warning("Page 1 returned 0 links — retrying listing open...")
time.sleep(3)
self.listing_collector.open_cars_listing(page)
page_result, page_urls = self._recover_empty_listing_page(
page,
page_number=page_number,
all_raw_urls=all_raw_urls,
seen_urls=seen,
)
if not page_urls and page_number > 1:
logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number)
page.close()
page = self._reopen_listing_and_resume(
target_page_number=page_number,
make=make,
model=model,
)
page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
for item in page_result.vehicle_links:
normalized = self._normalize_vehicle_url(item.href)
all_raw_urls.append(item.href)
if normalized not in seen:
seen.add(normalized)
page_urls.append(normalized)
page_urls = self._extract_page_urls(page_result, all_raw_urls, seen)
if not page_urls:
logger.info("Page %d: 0 new links, stopping pagination", page_number)
break
@@ -349,9 +410,14 @@ class IAAIScraper:
if not page_result.next_page_detected:
logger.info("No next page detected, stopping")
break
if not self.listing_collector.go_to_next_page(page):
logger.info("Failed to navigate to next page, stopping")
break
if not self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1):
logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1)
page.close()
page = self._reopen_listing_and_resume(
target_page_number=page_number + 1,
make=make,
model=model,
)
finally:
page.close()
@@ -404,6 +470,8 @@ class IAAIScraper:
limit: int | None,
effective_only_new: bool,
started_at: float,
start_page: int = 1,
progress_callback: Callable[[int], None] | None = None,
) -> dict[str, Any]:
batch_size = self.settings.celery.batch_size
pending_urls: list[str] = []
@@ -540,42 +608,57 @@ class IAAIScraper:
failures.append({"vehicle_url": f"batch_{batch_start}", "error": str(batch_exc)})
return True
page = self._get_page_with_warmup()
page = self._get_page_with_warmup() if start_page <= 1 else None
try:
self.listing_collector.open_cars_listing(page)
applied_filters = self.listing_collector.apply_filters(page, make=make, model=model)
if start_page <= 1:
assert page is not None
self.listing_collector.open_cars_listing(page)
applied_filters = self.listing_collector.apply_filters(page, make=make, model=model)
else:
page = self._reopen_listing_and_resume(
target_page_number=start_page,
make=make,
model=model,
)
applied_filters = {"make": make, "model": model}
for page_number in range(1, max(1, self.settings.listing.max_pages_per_run) + 1):
for page_number in range(start_page, max(start_page, self.settings.listing.max_pages_per_run) + 1):
page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
pages_info.append({
"page_number": page_result.page_number,
"links_found": len(page_result.vehicle_links),
})
page_urls: list[str] = []
for item in page_result.vehicle_links:
normalized = self._normalize_vehicle_url(item.href)
all_raw_urls.append(item.href)
if normalized:
all_listing_origin_urls.add(normalized)
if normalized and normalized not in seen_urls:
seen_urls.add(normalized)
page_urls.append(normalized)
page_urls = self._extract_page_urls(
page_result,
all_raw_urls,
seen_urls,
all_listing_origin_urls,
)
if not page_urls:
if page_number == 1:
logger.warning("Page 1 returned 0 links — retrying listing open...")
time.sleep(3)
self.listing_collector.open_cars_listing(page)
page_result, page_urls = self._recover_empty_listing_page(
page,
page_number=page_number,
all_raw_urls=all_raw_urls,
seen_urls=seen_urls,
all_listing_origin_urls=all_listing_origin_urls,
)
if not page_urls and page_number > 1:
logger.warning("Page %d still empty after retry — reopening listing and resuming", page_number)
page.close()
page = self._reopen_listing_and_resume(
target_page_number=page_number,
make=make,
model=model,
)
page_result = self.listing_collector.collect_current_page(page, page_number=page_number)
for item in page_result.vehicle_links:
normalized = self._normalize_vehicle_url(item.href)
all_raw_urls.append(item.href)
if normalized:
all_listing_origin_urls.add(normalized)
if normalized and normalized not in seen_urls:
seen_urls.add(normalized)
page_urls.append(normalized)
page_urls = self._extract_page_urls(
page_result,
all_raw_urls,
seen_urls,
all_listing_origin_urls,
)
if not page_urls:
logger.info("Page %d: 0 new links, stopping pagination", page_number)
break
@@ -643,6 +726,12 @@ class IAAIScraper:
break
pending_urls = pending_urls[batch_size:]
if progress_callback is not None:
try:
progress_callback(page_number)
except Exception:
logger.warning("Failed to persist progress callback for page %d", page_number, exc_info=True)
if limit is not None and limit > 0 and total >= limit:
break
if (
@@ -651,8 +740,14 @@ class IAAIScraper:
or not page_result.next_page_detected
):
break
if not self.listing_collector.go_to_next_page(page):
break
if not self.listing_collector.go_to_next_page(page, expected_page_number=page_number + 1):
logger.warning("Failed to navigate to page %d — reopening listing and resuming", page_number + 1)
page.close()
page = self._reopen_listing_and_resume(
target_page_number=page_number + 1,
make=make,
model=model,
)
if pending_urls:
_process_pending_batch(pending_urls, cars_upserted + cars_failed)
@@ -1443,6 +1538,8 @@ class IAAIScraper:
lane: str = "iaai_cars",
limit: int | None = None,
only_new: bool | None = None,
start_page: int = 1,
progress_callback: Callable[[int], None] | None = None,
):
# Листинг + sync всех найденных машин.
# Применяем runtime_config как дефолты (CLI/API аргументы имеют приоритет).
@@ -1477,6 +1574,8 @@ class IAAIScraper:
limit=limit,
effective_only_new=effective_only_new,
started_at=started_at,
start_page=max(1, int(start_page)),
progress_callback=progress_callback,
)
listing = stream_result["listing"]
total = stream_result["total"]

85
iaai_scraper/worker/tasks.py
View File

@@ -1,6 +1,7 @@
# Задачи Celery для синхронизации автомобилей и листинга IAAI.
from concurrent.futures import ThreadPoolExecutor
import json
import logging
from threading import Event, Thread
import time
@@ -18,6 +19,7 @@ logger = logging.getLogger("iaai_scraper.worker.tasks")
SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
SYNC_FULL_SCAN_DONE_KEY = "iaai:state:sync_full_scan_done"
SYNC_LISTING_CHECKPOINT_KEY = "iaai:state:sync_listing_checkpoint"
SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task"
@@ -228,6 +230,55 @@ def _set_full_scan_done(redis_client: Redis, done: bool) -> None:
logger.warning("Failed to persist full scan state", exc_info=True)
def _load_sync_checkpoint(redis_client: Redis) -> dict[str, object] | None:
try:
raw = redis_client.get(SYNC_LISTING_CHECKPOINT_KEY)
except Exception:
logger.warning("Failed to read sync listing checkpoint", exc_info=True)
return None
if not raw:
return None
if isinstance(raw, str) and not raw.strip():
return None
try:
data = json.loads(str(raw))
except Exception:
logger.warning("Failed to decode sync listing checkpoint", exc_info=True)
return None
return data if isinstance(data, dict) else None
def _save_sync_checkpoint(
redis_client: Redis,
*,
task_id: str,
page_number: int,
make: str | None,
model: str | None,
lane: str,
) -> None:
payload = {
"status": "in_progress",
"task_id": task_id,
"last_successful_page": int(page_number),
"make": make,
"model": model,
"lane": lane,
"updated_at": int(time.time()),
}
try:
redis_client.set(SYNC_LISTING_CHECKPOINT_KEY, json.dumps(payload))
except Exception:
logger.warning("Failed to save sync listing checkpoint", exc_info=True)
def _clear_sync_checkpoint(redis_client: Redis) -> None:
try:
redis_client.delete(SYNC_LISTING_CHECKPOINT_KEY)
except Exception:
logger.warning("Failed to clear sync listing checkpoint", exc_info=True)
def _start_lock_heartbeat(
redis_client: Redis,
key: str,
@@ -350,6 +401,28 @@ def sync_listing_task(
force_bootstrap_full_scan = not full_scan_done_before_run
effective_limit = None if force_bootstrap_full_scan else limit
effective_only_new = False if force_bootstrap_full_scan else only_new
checkpoint = _load_sync_checkpoint(redis_client)
resume_from_page = 1
if checkpoint and str(checkpoint.get("status") or "") == "in_progress":
checkpoint_page = int(checkpoint.get("last_successful_page") or 0)
checkpoint_make = checkpoint.get("make")
checkpoint_model = checkpoint.get("model")
checkpoint_lane = checkpoint.get("lane")
same_scope = (
checkpoint_make == make
and checkpoint_model == model
and checkpoint_lane == lane
)
if checkpoint_page > 0 and same_scope:
resume_from_page = checkpoint_page + 1
logger.warning(
"Resuming sync_listing from page %d using checkpoint",
resume_from_page,
)
elif checkpoint_page > 0:
logger.info("Ignoring stale checkpoint due to different sync parameters")
_clear_sync_checkpoint(redis_client)
if force_bootstrap_full_scan:
logger.info(
@@ -377,6 +450,15 @@ def sync_listing_task(
lane=lane,
limit=effective_limit,
only_new=effective_only_new,
start_page=resume_from_page,
progress_callback=lambda page_number: _save_sync_checkpoint(
redis_client,
task_id=task_id,
page_number=page_number,
make=make,
model=model,
lane=lane,
),
)
result = _run_browser_job(_job)
@@ -385,11 +467,14 @@ def sync_listing_task(
bootstrap_completed = bool(result.get("full_scan_completed"))
if bootstrap_completed:
_set_full_scan_done(redis_client, True)
_clear_sync_checkpoint(redis_client)
logger.info("Bootstrap full scan completed; hourly schedule continues")
else:
_set_full_scan_done(redis_client, False)
logger.info("Bootstrap full scan not complete yet; queuing immediate continuation")
_enqueue_bootstrap_followup("bootstrap_not_completed")
elif result.get("status") == "success":
_clear_sync_checkpoint(redis_client)
summary = {
"task_id": task_id,