qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Refactor to new ingestion pipeline architecture with discovery, fetch, enrichment services

Browse Source
This commit is contained in:
qananasikq
2026-04-20 16:27:42 +03:00
parent 55203d33ea
commit 65d5f8e1eb
23 changed files with 3436 additions and 518 deletions
Download Patch File Download Diff File Expand all files Collapse all files

286
iaai_scraper/browser/listing.py
View File

@@ -3,13 +3,12 @@ import re
import time
from dataclasses import asdict, dataclass, field
from typing import Any
from urllib.parse import urljoin
from urllib.parse import parse_qsl, urlencode, urljoin, urlparse, urlunparse
from playwright.sync_api import Page
from .pace import HumanPacer
from ..core.config import Settings
from ..core.utils import first_non_empty
logger = logging.getLogger("iaai_scraper.listing")
VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
@@ -44,6 +43,7 @@ class ListingPageResult:
class ListingCollector:
_DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE = 40
_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
"a[aria-label*='Next']",
"button[aria-label*='Next']",
@@ -103,7 +103,33 @@ class ListingCollector:
return None
@staticmethod
def _wait_for_navigation_result(page: Page, old_first_href: str, expected_page_number: int | None) -> bool:
def _get_vehicle_link_fingerprint(page: Page, limit: int = 5) -> tuple[str, ...]:
try:
values = page.evaluate(
"""
(limit) => {
return Array.from(document.querySelectorAll("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"))
.map((el) => (el.getAttribute('href') || '').trim())
.filter(Boolean)
.slice(0, limit);
}
""",
limit,
)
if not isinstance(values, list):
return tuple()
return tuple(str(value) for value in values if value)
except Exception:
return tuple()
@staticmethod
def _wait_for_navigation_result(
page: Page,
old_first_href: str,
expected_page_number: int | None,
*,
old_fingerprint: tuple[str, ...] = (),
) -> bool:
if old_first_href:
try:
page.wait_for_function(
@@ -117,9 +143,12 @@ class ListingCollector:
except Exception:
pass
new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
if expected_page_number is not None:
current_page = ListingCollector._get_current_page_number(page)
if current_page == expected_page_number:
if current_page == expected_page_number and (
not old_fingerprint or (new_fingerprint and new_fingerprint != old_fingerprint)
):
return True
try:
@@ -127,11 +156,81 @@ class ListingCollector:
except Exception:
pass
if not new_fingerprint:
new_fingerprint = ListingCollector._get_vehicle_link_fingerprint(page)
current_page = ListingCollector._get_current_page_number(page)
if new_fingerprint and old_fingerprint and new_fingerprint != old_fingerprint:
return current_page is None or expected_page_number is None or current_page == expected_page_number
if expected_page_number is not None and current_page == expected_page_number:
return True
return not old_fingerprint
return not old_first_href
return not old_first_href and (not old_fingerprint or bool(new_fingerprint))
def _extract_next_page_href(self, page: Page, expected_page_number: int | None = None) -> str | None:
try:
href = page.evaluate(
"""
(expectedPageNumber) => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const disabled = (el) => {
if (!el) return true;
const cls = (el.getAttribute('class') || '').toLowerCase();
const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
};
const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
.filter((el) => visible(el) && !disabled(el));
const cleanHref = (el) => {
const href = (el?.getAttribute('href') || '').trim();
if (!href || href.startsWith('javascript:') || href.startsWith('#')) {
return '';
}
return href;
};
if (expectedPageNumber !== null && expectedPageNumber !== undefined) {
const numeric = controls.find((el) => {
const text = (el.textContent || '').trim();
return /^\d+$/.test(text) && parseInt(text, 10) === expectedPageNumber;
});
const numericHref = cleanHref(numeric);
if (numericHref) {
return numericHref;
}
}
const explicitNext = controls.find((el) => {
const text = (el.textContent || '').trim().toLowerCase();
const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
const title = (el.getAttribute('title') || '').trim().toLowerCase();
const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
const cls = (el.getAttribute('class') || '').trim().toLowerCase();
const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
return rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || ['next', '›', '»', '>'].includes(text);
});
return cleanHref(explicitNext);
}
""",
expected_page_number,
)
if not href:
return None
return urljoin(page.url or self.settings.home_url, str(href))
except Exception:
return None
@staticmethod
def _build_listing_page_url(url: str, page_number: int) -> str:
parsed = urlparse(url)
query_items = [
(key, value)
for key, value in parse_qsl(parsed.query, keep_blank_values=True)
if key.lower() not in {"page", "pagenumber", "currentpage"}
]
query_items.append(("page", str(page_number)))
return urlunparse(parsed._replace(query=urlencode(query_items)))
def open_cars_listing(self, page: Page, *, url_override: str | None = None) -> None:
url = url_override or self.settings.listing.cars_url
@@ -202,6 +301,31 @@ class ListingCollector:
# Короткая пауза вместо длинного sleep.
time.sleep(1.0)
def _get_listing_html(self, page: Page, page_number: int) -> str:
try:
return page.locator("html").inner_html(timeout=5_000)
except Exception as exc:
logger.warning("listing html read failed on page %d: %s", page_number, exc)
return ""
def _has_next_page_from_html(self, html: str, current_page_number: int | None = None) -> bool:
if not html:
return False
normalized = html.replace("\\/", "/")
if re.search(
r"rel\s*=\s*['\"]next['\"]|aria-label\s*=\s*['\"][^'\"]*next|title\s*=\s*['\"][^'\"]*next|class\s*=\s*['\"][^'\"]*next|icon-arrow-right|arrow-right|>\s*next\s*<|>\s*[›»>]\s*<",
normalized,
re.IGNORECASE,
):
return True
if current_page_number is not None:
next_page = current_page_number + 1
if re.search(rf">\s*{next_page}\s*<", normalized, re.IGNORECASE):
return True
if re.search(rf"page={next_page}(?:\D|$)", normalized, re.IGNORECASE):
return True
return False
def apply_filters(
self,
page: Page,
@@ -279,74 +403,30 @@ class ListingCollector:
return False
def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
# Считываем ссылки одним проходом по DOM.
# Для IAAI HTML/hydration-извлечение стабильнее, чем прямой DOM eval.
self._accept_cookie_banner(page)
self._wait_for_listing_content(page)
try:
raw_items = page.eval_on_selector_all(
"a[href], [data-href], [href]",
"""
(nodes) => nodes.map((node) => ({
href:
node.getAttribute('href') ||
node.getAttribute('data-href') ||
node.getAttribute('data-url') ||
'',
title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
text: (node.textContent || '').trim(),
}))
""",
)
except Exception as exc:
logger.warning("collect_current_page failed on page %d: %s", page_number, exc)
raw_items = []
total = min(len(raw_items), self.settings.listing.page_link_limit)
links: list[ListingVehicleLink] = []
seen: set[str] = set()
for idx in range(total):
item = raw_items[idx] if isinstance(raw_items[idx], dict) else {}
href = str(item.get("href") or "")
match = VEHICLE_HREF_RE.search(href)
if not match:
continue
lot_number = match.group(1)
absolute = urljoin(self.settings.home_url, match.group(0))
html = self._get_listing_html(page, page_number)
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
if absolute in seen:
continue
seen.add(absolute)
title = first_non_empty([item.get("title"), item.get("text"), ""]) or ""
links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
if not links:
try:
page.wait_for_timeout(1_500)
except Exception:
pass
try:
html = page.content()
except Exception as exc:
logger.debug("page.content() failed on page %d: %s", page_number, exc)
html = ""
if links:
logger.info(
"Page %d: recovered %d vehicle links from HTML fallback",
page_number,
len(links),
)
for absolute, lot_number in self._extract_vehicle_links_from_html(html):
if absolute in seen:
continue
seen.add(absolute)
links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
if len(links) >= self.settings.listing.max_vehicles_per_run:
break
if links:
logger.info(
"Page %d: recovered %d vehicle links from HTML fallback",
page_number,
len(links),
)
next_page_detected = self._has_next_page(page)
next_page_detected = self._has_next_page_from_html(html, current_page_number=page_number)
if not next_page_detected and not html:
next_page_detected = self._has_next_page(page)
return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
@@ -373,12 +453,59 @@ class ListingCollector:
def go_to_next_page(self, page: Page, expected_page_number: int | None = None) -> bool:
# Запоминаем первую ссылку текущей страницы для определения смены контента.
old_first_href = ""
old_fingerprint: tuple[str, ...] = tuple()
try:
first_link = page.locator(VEHICLE_LINK_SELECTOR).first
if first_link.count() > 0:
old_first_href = first_link.get_attribute("href") or ""
except Exception:
pass
old_fingerprint = self._get_vehicle_link_fingerprint(page)
if expected_page_number is not None and expected_page_number > 1:
direct_page_url = self._build_listing_page_url(
page.url or self.settings.listing.cars_url,
expected_page_number,
)
try:
logger.debug("Navigating directly to listing page %d via URL: %s", expected_page_number, direct_page_url)
page.goto(direct_page_url, wait_until="domcontentloaded", timeout=15_000)
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("Direct page-number navigation failed for page %d via %s: %s", expected_page_number, direct_page_url, exc)
next_href = self._extract_next_page_href(page, expected_page_number)
if next_href:
try:
logger.debug("Navigating directly to next listing page: %s", next_href)
page.goto(next_href, wait_until="domcontentloaded", timeout=15_000)
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
self.pacer.after_page_change()
return True
except Exception as exc:
logger.debug("Direct next-page navigation failed for %s: %s", next_href, exc)
if (
expected_page_number is not None
and expected_page_number >= self._DEEP_PAGINATION_DIRECT_ONLY_FROM_PAGE
):
logger.warning(
"Deep pagination direct navigation failed for page %d; skipping flaky UI pagination fallbacks",
expected_page_number,
)
return False
for selector in self._NEXT_PAGE_SELECTORS:
locator = page.locator(selector).first
@@ -398,7 +525,12 @@ class ListingCollector:
except Exception:
continue
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
self.pacer.after_page_change()
return True
@@ -458,7 +590,12 @@ class ListingCollector:
"""
))
if clicked:
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
self.pacer.after_page_change()
return True
except Exception as exc:
@@ -489,7 +626,12 @@ class ListingCollector:
"""
))
if clicked:
if self._wait_for_navigation_result(page, old_first_href, expected_page_number):
if self._wait_for_navigation_result(
page,
old_first_href,
expected_page_number,
old_fingerprint=old_fingerprint,
):
self.pacer.after_page_change()
return True
except Exception as exc:
@@ -574,10 +716,18 @@ class ListingCollector:
len(all_links) >= self.settings.listing.max_vehicles_per_run
or self.settings.listing.collect_current_page_only
or not self.settings.listing.include_pagination
or not page_result.next_page_detected
):
break
if not self.go_to_next_page(page):
blind_page_probe = not page_result.next_page_detected
if not self.go_to_next_page(page, expected_page_number=page_number + 1):
if blind_page_probe:
logger.info(
"Stopping pagination on page %d: direct page probe for %d failed and no next-page control was detected",
page_number,
page_number + 1,
)
break
break
return {

44
iaai_scraper/cli.py
View File

@@ -38,6 +38,17 @@ def build_parser() -> argparse.ArgumentParser:
sync_listing_parser.add_argument("--only-new", choices=["true", "false"], default=None)
sync_listing_parser.add_argument("--output", default=str(default_output_dir / "iaai_sync_listing.json"))
# Новые команды для ingestion pipeline
subparsers.add_parser("discover-vehicles", help="Discover new vehicle URLs from sitemap")
fetch_parser = subparsers.add_parser("fetch-pending", help="Fetch raw data for pending candidates")
fetch_parser.add_argument("--limit", type=int, default=10, help="Max candidates to process")
enrich_parser = subparsers.add_parser("enrich-snapshots", help="Parse and enrich raw snapshots")
enrich_parser.add_argument("--limit", type=int, default=10, help="Max snapshots to process")
subparsers.add_parser("run-pipeline", help="Run full ingestion pipeline (discover -> fetch -> enrich)")
return parser
@@ -64,6 +75,39 @@ def main() -> None:
data = scraper.scrape_vehicle_detail(args.vehicle_url)
elif args.command == "sync-vehicle":
data = scraper.sync_vehicle(args.vehicle_url, lane=args.lane)
elif args.command == "sync-listing":
only_new = None if args.only_new is None else args.only_new == "true"
data = scraper.sync_listing(
make=args.make,
model=args.model,
lane=args.lane,
limit=args.limit,
only_new=only_new,
)
elif args.command == "discover-vehicles":
from .discovery_service import DiscoveryService
discovery = DiscoveryService()
count = discovery.discover_new_vehicles()
print(f"Discovered {count} new vehicle candidates")
return
elif args.command == "fetch-pending":
from .fetch_service import FetchService
fetch = FetchService()
count = fetch.process_pending_candidates(limit=args.limit)
print(f"Successfully fetched {count} candidates")
return
elif args.command == "enrich-snapshots":
from .enrichment_service import EnrichmentService
enrichment = EnrichmentService()
count = enrichment.process_unparsed_snapshots(limit=args.limit)
print(f"Successfully enriched {count} snapshots")
return
elif args.command == "run-pipeline":
from .scheduler_service import SchedulerService
scheduler = SchedulerService()
scheduler.run_full_pipeline()
print("Pipeline completed")
return
else:
only_new = None if args.only_new is None else args.only_new == "true"
data = scraper.sync_listing(

20
iaai_scraper/core/config.py
View File

@@ -106,6 +106,7 @@ class ListingConfig:
cars_url: str = _env_str("IAAI_CARS_LISTING_URL", "https://www.iaai.com/Vehiclelisting/Cars")
max_pages_per_run: int = _env_int("IAAI_MAX_PAGES_PER_RUN", 9999)
max_vehicles_per_run: int = _env_int("IAAI_MAX_VEHICLES_PER_RUN", 50000)
resume_max_nav_pages: int = _env_int("IAAI_LISTING_RESUME_MAX_NAV_PAGES", 90)
page_link_limit: int = _env_int("IAAI_PAGE_LINK_LIMIT", 500)
include_pagination: bool = _env_bool("IAAI_INCLUDE_PAGINATION", True)
collect_current_page_only: bool = _env_bool("IAAI_COLLECT_CURRENT_PAGE_ONLY", False)
@@ -118,6 +119,22 @@ class ListingConfig:
listing_segments_json: str = _env_str("IAAI_LISTING_SEGMENTS", "")
@dataclass(slots=True)
class DiscoveryConfig:
mode: str = _env_str("IAAI_DISCOVERY_MODE", "sitemap")
hourly_mode: str = _env_str("IAAI_HOURLY_MODE", "rolling_refresh")
hourly_refresh_batch_size: int = _env_int("IAAI_HOURLY_REFRESH_BATCH_SIZE", 3000)
sitemap_timeout_seconds: int = _env_int("IAAI_SITEMAP_TIMEOUT_SECONDS", 30)
sitemap_retry_attempts: int = _env_int("IAAI_SITEMAP_RETRY_ATTEMPTS", 4)
sitemap_retry_backoff_seconds: float = _env_float("IAAI_SITEMAP_RETRY_BACKOFF_SECONDS", 1.25)
sitemap_direct_probe_limit: int = _env_int("IAAI_SITEMAP_DIRECT_PROBE_LIMIT", 12)
sitemap_direct_probe_stop_after_misses: int = _env_int(
"IAAI_SITEMAP_DIRECT_PROBE_STOP_AFTER_MISSES",
3,
)
sitemap_use_curl_cffi: bool = _env_bool("IAAI_SITEMAP_USE_CURL_CFFI", True)
# Список брендов IAAI для автоматической сегментации.
# Покрывает >99% автомобилей на сайте. Порядок: от крупных к мелким.
IAAI_DEFAULT_MAKES: tuple[str, ...] = (
@@ -217,6 +234,7 @@ class CeleryConfig:
result_backend: str = _env_str("CELERY_RESULT_BACKEND", "")
task_soft_time_limit: int = _env_int("CELERY_TASK_SOFT_TIME_LIMIT", 3300)
task_time_limit: int = _env_int("CELERY_TASK_TIME_LIMIT", 3600)
task_stall_timeout_seconds: int = _env_int("CELERY_TASK_STALL_TIMEOUT_SECONDS", 600)
worker_concurrency: int = _env_int("CELERY_WORKER_CONCURRENCY", 4)
worker_max_tasks_per_child: int = _env_int("CELERY_WORKER_MAX_TASKS_PER_CHILD", 5)
broker_visibility_timeout: int = _env_int("CELERY_BROKER_VISIBILITY_TIMEOUT", 7200)
@@ -224,6 +242,7 @@ class CeleryConfig:
beat_sync_limit: int | None = _env_int("CELERY_BEAT_SYNC_LIMIT", 0) or None
batch_size: int = _env_int("CELERY_BATCH_SIZE", 50)
parallel_tabs: int = _env_int("IAAI_PARALLEL_TABS", 8)
parallel_segments: bool = _env_bool("IAAI_PARALLEL_SEGMENTS", False)
block_resources: bool = _env_bool("IAAI_BLOCK_RESOURCES", True)
@@ -278,6 +297,7 @@ class Settings:
capture: CaptureConfig = field(default_factory=CaptureConfig)
pace: HumanPaceConfig = field(default_factory=HumanPaceConfig)
listing: ListingConfig = field(default_factory=ListingConfig)
discovery: DiscoveryConfig = field(default_factory=DiscoveryConfig)
database: DatabaseConfig = field(default_factory=DatabaseConfig)
redis: RedisConfig = field(default_factory=RedisConfig)
celery: CeleryConfig = field(default_factory=CeleryConfig)

3
iaai_scraper/core/utils.py
View File

@@ -17,7 +17,8 @@ def first_non_empty(values: Iterable[Any]) -> Any | None:
return None
# Регулярные выражения для lot и price.
# Регулярные выражения для VIN, lot и price.
VIN_RE = re.compile(r"\b([A-HJ-NPR-Z0-9]{17})\b", re.IGNORECASE)
LOT_RE = re.compile(r"\b(\d{7,10})\b")
PRICE_RE = re.compile(r"\$\s?([\d,]+(?:\.\d{1,2})?)")

17
iaai_scraper/discovery/__init__.py Normal file
View File

@@ -0,0 +1,17 @@
from .sitemap import (
DEFAULT_SITEMAP_INDEX_URL,
SitemapDiscoveryError,
SitemapBlockedError,
SitemapDiscoveryResult,
discover_vehicle_urls_from_sitemap,
discover_vehicle_urls_from_sitemap_with_stats,
)
__all__ = [
"DEFAULT_SITEMAP_INDEX_URL",
"SitemapBlockedError",
"SitemapDiscoveryError",
"SitemapDiscoveryResult",
"discover_vehicle_urls_from_sitemap",
"discover_vehicle_urls_from_sitemap_with_stats",
]

441
iaai_scraper/discovery/sitemap.py Normal file
View File

@@ -0,0 +1,441 @@
from __future__ import annotations
import gzip
import io
import logging
import random
import re
import time
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from typing import Iterable
from urllib.error import HTTPError, URLError
from urllib.parse import urljoin, urlsplit, urlunsplit
from urllib.request import Request, urlopen
from ..core.config import Settings
try:
from curl_cffi import requests as curl_requests
except ImportError: # pragma: no cover - optional runtime dependency guard
curl_requests = None
logger = logging.getLogger("iaai_scraper.discovery.sitemap")
DEFAULT_SITEMAP_INDEX_URL = "https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap_index.xml"
_LOC_TAG_RE = re.compile(rb"<loc>\s*(.*?)\s*</loc>", re.IGNORECASE | re.DOTALL)
_XML_PREFIX_RE = re.compile(rb"^\s*(<\?xml\b.*?\?>)?\s*<", re.IGNORECASE | re.DOTALL)
_BLOCK_MARKERS = (
b"pardon our interruption",
b"incapsula",
b"please stand by",
b"_incapsula_resource",
b"as you were browsing something about your browser",
)
_HTML_MARKERS = (b"<html", b"<!doctype html", b"<script", b"document.getelementsbyclassname")
_CURL_IMPERSONATE_CHOICES = ("chrome136", "chrome133", "chrome131", "safari184")
class SitemapDiscoveryError(RuntimeError):
pass
class SitemapBlockedError(SitemapDiscoveryError):
pass
@dataclass(slots=True)
class SitemapFetchResult:
url: str
payload: bytes
source: str
blocked: bool = False
status_code: int | None = None
content_type: str | None = None
@dataclass(slots=True)
class SitemapDiscoveryStats:
transport: str
fetched_sitemaps: int = 0
blocked_sitemaps: int = 0
malformed_sitemaps: int = 0
direct_probe_hits: int = 0
direct_probe_misses: int = 0
@dataclass(slots=True)
class SitemapDiscoveryResult:
vehicle_urls: list[str]
stats: SitemapDiscoveryStats
_DEFAULT_HEADERS = {
"Accept": "application/xml,text/xml,application/xhtml+xml,text/html;q=0.9,*/*;q=0.8",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en-US,en;q=0.9",
"Cache-Control": "no-cache",
"Pragma": "no-cache",
"Upgrade-Insecure-Requests": "1",
}
class _SitemapDownloader:
def __init__(self, settings: Settings) -> None:
self.settings = settings
self.discovery = settings.discovery
self.proxy_url = settings.proxy.server
self.proxy_auth = None
if settings.proxy.username:
password = settings.proxy.password or ""
self.proxy_auth = (settings.proxy.username, password)
self._transport_name = self._resolve_transport_name()
@property
def transport_name(self) -> str:
return self._transport_name
def _resolve_transport_name(self) -> str:
if self.discovery.sitemap_use_curl_cffi and curl_requests is not None:
return "curl_cffi"
return "urllib"
def fetch(self, url: str) -> SitemapFetchResult:
attempts = max(1, int(self.discovery.sitemap_retry_attempts))
last_exc: Exception | None = None
for attempt in range(1, attempts + 1):
try:
result = self._fetch_once(url)
if result.blocked:
raise SitemapBlockedError(f"Anti-bot page returned for {url}")
return result
except SitemapBlockedError as exc:
last_exc = exc
logger.warning(
"Sitemap fetch blocked (attempt %d/%d, transport=%s): %s",
attempt,
attempts,
self.transport_name,
url,
)
except Exception as exc:
last_exc = exc
logger.warning(
"Sitemap fetch failed (attempt %d/%d, transport=%s): %s -> %s",
attempt,
attempts,
self.transport_name,
url,
exc,
)
if attempt >= attempts:
break
time.sleep(self._backoff_delay(attempt))
if last_exc is None:
raise SitemapDiscoveryError(f"Failed to fetch sitemap: {url}")
if isinstance(last_exc, SitemapDiscoveryError):
raise last_exc
raise SitemapDiscoveryError(f"Failed to fetch sitemap {url}: {last_exc}") from last_exc
def _fetch_once(self, url: str) -> SitemapFetchResult:
if self.transport_name == "curl_cffi":
return self._fetch_with_curl_cffi(url)
return self._fetch_with_urllib(url)
def _build_headers(self) -> dict[str, str]:
headers = dict(_DEFAULT_HEADERS)
headers["User-Agent"] = self.settings.fingerprint.user_agent
return headers
def _fetch_with_curl_cffi(self, url: str) -> SitemapFetchResult:
assert curl_requests is not None
response = curl_requests.get(
url,
headers=self._build_headers(),
timeout=self.discovery.sitemap_timeout_seconds,
impersonate=random.choice(_CURL_IMPERSONATE_CHOICES),
proxies={"http": self.proxy_url, "https": self.proxy_url} if self.proxy_url else None,
proxy_auth=self.proxy_auth,
allow_redirects=True,
)
payload = self._decode_payload(
payload=response.content,
encoding=(response.headers.get("Content-Encoding") or ""),
url=url,
)
blocked = _looks_like_block_page(payload)
return SitemapFetchResult(
url=url,
payload=payload,
source="curl_cffi",
blocked=blocked,
status_code=int(response.status_code),
content_type=response.headers.get("Content-Type"),
)
def _fetch_with_urllib(self, url: str) -> SitemapFetchResult:
request = Request(url, headers=self._build_headers())
try:
with urlopen(request, timeout=self.discovery.sitemap_timeout_seconds) as response:
payload = response.read()
decoded = self._decode_payload(
payload=payload,
encoding=(response.headers.get("Content-Encoding") or ""),
url=url,
)
return SitemapFetchResult(
url=url,
payload=decoded,
source="urllib",
blocked=_looks_like_block_page(decoded),
status_code=getattr(response, "status", None),
content_type=response.headers.get("Content-Type"),
)
except HTTPError as exc:
payload = exc.read() if hasattr(exc, "read") else b""
decoded = self._decode_payload(payload=payload, encoding=exc.headers.get("Content-Encoding", ""), url=url)
blocked = exc.code in {403, 429} or _looks_like_block_page(decoded)
if blocked:
raise SitemapBlockedError(f"HTTP {exc.code} for {url}") from exc
raise SitemapDiscoveryError(f"HTTP {exc.code} for {url}") from exc
except URLError as exc:
raise SitemapDiscoveryError(f"Network error for {url}: {exc}") from exc
@staticmethod
def _decode_payload(*, payload: bytes, encoding: str, url: str) -> bytes:
normalized = (encoding or "").lower().strip()
if normalized == "gzip" or url.lower().endswith(".gz"):
try:
return gzip.GzipFile(fileobj=io.BytesIO(payload)).read()
except OSError:
return payload
return payload
def _backoff_delay(self, attempt: int) -> float:
base = max(0.1, float(self.discovery.sitemap_retry_backoff_seconds))
jitter = random.uniform(0.05, 0.35)
return base * (2 ** (attempt - 1)) + jitter
def _is_vehicle_sitemap_url(url: str) -> bool:
lowered = url.strip().lower()
if "sitemapbranches" in lowered or "sitemapauctions" in lowered:
return False
return lowered.endswith(".xml") or lowered.endswith(".xml.gz")
def _normalize_vehicle_url(url: str) -> str:
parts = urlsplit(url.strip())
return urlunsplit((parts.scheme, parts.netloc, parts.path, "", ""))
def _local_name(tag: str) -> str:
if "}" in tag:
return tag.rsplit("}", 1)[1]
return tag
def _looks_like_block_page(payload: bytes) -> bool:
sample = payload[:8192].lower()
if any(marker in sample for marker in _BLOCK_MARKERS):
return True
if any(marker in sample for marker in _HTML_MARKERS) and b"<loc>" not in sample:
return True
return False
def _iter_loc_values_fallback(xml_bytes: bytes) -> Iterable[str]:
for match in _LOC_TAG_RE.finditer(xml_bytes):
try:
value = match.group(1).decode("utf-8", errors="ignore").strip()
except Exception:
continue
if value:
yield value
def _iter_loc_values(xml_bytes: bytes) -> Iterable[str]:
if _looks_like_block_page(xml_bytes):
raise SitemapBlockedError("Anti-bot content returned instead of sitemap XML")
if not _XML_PREFIX_RE.search(xml_bytes[:256]):
fallback_values = list(_iter_loc_values_fallback(xml_bytes))
if fallback_values:
yield from fallback_values
return
try:
root = ET.fromstring(xml_bytes)
except ET.ParseError as exc:
fallback_values = list(_iter_loc_values_fallback(xml_bytes))
if fallback_values:
logger.warning(
"Falling back to regex sitemap loc extraction after XML parse error: %s",
exc,
)
yield from fallback_values
return
raise SitemapDiscoveryError(f"Invalid sitemap XML: {exc}") from exc
for element in root.iter():
if _local_name(element.tag) != "loc":
continue
if not element.text:
continue
value = element.text.strip()
if value:
yield value
def _filter_vehicle_urls(urls: Iterable[str]) -> list[str]:
result: list[str] = []
seen: set[str] = set()
for url in urls:
normalized = _normalize_vehicle_url(url)
if "/VehicleDetail/" not in normalized and "/vehicledetail/" not in normalized:
continue
if normalized in seen:
continue
seen.add(normalized)
result.append(normalized)
return result
def _looks_like_vehicle_detail_sitemap(urls: list[str]) -> bool:
return any("/vehicledetail/" in url.lower() for url in urls)
def _derive_direct_probe_urls(index_url: str, limit: int) -> list[str]:
base_dir = index_url.rsplit("/", 1)[0] + "/"
return [urljoin(base_dir, f"sitemap{idx}.xml") for idx in range(1, max(1, limit) + 1)]
def _discover_sitemap_urls(index_url: str, downloader: _SitemapDownloader, stats: SitemapDiscoveryStats) -> list[str]:
logger.info("Downloading sitemap index: %s", index_url)
try:
index_result = downloader.fetch(index_url)
sitemap_urls = [url for url in _iter_loc_values(index_result.payload) if _is_vehicle_sitemap_url(url)]
if sitemap_urls:
return sitemap_urls
except SitemapBlockedError as exc:
stats.blocked_sitemaps += 1
logger.warning("Sitemap index blocked, switching to direct probe: %s", exc)
except SitemapDiscoveryError as exc:
stats.malformed_sitemaps += 1
logger.warning("Sitemap index unusable, switching to direct probe: %s", exc)
logger.warning("Sitemap index returned no usable sitemap URLs; switching to direct probe")
return _probe_direct_sitemap_urls(index_url, downloader, stats)
def _probe_direct_sitemap_urls(
index_url: str,
downloader: _SitemapDownloader,
stats: SitemapDiscoveryStats,
) -> list[str]:
discovered: list[str] = []
consecutive_misses = 0
probe_urls = _derive_direct_probe_urls(index_url, downloader.discovery.sitemap_direct_probe_limit)
for sitemap_url in probe_urls:
try:
result = downloader.fetch(sitemap_url)
raw_urls = list(_iter_loc_values(result.payload))
except SitemapBlockedError:
stats.blocked_sitemaps += 1
consecutive_misses += 1
stats.direct_probe_misses += 1
if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
break
continue
except SitemapDiscoveryError:
consecutive_misses += 1
stats.direct_probe_misses += 1
if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
break
continue
if not raw_urls:
consecutive_misses += 1
stats.direct_probe_misses += 1
if consecutive_misses >= downloader.discovery.sitemap_direct_probe_stop_after_misses:
break
continue
consecutive_misses = 0
stats.direct_probe_hits += 1
discovered.append(sitemap_url)
return discovered
def _collect_vehicle_urls_from_sitemap(
sitemap_url: str,
downloader: _SitemapDownloader,
stats: SitemapDiscoveryStats,
) -> list[str]:
logger.info("Downloading sitemap: %s", sitemap_url)
try:
result = downloader.fetch(sitemap_url)
raw_urls = list(_iter_loc_values(result.payload))
except SitemapBlockedError as exc:
stats.blocked_sitemaps += 1
logger.warning("Skipping blocked sitemap %s: %s", sitemap_url, exc)
return []
except SitemapDiscoveryError as exc:
stats.malformed_sitemaps += 1
logger.warning("Skipping malformed sitemap %s: %s", sitemap_url, exc)
return []
stats.fetched_sitemaps += 1
vehicle_urls = _filter_vehicle_urls(raw_urls)
if raw_urls and not vehicle_urls and not _looks_like_vehicle_detail_sitemap(raw_urls):
logger.info("Skipping non-vehicle sitemap %s", sitemap_url)
return []
logger.info("Sitemap %s yielded %d vehicle URLs", sitemap_url, len(vehicle_urls))
return vehicle_urls
def discover_vehicle_urls_from_sitemap(
index_url: str = DEFAULT_SITEMAP_INDEX_URL,
*,
settings: Settings | None = None,
) -> list[str]:
result = discover_vehicle_urls_from_sitemap_with_stats(index_url=index_url, settings=settings)
return result.vehicle_urls
def discover_vehicle_urls_from_sitemap_with_stats(
index_url: str = DEFAULT_SITEMAP_INDEX_URL,
*,
settings: Settings | None = None,
) -> SitemapDiscoveryResult:
runtime_settings = settings or Settings()
downloader = _SitemapDownloader(runtime_settings)
stats = SitemapDiscoveryStats(transport=downloader.transport_name)
sitemap_urls = _discover_sitemap_urls(index_url, downloader, stats)
if not sitemap_urls:
raise SitemapDiscoveryError("Sitemap discovery found no sitemap URLs")
all_vehicle_urls: list[str] = []
for sitemap_url in sitemap_urls:
all_vehicle_urls.extend(_collect_vehicle_urls_from_sitemap(sitemap_url, downloader, stats))
deduped = _filter_vehicle_urls(all_vehicle_urls)
if not deduped:
raise SitemapDiscoveryError(
"No vehicle detail URLs discovered from vehicle sitemaps; likely anti-bot or upstream sitemap issue"
)
logger.info(
"Sitemap discovery done: %d vehicle URLs (transport=%s fetched=%d blocked=%d malformed=%d direct_hits=%d direct_misses=%d)",
len(deduped),
stats.transport,
stats.fetched_sitemaps,
stats.blocked_sitemaps,
stats.malformed_sitemaps,
stats.direct_probe_hits,
stats.direct_probe_misses,
)
return SitemapDiscoveryResult(vehicle_urls=deduped, stats=stats)

129
iaai_scraper/discovery_service.py Normal file
View File

@@ -0,0 +1,129 @@
import logging
from datetime import datetime, timezone
from typing import List
from sqlalchemy.orm import Session
from .core.config import settings
from .discovery import discover_vehicle_urls_from_sitemap_with_stats, SitemapDiscoveryError
from .storage.db import get_db_session
from .storage.models import VehicleCandidate
logger = logging.getLogger("iaai_scraper.discovery_service")
class DiscoveryService:
"""Сервис для обнаружения новых автомобилей через sitemap IAAI."""
def __init__(self):
self.db: Session = get_db_session()
def discover_new_vehicles(self, max_urls: int = None) -> int:
"""
Обнаруживает новые URL автомобилей и добавляет их в vehicle_candidates.
Args:
max_urls: Максимальное количество URL для обработки (None = все)
Returns:
Количество добавленных кандидатов
"""
try:
logger.info("Starting vehicle discovery from sitemap")
result = discover_vehicle_urls_from_sitemap_with_stats()
if not result.urls:
logger.info("No new URLs discovered")
return 0
# Ограничиваем количество URL
urls_to_process = result.urls[:max_urls] if max_urls else result.urls
logger.info(f"Discovered {len(urls_to_process)} potential vehicle URLs (from {len(result.urls)} total)")
# Фильтруем уже существующие кандидаты
existing_urls = self._get_existing_candidate_urls(urls_to_process)
new_urls = [url for url in urls_to_process if url not in existing_urls]
if not new_urls:
logger.info("All discovered URLs already exist as candidates")
return 0
# Добавляем новых кандидатов
added_count = self._add_candidates(new_urls)
logger.info(f"Added {added_count} new vehicle candidates")
return added_count
except SitemapDiscoveryError as e:
logger.error(f"Sitemap discovery failed: {e}")
raise
except Exception as e:
logger.error(f"Unexpected error during discovery: {e}")
raise
def _get_existing_candidate_urls(self, urls: List[str]) -> set:
"""Получает множество уже существующих URL кандидатов."""
if not urls:
return set()
# Разбиваем на батчи для эффективности
batch_size = 1000
existing = set()
for i in range(0, len(urls), batch_size):
batch = urls[i:i + batch_size]
result = self.db.query(VehicleCandidate.url).filter(
VehicleCandidate.url.in_(batch)
).all()
existing.update(row[0] for row in result)
return existing
def _add_candidates(self, urls: List[str]) -> int:
"""Добавляет новые кандидаты в базу данных."""
now = datetime.now(timezone.utc)
candidates = [
VehicleCandidate(
url=url,
discovered_at=now,
status="pending",
priority=0
)
for url in urls
]
self.db.add_all(candidates)
self.db.commit()
return len(candidates)
def get_pending_candidates(self, limit: int = 100) -> List[VehicleCandidate]:
"""Получает кандидатов в статусе 'pending' для обработки."""
return self.db.query(VehicleCandidate).filter(
VehicleCandidate.status == "pending"
).order_by(VehicleCandidate.priority.desc(), VehicleCandidate.discovered_at).limit(limit).all()
def mark_candidate_processing(self, candidate_id: int):
"""Помечает кандидата как обрабатываемого."""
self.db.query(VehicleCandidate).filter(
VehicleCandidate.id == candidate_id
).update({
"status": "processing",
"last_attempt_at": datetime.now(timezone.utc),
"attempts": VehicleCandidate.attempts + 1
})
self.db.commit()
def mark_candidate_processed(self, candidate_id: int):
"""Помечает кандидата как обработанного."""
self.db.query(VehicleCandidate).filter(
VehicleCandidate.id == candidate_id
).update({"status": "processed"})
self.db.commit()
def mark_candidate_failed(self, candidate_id: int):
"""Помечает кандидата как неудачного."""
self.db.query(VehicleCandidate).filter(
VehicleCandidate.id == candidate_id
).update({"status": "failed"})
self.db.commit()

140
iaai_scraper/enrichment_service.py Normal file
View File

@@ -0,0 +1,140 @@
import json
import logging
from datetime import datetime, timezone
from typing import Optional
from sqlalchemy.orm import Session
from .core.config import settings
from .parsing.mapper import CarMapper
from .parsing.parser import VehicleParser
from .storage.db import get_db_session
from .storage.models import VehicleRawSnapshot, VehicleParseResult, Car
from .storage.schemas import CarRecord
logger = logging.getLogger("iaai_scraper.enrichment_service")
class EnrichmentService:
"""Сервис для парсинга сырых данных и обогащения автомобилей."""
def __init__(self):
self.db: Session = get_db_session()
self.parser = VehicleParser()
self.mapper = CarMapper()
def enrich_vehicle(self, snapshot: VehicleRawSnapshot) -> Optional[VehicleParseResult]:
"""
Парсит snapshot и сохраняет результат.
Args:
snapshot: Raw snapshot для парсинга
Returns:
VehicleParseResult если парсинг успешен
"""
logger.info(f"Enriching snapshot {snapshot.id} for candidate {snapshot.candidate_id}")
if not snapshot.success or not snapshot.raw_data:
logger.warning(f"Snapshot {snapshot.id} has no data to parse")
return self._save_parse_result(snapshot.id, False, None, "No raw data available")
try:
# Парсим данные
parsed_data = self._parse_raw_data(snapshot.raw_data)
if not parsed_data:
return self._save_parse_result(snapshot.id, False, None, "Parsing failed")
# Маппим в CarRecord
car_record = self._map_to_car_record(parsed_data)
if not car_record:
return self._save_parse_result(snapshot.id, False, None, "Mapping failed")
# Сохраняем в cars таблицу
self._save_car(car_record)
# Сохраняем успешный результат парсинга
return self._save_parse_result(snapshot.id, True, json.dumps(parsed_data))
except Exception as e:
error_msg = f"Unexpected error during enrichment: {str(e)}"
logger.error(f"Error enriching snapshot {snapshot.id}: {error_msg}")
return self._save_parse_result(snapshot.id, False, None, error_msg)
def _parse_raw_data(self, raw_data: str) -> Optional[dict]:
"""Парсит сырые данные в словарь."""
try:
# Если это JSON от browser capture
if raw_data.strip().startswith('{'):
data = json.loads(raw_data)
# Извлекаем vehicle_summary из scrape result
return data.get("vehicle_summary", {})
# Если HTML, используем VehicleParser
parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
return parsed.get("vehicle_summary", {})
except json.JSONDecodeError:
# Если не JSON, пробуем как HTML
try:
parsed = self.parser.parse_vehicle_page(raw_data, "dummy_url")
return parsed.get("vehicle_summary", {})
except Exception:
return None
def _map_to_car_record(self, parsed_data: dict) -> Optional[CarRecord]:
"""Маппит parsed data в CarRecord."""
try:
# Используем CarMapper
payload_insights = {} # TODO: extract from raw data if needed
return self.mapper.map_to_car_record("dummy_url", parsed_data, payload_insights)
except Exception as e:
logger.error(f"Mapping failed: {e}")
return None
def _save_car(self, car_record: CarRecord):
"""Сохраняет CarRecord в базу данных."""
# Используем PersistenceService
from .storage.db import PersistenceService
from .core.config import settings
persistence = PersistenceService(settings)
persistence.create_tables()
upsert_result = persistence.upsert_car(car_record)
logger.info(f"Car upserted: {upsert_result}")
def _save_parse_result(self, snapshot_id: int, success: bool,
parsed_data: Optional[str], error_message: Optional[str] = None) -> VehicleParseResult:
"""Сохраняет результат парсинга."""
result = VehicleParseResult(
snapshot_id=snapshot_id,
success=success,
parsed_data=parsed_data,
error_message=error_message
)
self.db.add(result)
self.db.commit()
self.db.refresh(result)
return result
def process_unparsed_snapshots(self, limit: int = 10) -> int:
"""
Обрабатывает snapshots без результатов парсинга.
Returns:
Количество успешно обогащенных snapshots
"""
# Находим snapshots без parse results
snapshots = self.db.query(VehicleRawSnapshot).outerjoin(
VehicleParseResult, VehicleRawSnapshot.id == VehicleParseResult.snapshot_id
).filter(
VehicleRawSnapshot.success == True,
VehicleParseResult.id.is_(None)
).limit(limit).all()
enriched_count = 0
for snapshot in snapshots:
result = self.enrich_vehicle(snapshot)
if result and result.success:
enriched_count += 1
return enriched_count

119
iaai_scraper/fetch_service.py Normal file
View File

@@ -0,0 +1,119 @@
import logging
from datetime import datetime, timezone
from typing import Optional
from sqlalchemy.orm import Session
from .core.config import settings
from .scraper import IAAIScraper
from .storage.db import get_db_session
from .storage.models import VehicleCandidate, VehicleRawSnapshot
logger = logging.getLogger("iaai_scraper.fetch_service")
class FetchService:
"""Сервис для захвата сырых данных автомобилей (HTTP/browser fallback)."""
def __init__(self):
self.db: Session = get_db_session()
self.scraper = IAAIScraper()
def fetch_vehicle_data(self, candidate: VehicleCandidate) -> Optional[VehicleRawSnapshot]:
"""
Захватывает данные для кандидата автомобиля.
Args:
candidate: Кандидат для обработки
Returns:
VehicleRawSnapshot если захват успешен, None если неудача
"""
logger.info(f"Fetching data for candidate {candidate.id}: {candidate.url}")
try:
# Сначала пытаемся HTTP fast-path
raw_data = self._try_http_capture(candidate.url)
if raw_data:
return self._save_snapshot(candidate.id, "http", True, raw_data)
# Если HTTP не сработал, используем browser fallback
logger.info(f"HTTP failed for {candidate.url}, trying browser fallback")
raw_data = self._try_browser_capture(candidate.url)
if raw_data:
return self._save_snapshot(candidate.id, "browser", True, raw_data)
# Оба метода failed
logger.warning(f"Both HTTP and browser capture failed for {candidate.url}")
self._save_snapshot(candidate.id, "browser", False, None, "Both capture methods failed")
return None
except Exception as e:
error_msg = f"Unexpected error during capture: {str(e)}"
logger.error(f"Error fetching {candidate.url}: {error_msg}")
self._save_snapshot(candidate.id, "unknown", False, None, error_msg)
return None
def _try_http_capture(self, url: str) -> Optional[str]:
"""Пытается захватить данные через HTTP."""
try:
# Используем существующий метод из scraper
# Но нам нужно инициализировать scraper с сессией
# Пока заглушка - интегрируем позже
# Для теста вернем None, чтобы использовать browser
return None
except Exception as e:
logger.debug(f"HTTP capture failed for {url}: {e}")
return None
def _try_browser_capture(self, url: str) -> Optional[str]:
"""Пытается захватить данные через browser."""
try:
# Используем scrape_vehicle_detail из scraper
with IAAIScraper() as scraper:
result = scraper.scrape_vehicle_detail(url)
# Возвращаем HTML или JSON данные
return result.get("raw_html") or json.dumps(result)
except Exception as e:
logger.debug(f"Browser capture failed for {url}: {e}")
return None
def _save_snapshot(self, candidate_id: int, method: str, success: bool,
raw_data: Optional[str], error_message: Optional[str] = None) -> VehicleRawSnapshot:
"""Сохраняет snapshot в базу данных."""
snapshot = VehicleRawSnapshot(
candidate_id=candidate_id,
method=method,
success=success,
raw_data=raw_data,
error_message=error_message
)
self.db.add(snapshot)
self.db.commit()
self.db.refresh(snapshot)
return snapshot
def process_pending_candidates(self, limit: int = 10) -> int:
"""
Обрабатывает ожидающих кандидатов.
Returns:
Количество успешно обработанных кандидатов
"""
from .discovery_service import DiscoveryService
discovery = DiscoveryService()
candidates = discovery.get_pending_candidates(limit)
processed_count = 0
for candidate in candidates:
discovery.mark_candidate_processing(candidate.id)
snapshot = self.fetch_vehicle_data(candidate)
if snapshot and snapshot.success:
discovery.mark_candidate_processed(candidate.id)
processed_count += 1
else:
discovery.mark_candidate_failed(candidate.id)
return processed_count

69
iaai_scraper/scheduler_service.py Normal file
View File

@@ -0,0 +1,69 @@
import logging
import time
from datetime import datetime, timezone, timedelta
from .core.config import settings
from .discovery_service import DiscoveryService
from .fetch_service import FetchService
from .enrichment_service import EnrichmentService
logger = logging.getLogger("iaai_scraper.scheduler_service")
class SchedulerService:
"""Сервис для планирования и координации ingestion pipeline."""
def __init__(self):
self.discovery = DiscoveryService()
self.fetch = FetchService()
self.enrichment = EnrichmentService()
def run_full_pipeline(self):
"""Запускает полный цикл ingestion: discovery -> fetch -> enrichment."""
logger.info("Starting full ingestion pipeline")
try:
# 1. Discovery phase
logger.info("Phase 1: Discovery")
discovered_count = self.discovery.discover_new_vehicles()
logger.info(f"Discovered {discovered_count} new candidates")
# 2. Fetch phase
logger.info("Phase 2: Fetch")
fetched_count = self.fetch.process_pending_candidates(limit=50)
logger.info(f"Successfully fetched {fetched_count} candidates")
# 3. Enrichment phase
logger.info("Phase 3: Enrichment")
enriched_count = self.enrichment.process_unparsed_snapshots(limit=50)
logger.info(f"Successfully enriched {enriched_count} snapshots")
logger.info("Ingestion pipeline completed")
except Exception as e:
logger.error(f"Pipeline failed: {e}")
raise
def run_continuous_pipeline(self, interval_minutes: int = 30):
"""Запускает непрерывный цикл ingestion с интервалом."""
logger.info(f"Starting continuous ingestion pipeline with {interval_minutes}min intervals")
while True:
try:
self.run_full_pipeline()
except Exception as e:
logger.error(f"Pipeline iteration failed: {e}")
logger.info(f"Sleeping for {interval_minutes} minutes")
time.sleep(interval_minutes * 60)
def run_targeted_enrichment(self):
"""Запускает только enrichment для существующих snapshots."""
logger.info("Running targeted enrichment")
enriched_count = self.enrichment.process_unparsed_snapshots(limit=100)
logger.info(f"Enriched {enriched_count} snapshots")
def cleanup_old_data(self, days_to_keep: int = 30):
"""Очищает старые данные (опционально)."""
# TODO: implement if needed
pass

782
iaai_scraper/scraper.py
View File

File diff suppressed because it is too large Load Diff

87
iaai_scraper/storage/db.py
View File

@@ -14,6 +14,13 @@ from .schemas import CarRecord
logger = logging.getLogger("iaai_scraper.db")
def get_db_session() -> Session:
"""Получить новую сессию базы данных."""
settings = Settings()
persistence = PersistenceService(settings)
return persistence.session_factory()
CAR_DB_FIELDS = {
col.key for col in Car.__table__.columns
if col.key not in ("id",)
@@ -517,21 +524,38 @@ class PersistenceService:
is_postgres = "postgresql" in self.settings.database.url
with self.session_scope() as session:
# Страховка от ложного mark_sold при битом/неполном full-scan:
# если текущий список активных URL аномально мал относительно уже активных машин в БД,
# ничего не помечаем проданным.
active_db_count = int(session.execute(
select(func.count())
.select_from(Car)
.where(Car.is_sold == False) # noqa: E712
.where(Car.origin_id.like(f"{lane}:%"))
).scalar_one() or 0)
current_active_count = len(active_origin_urls)
if active_db_count >= 1000 and current_active_count < max(500, int(active_db_count * 0.25)):
logger.warning(
"Skipping mark_sold: suspiciously small active set (%d URLs vs %d active in DB)",
current_active_count,
active_db_count,
)
return 0
if is_postgres:
# Создаём временную таблицу с активными URL.
session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT NOT NULL) ON COMMIT DROP"))
session.execute(text("CREATE TEMP TABLE IF NOT EXISTS _active_urls (url TEXT PRIMARY KEY) ON COMMIT DROP"))
session.execute(text("TRUNCATE _active_urls"))
# Вставляем активные URL чанками.
# Вставляем активные URL чанками через executemany.
url_list = list(active_origin_urls)
for i in range(0, len(url_list), _IN_CHUNK_SIZE):
chunk = url_list[i:i + _IN_CHUNK_SIZE]
values = ",".join(f"(:{f'u{j}'})" for j in range(len(chunk)))
params = {f"u{j}": url for j, url in enumerate(chunk)}
session.execute(text(f"INSERT INTO _active_urls (url) VALUES {values}"), params)
# Создаём индекс на временной таблице для ускорения JOIN.
session.execute(text("CREATE INDEX IF NOT EXISTS _ix_active_urls ON _active_urls (url)"))
session.execute(
text("INSERT INTO _active_urls (url) VALUES (:url) ON CONFLICT DO NOTHING"),
[{"url": url} for url in chunk],
)
# Массовая пометка проданных в PostgreSQL.
result = session.execute(text("""
@@ -543,10 +567,10 @@ class PersistenceService:
LEFT JOIN _active_urls a ON c.origin_url = a.url
WHERE a.url IS NULL
AND c.is_sold = FALSE
AND c.origin_id LIKE 'iaai:%%'
AND c.origin_id LIKE :lane_prefix
) sub
WHERE cars.id = sub.id
"""))
"""), {"lane_prefix": f"{lane}:%"})
count = result.rowcount or 0
else:
# Упрощённый путь для SQLite.
@@ -554,7 +578,7 @@ class PersistenceService:
update(Car)
.where(Car.origin_url.notin_(active_origin_urls))
.where(Car.is_sold == False) # noqa: E712
.where(Car.origin_id.like("iaai:%"))
.where(Car.origin_id.like(f"{lane}:%"))
.values(is_sold=True)
)
result = session.execute(stmt)
@@ -573,4 +597,43 @@ class PersistenceService:
result = session.execute(
select(Car.origin_id).where(Car.origin_id.like(f"{prefix}%")).execution_options(yield_per=10000)
)
return {str(row[0]) for row in result if row and row[0]}
return {str(row[0]) for row in result if row and row[0]}
def get_all_active_origin_urls_for_lane(self, prefix: str = "iaai:") -> set[str]:
"""Возвращает все активные (не sold) origin_url для указанного lane/prefix."""
with self.session_scope() as session:
result = session.execute(
select(Car.origin_url)
.where(Car.origin_id.like(f"{prefix}%"))
.where(Car.is_sold == False) # noqa: E712
.execution_options(yield_per=10000)
)
return {str(row[0]) for row in result if row and row[0]}
def get_active_origin_urls_batch_for_refresh(
self,
*,
prefix: str = "iaai:",
offset: int = 0,
limit: int = 3000,
) -> list[str]:
"""Возвращает батч активных origin_url для циклического hourly refresh."""
with self.session_scope() as session:
rows = session.execute(
select(Car.origin_url)
.where(Car.origin_id.like(f"{prefix}%"))
.where(Car.is_sold == False) # noqa: E712
.order_by(Car.last_seen_at.asc(), Car.id.asc())
.offset(max(0, int(offset)))
.limit(max(1, int(limit)))
).all()
return [str(row[0]) for row in rows if row and row[0]]
def count_active_cars_for_lane(self, prefix: str = "iaai:") -> int:
with self.session_scope() as session:
return int(session.execute(
select(func.count())
.select_from(Car)
.where(Car.origin_id.like(f"{prefix}%"))
.where(Car.is_sold == False) # noqa: E712
).scalar_one() or 0)

62
iaai_scraper/storage/models.py
View File

@@ -80,3 +80,65 @@ class SyncRun(Base):
cars_failed: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
images_upserted: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
error_summary: Mapped[str | None] = mapped_column(Text, nullable=True)
class VehicleCandidate(Base):
__tablename__ = "vehicle_candidates"
__table_args__ = (
Index("ix_vehicle_candidates_url", "url"),
Index("ix_vehicle_candidates_status_discovered", "status", "discovered_at"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
url: Mapped[str] = mapped_column(String(), nullable=False, unique=True, index=True)
discovered_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
status: Mapped[str] = mapped_column(String(20), nullable=False, default="pending", index=True) # pending, processing, processed, failed
priority: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
last_attempt_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True), nullable=True)
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
raw_snapshots: Mapped[list["VehicleRawSnapshot"]] = relationship("VehicleRawSnapshot", back_populates="candidate", cascade="all, delete-orphan")
class VehicleRawSnapshot(Base):
__tablename__ = "vehicle_raw_snapshots"
__table_args__ = (
Index("ix_vehicle_raw_snapshots_candidate_id", "candidate_id"),
Index("ix_vehicle_raw_snapshots_captured_at", "captured_at"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
candidate: Mapped[VehicleCandidate] = relationship("VehicleCandidate", back_populates="raw_snapshots")
captured_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
method: Mapped[str] = mapped_column(String(20), nullable=False) # http, browser
success: Mapped[bool] = mapped_column(Boolean, nullable=False)
raw_data: Mapped[str | None] = mapped_column(Text, nullable=True) # HTML or JSON content
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
parse_results: Mapped[list["VehicleParseResult"]] = relationship("VehicleParseResult", back_populates="snapshot", cascade="all, delete-orphan")
class VehicleParseResult(Base):
__tablename__ = "vehicle_parse_results"
__table_args__ = (
Index("ix_vehicle_parse_results_snapshot_id", "snapshot_id"),
Index("ix_vehicle_parse_results_parsed_at", "parsed_at"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
snapshot_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_raw_snapshots.id", ondelete="CASCADE"), nullable=False, index=True)
snapshot: Mapped[VehicleRawSnapshot] = relationship("VehicleRawSnapshot", back_populates="parse_results")
parsed_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, default=func.now(), index=True)
success: Mapped[bool] = mapped_column(Boolean, nullable=False)
parsed_data: Mapped[str | None] = mapped_column(Text, nullable=True) # JSON with parsed vehicle data
error_message: Mapped[str | None] = mapped_column(Text, nullable=True)
class VehicleRetryQueue(Base):
__tablename__ = "vehicle_retry_queue"
__table_args__ = (
Index("ix_vehicle_retry_queue_candidate_id", "candidate_id"),
Index("ix_vehicle_retry_queue_retry_at", "retry_at"),
)
id: Mapped[int] = mapped_column(BigInteger().with_variant(Integer, "sqlite"), primary_key=True, autoincrement=True)
candidate_id: Mapped[int] = mapped_column(Integer, ForeignKey("vehicle_candidates.id", ondelete="CASCADE"), nullable=False, index=True)
reason: Mapped[str] = mapped_column(String(50), nullable=False) # parse_failed, capture_failed, etc.
retry_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), nullable=False, index=True)
attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=0)
max_attempts: Mapped[int] = mapped_column(Integer, nullable=False, default=3)

725
iaai_scraper/worker/tasks.py
View File

@@ -1,7 +1,10 @@
# Задачи Celery для синхронизации автомобилей и листинга IAAI.
from concurrent.futures import ThreadPoolExecutor
import json
import logging
import os
import signal
from threading import Event, Thread
import time
import uuid
@@ -13,6 +16,7 @@ from redis import Redis
from ..core.config import Settings, parse_listing_segments
from ..scraper import IAAIScraper
from ..storage.db import PersistenceService
from ..discovery import SitemapDiscoveryError, discover_vehicle_urls_from_sitemap_with_stats
logger = logging.getLogger("iaai_scraper.worker.tasks")
@@ -25,6 +29,13 @@ SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_LIMIT = 3
SYNC_LISTING_BOOTSTRAP_FAILURE_STREAK_TTL_SECONDS = 24 * 60 * 60
SYNC_LISTING_FOLLOWUP_PENDING_KEY = "iaai:state:sync_listing_followup_pending"
SYNC_LISTING_TASK_NAME = "iaai_scraper.worker.tasks.sync_listing_task"
SYNC_SEGMENT_LOCK_KEY_FMT = "iaai:locks:sync_segment:{idx}"
SYNC_SEGMENTS_PROGRESS_KEY = "iaai:state:sync_segments_progress"
SYNC_SEGMENTS_TOTAL_KEY = "iaai:state:sync_segments_total"
SYNC_SEGMENTS_PROGRESS_TTL_SECONDS = 24 * 60 * 60
TASK_PROGRESS_KEY_FMT = "iaai:state:task_progress:{task_id}"
SITEMAP_HOURLY_LAST_COUNT_KEY = "iaai:state:sitemap_hourly_last_count"
SITEMAP_HOURLY_REFRESH_OFFSET_KEY = "iaai:state:sitemap_hourly_refresh_offset"
def _retry_with_backoff(func, *, attempts: int = 5, base_delay_s: float = 1.0):
@@ -91,6 +102,283 @@ def _sync_listing_lock_ttl_seconds() -> int:
return max(effective_hard + 120, 300)
def _sync_segment_lock_ttl_seconds() -> int:
return 300
def _task_progress_key(task_id: str) -> str:
return TASK_PROGRESS_KEY_FMT.format(task_id=task_id)
def _update_task_progress(
redis_client: Redis,
*,
task_id: str,
stage: str,
ttl_seconds: int,
**payload,
) -> None:
try:
data = {
"task_id": task_id,
"stage": stage,
"ts": int(time.time()),
**payload,
}
redis_client.set(
_task_progress_key(task_id),
json.dumps(data, ensure_ascii=False),
ex=max(60, int(ttl_seconds)),
)
except Exception:
logger.warning("Failed to update task progress for %s", task_id, exc_info=True)
def _clear_task_progress(redis_client: Redis, task_id: str) -> None:
try:
redis_client.delete(_task_progress_key(task_id))
except Exception:
logger.warning("Failed to clear task progress for %s", task_id, exc_info=True)
def _hourly_sitemap_diff_sync(*, lane: str, limit: int | None, only_new: bool | None) -> dict[str, object]:
del limit, only_new
settings = Settings()
persistence = _get_persistence()
persistence.create_tables()
discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings)
discovered_urls = discovery_result.vehicle_urls
active_urls = set(discovered_urls)
existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:")
new_urls = [url for url in discovered_urls if url not in existing_urls]
sold_count = 0
if active_urls:
sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai")
cars_upserted = 0
cars_failed = 0
images_upserted = 0
failures: list[dict[str, str]] = []
if new_urls:
with IAAIScraper(settings) as scraper:
batch_size = settings.celery.batch_size
for batch_start in range(0, len(new_urls), batch_size):
batch_urls = new_urls[batch_start:batch_start + batch_size]
batch_result = scraper.sync_batch(batch_urls, lane=lane)
cars_upserted += int(batch_result.get("cars_upserted", 0))
cars_failed += int(batch_result.get("cars_failed", 0))
images_upserted += int(batch_result.get("images_upserted", 0))
failures.extend(batch_result.get("failures", []))
status = "success" if not failures else ("partial_success" if cars_upserted else "failed")
return {
"status": status,
"run_id": None,
"cars_upserted": cars_upserted,
"cars_failed": cars_failed,
"images_upserted": images_upserted,
"skipped_existing": len(discovered_urls) - len(new_urls),
"elapsed_seconds": None,
"failures": failures,
"full_scan_completed": True,
"hourly_mode": "sitemap_diff",
"discovered_urls": len(discovered_urls),
"new_urls": len(new_urls),
"sold_marked": sold_count,
"transport": discovery_result.stats.transport,
}
def _hourly_sitemap_full_refresh_sync(*, lane: str, limit: int | None, only_new: bool | None) -> dict[str, object]:
del limit, only_new
settings = Settings()
persistence = _get_persistence()
persistence.create_tables()
discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings)
discovered_urls = discovery_result.vehicle_urls
active_urls = set(discovered_urls)
sold_count = 0
if active_urls:
sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai")
cars_upserted = 0
cars_failed = 0
images_upserted = 0
failures: list[dict[str, str]] = []
with IAAIScraper(settings) as scraper:
batch_size = settings.celery.batch_size
for batch_start in range(0, len(discovered_urls), batch_size):
batch_urls = discovered_urls[batch_start:batch_start + batch_size]
batch_result = scraper.sync_batch(batch_urls, lane=lane)
cars_upserted += int(batch_result.get("cars_upserted", 0))
cars_failed += int(batch_result.get("cars_failed", 0))
images_upserted += int(batch_result.get("images_upserted", 0))
failures.extend(batch_result.get("failures", []))
status = "success" if not failures else ("partial_success" if cars_upserted else "failed")
return {
"status": status,
"run_id": None,
"cars_upserted": cars_upserted,
"cars_failed": cars_failed,
"images_upserted": images_upserted,
"skipped_existing": 0,
"elapsed_seconds": None,
"failures": failures,
"full_scan_completed": True,
"hourly_mode": "sitemap_full_refresh",
"discovered_urls": len(discovered_urls),
"new_urls": None,
"sold_marked": sold_count,
"transport": discovery_result.stats.transport,
}
def _hourly_sitemap_rolling_refresh_sync(
*,
redis_client: Redis,
lane: str,
limit: int | None,
only_new: bool | None,
) -> dict[str, object]:
del limit, only_new
settings = Settings()
persistence = _get_persistence()
persistence.create_tables()
discovery_result = discover_vehicle_urls_from_sitemap_with_stats(settings=settings)
discovered_urls = discovery_result.vehicle_urls
active_urls = set(discovered_urls)
sold_count = 0
if active_urls:
sold_count = persistence.mark_sold_not_in_listing_by_urls(active_urls, lane="iaai")
existing_urls = persistence.get_all_active_origin_urls_for_lane("iaai:")
new_urls = [url for url in discovered_urls if url not in existing_urls]
total_active = persistence.count_active_cars_for_lane("iaai:")
batch_size = max(1, int(settings.discovery.hourly_refresh_batch_size))
try:
offset = int(redis_client.get(SITEMAP_HOURLY_REFRESH_OFFSET_KEY) or 0)
except Exception:
offset = 0
refresh_urls = persistence.get_active_origin_urls_batch_for_refresh(
prefix="iaai:",
offset=offset,
limit=batch_size,
)
if not refresh_urls and total_active > 0:
offset = 0
refresh_urls = persistence.get_active_origin_urls_batch_for_refresh(
prefix="iaai:",
offset=0,
limit=batch_size,
)
next_offset = 0
if total_active > 0:
next_offset = offset + len(refresh_urls)
if next_offset >= total_active:
next_offset = 0
try:
redis_client.set(SITEMAP_HOURLY_REFRESH_OFFSET_KEY, str(next_offset))
except Exception:
logger.warning("Failed to persist hourly rolling refresh offset", exc_info=True)
seen: set[str] = set()
target_urls: list[str] = []
for url in new_urls + refresh_urls:
if url in seen:
continue
seen.add(url)
target_urls.append(url)
cars_upserted = 0
cars_failed = 0
images_upserted = 0
failures: list[dict[str, str]] = []
if target_urls:
with IAAIScraper(settings) as scraper:
worker_batch_size = settings.celery.batch_size
for batch_start in range(0, len(target_urls), worker_batch_size):
batch_urls = target_urls[batch_start:batch_start + worker_batch_size]
batch_result = scraper.sync_batch(batch_urls, lane=lane)
cars_upserted += int(batch_result.get("cars_upserted", 0))
cars_failed += int(batch_result.get("cars_failed", 0))
images_upserted += int(batch_result.get("images_upserted", 0))
failures.extend(batch_result.get("failures", []))
status = "success" if not failures else ("partial_success" if cars_upserted else "failed")
return {
"status": status,
"run_id": None,
"cars_upserted": cars_upserted,
"cars_failed": cars_failed,
"images_upserted": images_upserted,
"skipped_existing": max(0, len(discovered_urls) - len(new_urls)),
"elapsed_seconds": None,
"failures": failures,
"full_scan_completed": True,
"hourly_mode": "sitemap_rolling_refresh",
"discovered_urls": len(discovered_urls),
"new_urls": len(new_urls),
"refresh_urls": len(refresh_urls),
"sold_marked": sold_count,
"transport": discovery_result.stats.transport,
"refresh_offset": offset,
"refresh_next_offset": next_offset,
"active_total": total_active,
}
def _start_stall_watchdog(
redis_client: Redis,
*,
task_id: str,
stall_timeout_seconds: int,
) -> tuple[Event, Thread]:
stop_event = Event()
interval_seconds = max(5.0, min(30.0, stall_timeout_seconds / 3))
def _watchdog() -> None:
key = _task_progress_key(task_id)
while not stop_event.wait(interval_seconds):
try:
raw = redis_client.get(key)
if not raw:
continue
data = json.loads(raw)
last_ts = int(data.get("ts") or 0)
if not last_ts:
continue
age = int(time.time()) - last_ts
if age < stall_timeout_seconds:
continue
logger.error(
"Task %s stalled for %ss at stage=%s payload=%s; killing worker process for redelivery",
task_id,
age,
data.get("stage"),
data,
)
except Exception:
logger.warning("Failed to inspect task progress for stall watchdog", exc_info=True)
continue
os.kill(os.getpid(), signal.SIGKILL)
thread = Thread(target=_watchdog, name=f"task-stall-watchdog-{task_id[:8]}", daemon=True)
thread.start()
return stop_event, thread
def _get_persistence() -> PersistenceService:
settings = Settings()
persistence = PersistenceService(settings)
@@ -360,6 +648,204 @@ def _start_lock_heartbeat(
return stop_event, thread
def _reset_segments_progress(redis_client: Redis, total: int) -> None:
try:
pipe = redis_client.pipeline()
pipe.delete(SYNC_SEGMENTS_PROGRESS_KEY)
pipe.set(SYNC_SEGMENTS_TOTAL_KEY, str(int(total)), ex=SYNC_SEGMENTS_PROGRESS_TTL_SECONDS)
pipe.execute()
except Exception:
logger.warning("Failed to reset segments progress", exc_info=True)
def _mark_segment_completed(redis_client: Redis, segment_index: int) -> tuple[int, int]:
"""Помечает сегмент завершённым. Возвращает (completed_count, total)."""
try:
pipe = redis_client.pipeline()
pipe.sadd(SYNC_SEGMENTS_PROGRESS_KEY, str(int(segment_index)))
pipe.expire(SYNC_SEGMENTS_PROGRESS_KEY, SYNC_SEGMENTS_PROGRESS_TTL_SECONDS)
pipe.scard(SYNC_SEGMENTS_PROGRESS_KEY)
pipe.get(SYNC_SEGMENTS_TOTAL_KEY)
results = pipe.execute()
completed = int(results[2] or 0)
total = int(results[3] or 0) if results[3] else 0
return completed, total
except Exception:
logger.warning("Failed to mark segment %d completed", segment_index, exc_info=True)
return 0, 0
@shared_task(
name="iaai_scraper.worker.tasks.sync_segment_task",
bind=True,
max_retries=2,
default_retry_delay=60,
acks_late=True,
)
def sync_segment_task(
self,
segment_index: int,
segment: dict,
lane: str = "iaai_cars",
only_new: bool | None = None,
is_bootstrap: bool = False,
):
"""Обработка одного сегмента листинга. Запускается параллельно несколькими воркерами."""
persistence = _get_persistence()
persistence.create_tables()
task_id = self.request.id or "unknown"
redis_client = _get_redis()
settings = Settings()
# Per-segment lock — защита от случайного дубля
seg_lock_key = SYNC_SEGMENT_LOCK_KEY_FMT.format(idx=int(segment_index))
owner_token = f"{task_id}:{uuid.uuid4().hex}"
lock_ttl = _sync_segment_lock_ttl_seconds()
lock_acquired = _acquire_lock(redis_client, seg_lock_key, owner_token, lock_ttl)
if not lock_acquired:
logger.info("sync_segment_task[%d] skipped: already running", segment_index)
return {"status": "skipped", "segment_index": segment_index, "reason": "duplicate"}
seg_make = segment.get("make")
seg_year_min = segment.get("year_min")
seg_year_max = segment.get("year_max")
seg_label = f"{seg_make or 'ALL'}"
if seg_year_min is not None or seg_year_max is not None:
seg_label += f" ({seg_year_min}-{seg_year_max})"
heartbeat_stop: Event | None = None
heartbeat_thread: Thread | None = None
watchdog_stop: Event | None = None
watchdog_thread: Thread | None = None
stall_timeout = max(120, int(settings.celery.task_stall_timeout_seconds))
progress_ttl = max(lock_ttl + 120, stall_timeout + 120)
_update_task_progress(
redis_client,
task_id=task_id,
stage="segment_task_started",
ttl_seconds=progress_ttl,
segment_index=segment_index,
segment_label=seg_label,
)
heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(redis_client, seg_lock_key, owner_token, lock_ttl)
watchdog_stop, watchdog_thread = _start_stall_watchdog(
redis_client,
task_id=task_id,
stall_timeout_seconds=stall_timeout,
)
try:
def _job():
with IAAIScraper() as scraper:
scraper.set_progress_callback(
lambda stage, meta: _update_task_progress(
redis_client,
task_id=task_id,
stage=stage,
ttl_seconds=progress_ttl,
segment_index=segment_index,
segment_label=seg_label,
**meta,
)
)
base_url = scraper.settings.listing.cars_url
seg_url = scraper._build_segment_listing_url(base_url, seg_make) if seg_make else None
return scraper.sync_listing(
make=None if seg_url else seg_make,
model=None,
lane=lane,
only_new=only_new,
listing_url=seg_url,
year_min=seg_year_min,
year_max=seg_year_max,
skip_mark_sold=True,
)
result = _run_browser_job(_job)
segment_done = bool(result.get("full_scan_completed", False))
_update_task_progress(
redis_client,
task_id=task_id,
stage="segment_task_completed",
ttl_seconds=progress_ttl,
segment_index=segment_index,
segment_label=seg_label,
status=result.get("status", "success"),
cars_upserted=result.get("cars_upserted", 0),
cars_failed=result.get("cars_failed", 0),
)
if is_bootstrap and segment_done:
completed, total = _mark_segment_completed(redis_client, segment_index)
logger.warning(
"Segment %d (%s) bootstrap done: %d/%d completed",
segment_index, seg_label, completed, total,
)
if total > 0 and completed >= total:
_set_full_scan_done(redis_client, True)
_clear_sync_checkpoint(redis_client)
_clear_bootstrap_failure_streak(redis_client)
logger.warning("All %d segments completed; bootstrap full scan done", total)
return {
"status": result.get("status", "success"),
"segment_index": segment_index,
"segment_label": seg_label,
"cars_upserted": result.get("cars_upserted", 0),
"cars_failed": result.get("cars_failed", 0),
"vehicles_collected": result.get("listing", {}).get("vehicles_collected", 0),
"full_scan_completed": segment_done,
}
except SoftTimeLimitExceeded:
logger.warning("sync_segment_task[%d] soft timeout — partial progress saved", segment_index)
_update_task_progress(
redis_client,
task_id=task_id,
stage="segment_task_soft_timeout",
ttl_seconds=progress_ttl,
segment_index=segment_index,
segment_label=seg_label,
)
return {
"status": "timed_out",
"segment_index": segment_index,
"segment_label": seg_label,
}
except Exception as exc:
logger.error("sync_segment_task[%d] failed: %s — %s", segment_index, seg_label, exc, exc_info=True)
_update_task_progress(
redis_client,
task_id=task_id,
stage="segment_task_failed",
ttl_seconds=progress_ttl,
segment_index=segment_index,
segment_label=seg_label,
error=str(exc),
)
try:
raise self.retry(exc=exc)
except self.MaxRetriesExceededError:
return {
"status": "failed",
"segment_index": segment_index,
"segment_label": seg_label,
"error": str(exc),
}
finally:
if watchdog_stop is not None:
watchdog_stop.set()
if watchdog_thread is not None:
watchdog_thread.join(timeout=1)
if heartbeat_stop is not None:
heartbeat_stop.set()
if heartbeat_thread is not None:
heartbeat_thread.join(timeout=1)
_clear_task_progress(redis_client, task_id)
_release_lock_if_owner(redis_client, seg_lock_key, owner_token)
@shared_task(
name="iaai_scraper.worker.tasks.sync_vehicle_task",
bind=True,
@@ -477,10 +963,20 @@ def sync_listing_task(
}
try:
settings = Settings()
full_scan_done_before_run = _is_full_scan_done(redis_client)
force_bootstrap_full_scan = not full_scan_done_before_run
effective_limit = None if force_bootstrap_full_scan else limit
effective_only_new = False if force_bootstrap_full_scan else only_new
hourly_mode = settings.discovery.hourly_mode.strip().lower()
discovery_mode = settings.discovery.mode.strip().lower()
prefer_sitemap_mainline = (
make is None
and model is None
and effective_limit is None
and not effective_only_new
)
use_hourly_sitemap_sync = full_scan_done_before_run and prefer_sitemap_mainline
# Segment-level checkpoint: хранит индекс последнего ПОЛНОСТЬЮ пройденного сегмента.
# Используется только во время bootstrap для пропуска уже обработанных сегментов.
@@ -503,6 +999,58 @@ def sync_listing_task(
effective_limit,
)
if use_hourly_sitemap_sync:
if hourly_mode == "diff":
logger.info("Hourly mode: running sitemap diff sync instead of full listing traversal")
result = _hourly_sitemap_diff_sync(
lane=lane,
limit=effective_limit,
only_new=effective_only_new,
)
elif hourly_mode == "full_refresh":
logger.info("Hourly mode: running sitemap full refresh of all active vehicles")
result = _hourly_sitemap_full_refresh_sync(
lane=lane,
limit=effective_limit,
only_new=effective_only_new,
)
else:
logger.info("Hourly mode: running sitemap rolling refresh of active vehicles")
result = _hourly_sitemap_rolling_refresh_sync(
redis_client=redis_client,
lane=lane,
limit=effective_limit,
only_new=effective_only_new,
)
try:
redis_client.set(SITEMAP_HOURLY_LAST_COUNT_KEY, str(result.get("discovered_urls", 0)))
except Exception:
logger.warning("Failed to persist hourly sitemap count", exc_info=True)
summary = {
"task_id": task_id,
"run_id": result.get("run_id"),
"status": result.get("status", "success"),
"cars_upserted": result.get("cars_upserted", 0),
"cars_failed": result.get("cars_failed", 0),
"images_upserted": result.get("images_upserted", 0),
"skipped_existing": result.get("skipped_existing", 0),
"elapsed_seconds": result.get("elapsed_seconds"),
"failures_count": len(result.get("failures") or []),
"hourly_mode": result.get("hourly_mode"),
"discovered_urls": result.get("discovered_urls", 0),
"new_urls": result.get("new_urls", 0),
"sold_marked": result.get("sold_marked", 0),
}
logger.info(
"sync_listing_task hourly diff completed: status=%s, new=%d, sold=%d, skipped=%d",
summary["status"],
summary["new_urls"],
summary["sold_marked"],
summary["skipped_existing"],
)
return summary
_clear_followup_pending(redis_client)
heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
@@ -514,9 +1062,91 @@ def sync_listing_task(
self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
# Определяем сегменты из конфига.
settings = Settings()
segments = parse_listing_segments(settings.listing.listing_segments_json)
use_segmented = bool(segments) and make is None and model is None
use_segmented = (
bool(segments)
and make is None
and model is None
and not prefer_sitemap_mainline
and discovery_mode != "sitemap"
)
if prefer_sitemap_mainline:
if discovery_mode != "sitemap":
logger.warning(
"Unfiltered full scan forcing sitemap discovery despite IAAI_DISCOVERY_MODE=%s",
discovery_mode or "unset",
)
if segments:
logger.info("Ignoring configured listing segments for unfiltered sitemap full scan")
# --- Параллельный диспатч сегментов: dispatch & exit ---
if use_segmented and settings.celery.parallel_segments:
# Сегменты уже завершённые (для bootstrap resume) пропускаем по Redis SET.
already_completed: set[int] = set()
if force_bootstrap_full_scan:
try:
raw = redis_client.smembers(SYNC_SEGMENTS_PROGRESS_KEY) or set()
already_completed = {int(x) for x in raw if str(x).strip().lstrip("-").isdigit()}
except Exception:
already_completed = set()
pending = [
(idx, seg) for idx, seg in enumerate(segments)
if idx not in already_completed
]
if not pending:
# Всё уже сделано — фиксируем bootstrap done.
if force_bootstrap_full_scan:
_set_full_scan_done(redis_client, True)
_clear_sync_checkpoint(redis_client)
_clear_bootstrap_failure_streak(redis_client)
logger.warning("Parallel segments: nothing to dispatch (all completed)")
return {
"status": "success",
"task_id": task_id,
"mode": "parallel_segments",
"segments_total": len(segments),
"segments_dispatched": 0,
"segments_already_completed": len(already_completed),
}
# При первом запуске bootstrap фиксируем total, чтобы знать когда остановиться.
if force_bootstrap_full_scan and not already_completed:
_reset_segments_progress(redis_client, len(segments))
dispatched = 0
for idx, seg in pending:
try:
self.app.send_task(
"iaai_scraper.worker.tasks.sync_segment_task",
kwargs={
"segment_index": idx,
"segment": seg,
"lane": lane,
"only_new": effective_only_new,
"is_bootstrap": force_bootstrap_full_scan,
},
queue="scraping",
)
dispatched += 1
except Exception:
logger.warning("Failed to dispatch segment %d", idx, exc_info=True)
logger.warning(
"Parallel segments dispatched: %d/%d (already_completed=%d, bootstrap=%s)",
dispatched, len(segments), len(already_completed), force_bootstrap_full_scan,
)
return {
"status": "success",
"task_id": task_id,
"mode": "parallel_segments",
"segments_total": len(segments),
"segments_dispatched": dispatched,
"segments_already_completed": len(already_completed),
}
# --- конец параллельной ветки ---
resume_from_segment = 0
if force_bootstrap_full_scan and use_segmented and last_completed_segment is not None:
@@ -608,10 +1238,7 @@ def sync_listing_task(
)
if force_bootstrap_full_scan:
_set_full_scan_done(redis_client, False)
# SoftTimeLimit считаем failure для circuit breaker:
# если сегмент систематически не укладывается в time limit,
# нельзя крутить followup бесконечно.
_enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=True)
_enqueue_bootstrap_followup("soft_time_limit_exceeded", count_as_failure=False)
# Partial progress уже записан в БД через finish_sync_run.
# Не retry — следующий запуск продолжит обработку по расписанию.
return {
@@ -646,3 +1273,89 @@ def sync_listing_task(
heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
if lock_acquired:
_release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token)
# Новые задачи для ingestion pipeline
@shared_task(
name="iaai_scraper.worker.tasks.discover_vehicles_task",
bind=True,
max_retries=2,
default_retry_delay=60,
acks_late=True,
)
def discover_vehicles_task(self, max_urls: int | None = None):
"""Задача для обнаружения новых URL автомобилей."""
from ..discovery_service import DiscoveryService
try:
discovery = DiscoveryService()
count = discovery.discover_new_vehicles(max_urls=max_urls)
logger.info("discover_vehicles_task completed: %d candidates added", count)
return {"status": "success", "candidates_added": count}
except Exception as exc:
logger.error("discover_vehicles_task failed: %s", exc, exc_info=True)
raise self.retry(exc=exc)
@shared_task(
name="iaai_scraper.worker.tasks.fetch_pending_candidates_task",
bind=True,
max_retries=2,
default_retry_delay=30,
acks_late=True,
)
def fetch_pending_candidates_task(self, limit: int = 10):
"""Задача для захвата данных ожидающих кандидатов."""
from ..fetch_service import FetchService
try:
fetch = FetchService()
count = fetch.process_pending_candidates(limit=limit)
logger.info("fetch_pending_candidates_task completed: %d candidates processed", count)
return {"status": "success", "candidates_processed": count}
except Exception as exc:
logger.error("fetch_pending_candidates_task failed: %s", exc, exc_info=True)
raise self.retry(exc=exc)
@shared_task(
name="iaai_scraper.worker.tasks.enrich_snapshots_task",
bind=True,
max_retries=2,
default_retry_delay=30,
acks_late=True,
)
def enrich_snapshots_task(self, limit: int = 10):
"""Задача для парсинга и обогащения snapshots."""
from ..enrichment_service import EnrichmentService
try:
enrichment = EnrichmentService()
count = enrichment.process_unparsed_snapshots(limit=limit)
logger.info("enrich_snapshots_task completed: %d snapshots enriched", count)
return {"status": "success", "snapshots_enriched": count}
except Exception as exc:
logger.error("enrich_snapshots_task failed: %s", exc, exc_info=True)
raise self.retry(exc=exc)
@shared_task(
name="iaai_scraper.worker.tasks.run_ingestion_pipeline_task",
bind=True,
max_retries=2,
default_retry_delay=120,
acks_late=True,
)
def run_ingestion_pipeline_task(self):
"""Задача для запуска полного ingestion pipeline."""
from ..scheduler_service import SchedulerService
try:
scheduler = SchedulerService()
scheduler.run_full_pipeline()
logger.info("run_ingestion_pipeline_task completed")
return {"status": "success"}
except Exception as exc:
logger.error("run_ingestion_pipeline_task failed: %s", exc, exc_info=True)
raise self.retry(exc=exc)