diff --git a/Dockerfile b/Dockerfile index 45b69b9..f68194e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,8 +13,9 @@ RUN pip install --no-cache-dir uv \ && pip install --no-cache-dir -r /tmp/requirements.txt \ && pip install --no-cache-dir playwright-stealth -# Install Firefox browser (headless-friendly, bypasses Incapsula) -RUN python -m playwright install firefox +# Install both Chromium and Firefox. Chromium is the default engine in Docker +# because it works more reliably with the current IAAI listing page. +RUN python -m playwright install chromium firefox COPY . . RUN pip install --no-cache-dir -e . diff --git a/docker-compose.yml b/docker-compose.yml index 0ddeda8..fe72c94 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -8,9 +8,9 @@ x-app-env: &app-env CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300} CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600} CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200} - # Без лимита beat может забирать слишком большой объём за один запуск. - # Консервативный дефолт для anti-fraud: 300 авто за запуск. - CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-300} + # 0 => без лимита (в коде интерпретируется как None). + # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто. + CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0} CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5} CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200} IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40} @@ -20,7 +20,7 @@ x-app-env: &app-env IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true} IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json} IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json} - IAAI_BROWSER_ENGINE: ${IAAI_BROWSER_ENGINE:-auto} + IAAI_BROWSER_ENGINE: chromium TZ: ${TZ:-UTC} x-env-file: &env-file diff --git a/iaai_scraper/browser/factory.py b/iaai_scraper/browser/factory.py index 69b393c..eb1f578 100644 --- a/iaai_scraper/browser/factory.py +++ b/iaai_scraper/browser/factory.py @@ -69,18 +69,21 @@ class BrowserFactory: self.settings = settings def _resolve_engine(self) -> str: - # Выбор движка браузера. + # Выбор движка браузера. engine = self.settings.browser_engine.strip().lower() if engine == "auto": - return "firefox" if self.settings.headless else "chromium" + # Для IAAI в headless-режиме Chromium со stealth-скриптами + # значительно стабильнее Firefox по anti-bot. + return "chromium" if engine in ("firefox", "chromium"): return engine logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine) - return "firefox" if self.settings.headless else "chromium" + return "chromium" def create_browser(self, playwright: Playwright) -> Browser: engine = self._resolve_engine() proxy_dict = self.settings.proxy.to_playwright_dict() + logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine) if engine == "firefox": launch_kwargs: dict = {"headless": self.settings.headless} diff --git a/iaai_scraper/browser/listing.py b/iaai_scraper/browser/listing.py index d4009c4..c44c6aa 100644 --- a/iaai_scraper/browser/listing.py +++ b/iaai_scraper/browser/listing.py @@ -13,6 +13,18 @@ from ..core.utils import first_non_empty logger = logging.getLogger("iaai_scraper.listing") VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE) +VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']" +COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = ( + "button:has-text('Accept All')", + "button:has-text('Accept all')", + "button:has-text('I Agree')", + "button:has-text('Agree')", + "button:has-text('Only necessary')", + "button:has-text('Только необходимые')", + "button:has-text('Принять все')", + "[id*='accept']", + "[class*='accept']", +) @dataclass(slots=True) @@ -32,6 +44,35 @@ class ListingPageResult: class ListingCollector: + _NEXT_PAGE_SELECTORS: tuple[str, ...] = ( + "a[aria-label*='Next']", + "button[aria-label*='Next']", + "a[aria-label*='next']", + "button[aria-label*='next']", + "a[title*='Next']", + "button[title*='Next']", + "a[title*='next']", + "button[title*='next']", + "a[rel='next']", + "link[rel='next']", + "a.pagination-next", + "button.pagination-next", + "a.next", + "button.next", + "a:has-text('Next')", + "button:has-text('Next')", + "a:has-text('NEXT')", + "button:has-text('NEXT')", + "a:has-text('›')", + "button:has-text('›')", + "a:has-text('»')", + "button:has-text('»')", + "a:has(img[src*='icon-arrow-right'])", + "button:has(img[src*='icon-arrow-right'])", + "a:has(img[src*='arrow-right'])", + "button:has(img[src*='arrow-right'])", + ) + def __init__(self, settings: Settings, pacer: HumanPacer) -> None: self.settings = settings self.pacer = pacer @@ -61,14 +102,50 @@ class ListingCollector: page.wait_for_load_state("domcontentloaded", timeout=12_000) except Exception: pass - try: - page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=4_000) - except Exception: - # Короткая пауза вместо длинного sleep. - time.sleep(0.25) + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) logger.info("Listing page URL: %s", page.url) self.pacer.after_listing_open() + def _accept_cookie_banner(self, page: Page) -> None: + for selector in COOKIE_ACCEPT_SELECTORS: + locator = page.locator(selector).first + try: + if locator.count() == 0: + continue + if not locator.is_visible(timeout=500): + continue + locator.click(timeout=2_000) + logger.info("Accepted cookie banner using selector: %s", selector) + try: + page.wait_for_load_state("domcontentloaded", timeout=3_000) + except Exception: + pass + return + except Exception: + continue + + def _wait_for_listing_content(self, page: Page) -> None: + try: + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000) + return + except Exception: + pass + + # Fallback: React/SSR разметка может появиться не сразу, даже если ещё нет в DOM. + try: + page.wait_for_function( + """() => { + const html = document.documentElement?.innerHTML || ''; + const text = document.body?.innerText || ''; + return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text); + }""", + timeout=12_000, + ) + except Exception: + # Короткая пауза вместо длинного sleep. + time.sleep(1.0) + def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]: applied = {"make": None, "model": None} if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make): @@ -81,14 +158,20 @@ class ListingCollector: def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult: # Считываем ссылки одним проходом по DOM. + self._accept_cookie_banner(page) + self._wait_for_listing_content(page) try: raw_items = page.eval_on_selector_all( - "a[href*='/VehicleDetail/']", + "a[href], [data-href], [href]", """ - (nodes) => nodes.map((a) => ({ - href: a.getAttribute('href') || '', - title: a.getAttribute('title') || '', - text: (a.textContent || '').trim(), + (nodes) => nodes.map((node) => ({ + href: + node.getAttribute('href') || + node.getAttribute('data-href') || + node.getAttribute('data-url') || + '', + title: node.getAttribute('title') || node.getAttribute('aria-label') || '', + text: (node.textContent || '').trim(), })) """, ) @@ -113,21 +196,69 @@ class ListingCollector: links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number)) if len(links) >= self.settings.listing.max_vehicles_per_run: break + + # Fallback: на IAAI ссылки иногда не рендерятся как , + # но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/"). + if not links: + try: + page.wait_for_timeout(1_500) + except Exception: + pass + try: + html = page.content() + except Exception as exc: + logger.debug("page.content() failed on page %d: %s", page_number, exc) + html = "" + + for absolute, lot_number in self._extract_vehicle_links_from_html(html): + if absolute in seen: + continue + seen.add(absolute) + links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number)) + if len(links) >= self.settings.listing.max_vehicles_per_run: + break + + if links: + logger.info( + "Page %d: recovered %d vehicle links from HTML fallback", + page_number, + len(links), + ) next_page_detected = self._has_next_page(page) return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected) + def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]: + if not html: + return [] + + # Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US" + normalized = html.replace("\\/", "/") + found: list[tuple[str, str]] = [] + seen: set[str] = set() + + for match in VEHICLE_HREF_RE.finditer(normalized): + lot_number = match.group(1) + absolute = urljoin(self.settings.home_url, match.group(0)) + if absolute in seen: + continue + seen.add(absolute) + found.append((absolute, lot_number)) + if len(found) >= self.settings.listing.page_link_limit: + break + + return found + def go_to_next_page(self, page: Page) -> bool: # Запоминаем первую ссылку текущей страницы для определения смены контента. old_first_href = "" try: - first_link = page.locator("a[href*='/VehicleDetail/']").first + first_link = page.locator(VEHICLE_LINK_SELECTOR).first if first_link.count() > 0: old_first_href = first_link.get_attribute("href") or "" except Exception: pass - selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"] - for selector in selectors: + for selector in self._NEXT_PAGE_SELECTORS: locator = page.locator(selector).first if locator.count() == 0: continue @@ -150,7 +281,7 @@ class ListingCollector: try: page.wait_for_function( f"""() => {{ - const a = document.querySelector("a[href*='/VehicleDetail/']"); + const a = document.querySelector("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"); return a && a.getAttribute('href') !== '{old_first_href}'; }}""", timeout=8000, @@ -164,11 +295,131 @@ class ListingCollector: pass try: - page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000) + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) except Exception: pass self.pacer.after_page_change() return True + + # Fallback для IAAI: пагинация часто рендерится как набор номеров страниц + # + стрелка с иконкой, без явного текста Next. + try: + clicked = bool(page.evaluate( + """ + () => { + const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length)); + const disabled = (el) => { + if (!el) return true; + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase(); + return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled'); + }; + + const controls = Array.from(document.querySelectorAll('a,button,[role="button"]')) + .filter((el) => visible(el) && !disabled(el)); + + const current = controls.find((el) => { + const text = (el.textContent || '').trim(); + const cls = (el.getAttribute('class') || '').toLowerCase(); + const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase(); + return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected')); + }); + + if (current) { + const currentPage = parseInt((current.textContent || '').trim(), 10); + const nextNumber = controls.find((el) => { + const text = (el.textContent || '').trim(); + return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1; + }); + if (nextNumber) { + nextNumber.click(); + return true; + } + } + + const iconNext = controls.find((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + }); + + if (iconNext) { + iconNext.click(); + return true; + } + + return false; + } + """ + )) + if clicked: + if old_first_href: + try: + page.wait_for_function( + f"""() => {{ + const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\"); + return a && a.getAttribute('href') !== '{old_first_href}'; + }}""", + timeout=12000, + ) + except Exception: + pass + else: + try: + page.wait_for_load_state("domcontentloaded", timeout=12000) + except Exception: + pass + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("Numeric/icon pagination fallback failed: %s", exc) + + # JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту. + try: + clicked = bool(page.evaluate( + """ + () => { + const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]')); + for (const el of candidates) { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); + const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text); + if (!disabled && visible && looksNext) { + el.click(); + return true; + } + } + return false; + } + """ + )) + if clicked: + if old_first_href: + try: + page.wait_for_function( + f"""() => {{ + const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\"); + return a && a.getAttribute('href') !== '{old_first_href}'; + }}""", + timeout=8000, + ) + except Exception: + pass + self.pacer.after_page_change() + return True + except Exception as exc: + logger.debug("JS next-page fallback failed: %s", exc) + + logger.warning("Could not navigate to next page from %s", page.url) return False def collect_listing_links( @@ -287,7 +538,7 @@ class ListingCollector: except Exception: pass try: - page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000) + page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000) except Exception: pass return True @@ -297,7 +548,26 @@ class ListingCollector: @staticmethod def _has_next_page(page: Page) -> bool: - for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]: + for selector in ListingCollector._NEXT_PAGE_SELECTORS: if page.locator(selector).count() > 0: return True + try: + return bool(page.evaluate( + """ + () => Array.from(document.querySelectorAll('a,button,[role="button"]')).some((el) => { + const text = (el.textContent || '').trim().toLowerCase(); + const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase(); + const title = (el.getAttribute('title') || '').trim().toLowerCase(); + const rel = (el.getAttribute('rel') || '').trim().toLowerCase(); + const cls = (el.getAttribute('class') || '').trim().toLowerCase(); + const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled'); + const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length); + const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]'); + const looksNumericNext = /^\d+$/.test(text); + return !disabled && visible && (rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || looksNumericNext || ['next', '›', '»', '>'].includes(text)); + }) + """ + )) + except Exception: + return False return False diff --git a/tests/test_listing.py b/tests/test_listing.py index aae31aa..c08b303 100644 --- a/tests/test_listing.py +++ b/tests/test_listing.py @@ -10,6 +10,7 @@ from iaai_scraper.browser.listing import ListingCollector class _FakePage: def __init__(self, counts: dict[str, int]) -> None: self._counts = counts + self._evaluate_result = False class _Locator: def __init__(self, count_value: int) -> None: @@ -21,6 +22,9 @@ class _FakePage: def locator(self, selector: str) -> "_FakePage._Locator": return _FakePage._Locator(self._counts.get(selector, 0)) + def evaluate(self, _script: str): + return self._evaluate_result + class TestListingUnit(unittest.TestCase): def test_has_next_page_true_for_known_selector(self) -> None: @@ -31,6 +35,30 @@ class TestListingUnit(unittest.TestCase): page = _FakePage({}) self.assertFalse(ListingCollector._has_next_page(page)) + def test_has_next_page_true_for_numeric_pagination_fallback(self) -> None: + page = _FakePage({}) + page._evaluate_result = True + self.assertTrue(ListingCollector._has_next_page(page)) + + def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None: + collector = ListingCollector(Settings(), HumanPacer(Settings())) + html = """ +
+

Car 1

+ +
+ """ + + links = collector._extract_vehicle_links_from_html(html) + + self.assertEqual( + links, + [ + ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"), + ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"), + ], + ) + if __name__ == "__main__": unittest.main()