fix listing parsing

Dockerfile

@@ -13,8 +13,9 @@ RUN pip install --no-cache-dir uv \
     && pip install --no-cache-dir -r /tmp/requirements.txt \
     && pip install --no-cache-dir playwright-stealth
 
-# Install Firefox browser (headless-friendly, bypasses Incapsula)
+# Install both Chromium and Firefox. Chromium is the default engine in Docker
-RUN python -m playwright install firefox
+# because it works more reliably with the current IAAI listing page.
+RUN python -m playwright install chromium firefox
 
 COPY . .
 RUN pip install --no-cache-dir -e .

docker-compose.yml

@@ -8,9 +8,9 @@ x-app-env: &app-env
   CELERY_TASK_SOFT_TIME_LIMIT: ${CELERY_TASK_SOFT_TIME_LIMIT:-3300}
   CELERY_TASK_TIME_LIMIT: ${CELERY_TASK_TIME_LIMIT:-3600}
   CELERY_BROKER_VISIBILITY_TIMEOUT: ${CELERY_BROKER_VISIBILITY_TIMEOUT:-7200}
-  # Без лимита beat может забирать слишком большой объём за один запуск.
+  # 0 => без лимита (в коде интерпретируется как None).
-  # Консервативный дефолт для anti-fraud: 300 авто за запуск.
+  # После первичного полного прохода hourly-режим должен успевать за всеми новыми авто.
-  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-300}
+  CELERY_BEAT_SYNC_LIMIT: ${CELERY_BEAT_SYNC_LIMIT:-0}
   CELERY_WORKER_MAX_TASKS_PER_CHILD: ${CELERY_WORKER_MAX_TASKS_PER_CHILD:-5}
   CELERY_BATCH_SIZE: ${CELERY_BATCH_SIZE:-200}
   IAAI_PARALLEL_TABS: ${IAAI_PARALLEL_TABS:-40}
@@ -20,7 +20,7 @@ x-app-env: &app-env
   IAAI_HUMAN_PACE_ENABLED: ${IAAI_HUMAN_PACE_ENABLED:-true}
   IAAI_TOKENS_FILE: ${IAAI_TOKENS_FILE:-/data/tokens.json}
   IAAI_RUNTIME_CONFIG_FILE: ${IAAI_RUNTIME_CONFIG_FILE:-/app/runtime_config.json}
-  IAAI_BROWSER_ENGINE: ${IAAI_BROWSER_ENGINE:-auto}
+  IAAI_BROWSER_ENGINE: chromium
   TZ: ${TZ:-UTC}
 
 x-env-file: &env-file

iaai_scraper/browser/factory.py

@@ -72,15 +72,18 @@ class BrowserFactory:
         # Выбор движка браузера.
         engine = self.settings.browser_engine.strip().lower()
         if engine == "auto":
-            return "firefox" if self.settings.headless else "chromium"
+            # Для IAAI в headless-режиме Chromium со stealth-скриптами
+            # значительно стабильнее Firefox по anti-bot.
+            return "chromium"
         if engine in ("firefox", "chromium"):
             return engine
         logger.warning("Unknown IAAI_BROWSER_ENGINE=%r, falling back to auto", engine)
-        return "firefox" if self.settings.headless else "chromium"
+        return "chromium"
 
     def create_browser(self, playwright: Playwright) -> Browser:
         engine = self._resolve_engine()
         proxy_dict = self.settings.proxy.to_playwright_dict()
+        logger.info("Resolved browser engine: requested=%s resolved=%s", self.settings.browser_engine, engine)
 
         if engine == "firefox":
             launch_kwargs: dict = {"headless": self.settings.headless}

iaai_scraper/browser/listing.py

@@ -13,6 +13,18 @@ from ..core.utils import first_non_empty
 
 logger = logging.getLogger("iaai_scraper.listing")
 VEHICLE_HREF_RE = re.compile(r"/VehicleDetail/(\d+)(?:~[A-Z]{2})?", re.IGNORECASE)
+VEHICLE_LINK_SELECTOR = "a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']"
+COOKIE_ACCEPT_SELECTORS: tuple[str, ...] = (
+	"button:has-text('Accept All')",
+	"button:has-text('Accept all')",
+	"button:has-text('I Agree')",
+	"button:has-text('Agree')",
+	"button:has-text('Only necessary')",
+	"button:has-text('Только необходимые')",
+	"button:has-text('Принять все')",
+	"[id*='accept']",
+	"[class*='accept']",
+)
 
 
 @dataclass(slots=True)
@@ -32,6 +44,35 @@ class ListingPageResult:
 
 
 class ListingCollector:
+	_NEXT_PAGE_SELECTORS: tuple[str, ...] = (
+		"a[aria-label*='Next']",
+		"button[aria-label*='Next']",
+		"a[aria-label*='next']",
+		"button[aria-label*='next']",
+		"a[title*='Next']",
+		"button[title*='Next']",
+		"a[title*='next']",
+		"button[title*='next']",
+		"a[rel='next']",
+		"link[rel='next']",
+		"a.pagination-next",
+		"button.pagination-next",
+		"a.next",
+		"button.next",
+		"a:has-text('Next')",
+		"button:has-text('Next')",
+		"a:has-text('NEXT')",
+		"button:has-text('NEXT')",
+		"a:has-text('›')",
+		"button:has-text('›')",
+		"a:has-text('»')",
+		"button:has-text('»')",
+		"a:has(img[src*='icon-arrow-right'])",
+		"button:has(img[src*='icon-arrow-right'])",
+		"a:has(img[src*='arrow-right'])",
+		"button:has(img[src*='arrow-right'])",
+	)
+
 	def __init__(self, settings: Settings, pacer: HumanPacer) -> None:
 		self.settings = settings
 		self.pacer = pacer
@@ -61,14 +102,50 @@ class ListingCollector:
 			page.wait_for_load_state("domcontentloaded", timeout=12_000)
 		except Exception:
 			pass
-		try:
+		self._accept_cookie_banner(page)
-			page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=4_000)
+		self._wait_for_listing_content(page)
-		except Exception:
-			# Короткая пауза вместо длинного sleep.
-			time.sleep(0.25)
 		logger.info("Listing page URL: %s", page.url)
 		self.pacer.after_listing_open()
 
+	def _accept_cookie_banner(self, page: Page) -> None:
+		for selector in COOKIE_ACCEPT_SELECTORS:
+			locator = page.locator(selector).first
+			try:
+				if locator.count() == 0:
+					continue
+				if not locator.is_visible(timeout=500):
+					continue
+				locator.click(timeout=2_000)
+				logger.info("Accepted cookie banner using selector: %s", selector)
+				try:
+					page.wait_for_load_state("domcontentloaded", timeout=3_000)
+				except Exception:
+					pass
+				return
+			except Exception:
+				continue
+
+	def _wait_for_listing_content(self, page: Page) -> None:
+		try:
+			page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=12_000)
+			return
+		except Exception:
+			pass
+
+		# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
+		try:
+			page.wait_for_function(
+				"""() => {
+					const html = document.documentElement?.innerHTML || '';
+					const text = document.body?.innerText || '';
+					return html.includes('/VehicleDetail/') || /\\b\d+\s+VEHICLES\b/i.test(text);
+				}""",
+				timeout=12_000,
+			)
+		except Exception:
+			# Короткая пауза вместо длинного sleep.
+			time.sleep(1.0)
+
 	def apply_filters(self, page: Page, make: str | None = None, model: str | None = None) -> dict[str, str | None]:
 		applied = {"make": None, "model": None}
 		if make and self._try_fill_filter_input(page, ["input[placeholder*='Make']", "input[aria-label*='Make']"], make):
@@ -81,14 +158,20 @@ class ListingCollector:
 
 	def collect_current_page(self, page: Page, page_number: int = 1) -> ListingPageResult:
 		# Считываем ссылки одним проходом по DOM.
+		self._accept_cookie_banner(page)
+		self._wait_for_listing_content(page)
 		try:
 			raw_items = page.eval_on_selector_all(
-				"a[href*='/VehicleDetail/']",
+				"a[href], [data-href], [href]",
 				"""
-				(nodes) => nodes.map((a) => ({
+				(nodes) => nodes.map((node) => ({
-					href: a.getAttribute('href') || '',
+					href:
-					title: a.getAttribute('title') || '',
+						node.getAttribute('href') ||
-					text: (a.textContent || '').trim(),
+						node.getAttribute('data-href') ||
+						node.getAttribute('data-url') ||
+						'',
+					title: node.getAttribute('title') || node.getAttribute('aria-label') || '',
+					text: (node.textContent || '').trim(),
 				}))
 				""",
 			)
@@ -113,21 +196,69 @@ class ListingCollector:
 			links.append(ListingVehicleLink(href=absolute, title=str(title).strip(), lot_number=lot_number))
 			if len(links) >= self.settings.listing.max_vehicles_per_run:
 				break
+
+		# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
+		# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
+		if not links:
+			try:
+				page.wait_for_timeout(1_500)
+			except Exception:
+				pass
+			try:
+				html = page.content()
+			except Exception as exc:
+				logger.debug("page.content() failed on page %d: %s", page_number, exc)
+				html = ""
+
+			for absolute, lot_number in self._extract_vehicle_links_from_html(html):
+				if absolute in seen:
+					continue
+				seen.add(absolute)
+				links.append(ListingVehicleLink(href=absolute, title="", lot_number=lot_number))
+				if len(links) >= self.settings.listing.max_vehicles_per_run:
+					break
+
+			if links:
+				logger.info(
+					"Page %d: recovered %d vehicle links from HTML fallback",
+					page_number,
+					len(links),
+				)
 		next_page_detected = self._has_next_page(page)
 		return ListingPageResult(source_url=page.url, page_number=page_number, vehicle_links=links, pagination_available=next_page_detected, next_page_detected=next_page_detected)
 
+	def _extract_vehicle_links_from_html(self, html: str) -> list[tuple[str, str]]:
+		if not html:
+			return []
+
+		# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
+		normalized = html.replace("\\/", "/")
+		found: list[tuple[str, str]] = []
+		seen: set[str] = set()
+
+		for match in VEHICLE_HREF_RE.finditer(normalized):
+			lot_number = match.group(1)
+			absolute = urljoin(self.settings.home_url, match.group(0))
+			if absolute in seen:
+				continue
+			seen.add(absolute)
+			found.append((absolute, lot_number))
+			if len(found) >= self.settings.listing.page_link_limit:
+				break
+
+		return found
+
 	def go_to_next_page(self, page: Page) -> bool:
 		# Запоминаем первую ссылку текущей страницы для определения смены контента.
 		old_first_href = ""
 		try:
-			first_link = page.locator("a[href*='/VehicleDetail/']").first
+			first_link = page.locator(VEHICLE_LINK_SELECTOR).first
 			if first_link.count() > 0:
 				old_first_href = first_link.get_attribute("href") or ""
 		except Exception:
 			pass
 
-		selectors = ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]
+		for selector in self._NEXT_PAGE_SELECTORS:
-		for selector in selectors:
 			locator = page.locator(selector).first
 			if locator.count() == 0:
 				continue
@@ -150,7 +281,7 @@ class ListingCollector:
 				try:
 					page.wait_for_function(
 						f"""() => {{
-							const a = document.querySelector("a[href*='/VehicleDetail/']");
+							const a = document.querySelector("a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']");
 							return a && a.getAttribute('href') !== '{old_first_href}';
 						}}""",
 						timeout=8000,
@@ -164,11 +295,131 @@ class ListingCollector:
 					pass
 
 			try:
-				page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
+				page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
 			except Exception:
 				pass
 			self.pacer.after_page_change()
 			return True
+
+		# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
+		# + стрелка с иконкой, без явного текста Next.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
+					const disabled = (el) => {
+						if (!el) return true;
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaDisabled = (el.getAttribute('aria-disabled') || '').toLowerCase();
+						return el.hasAttribute('disabled') || ariaDisabled === 'true' || cls.includes('disabled');
+					};
+
+					const controls = Array.from(document.querySelectorAll('a,button,[role="button"]'))
+						.filter((el) => visible(el) && !disabled(el));
+
+					const current = controls.find((el) => {
+						const text = (el.textContent || '').trim();
+						const cls = (el.getAttribute('class') || '').toLowerCase();
+						const ariaCurrent = (el.getAttribute('aria-current') || '').toLowerCase();
+						return /^\d+$/.test(text) && (ariaCurrent === 'page' || cls.includes('active') || cls.includes('current') || cls.includes('selected'));
+					});
+
+					if (current) {
+						const currentPage = parseInt((current.textContent || '').trim(), 10);
+						const nextNumber = controls.find((el) => {
+							const text = (el.textContent || '').trim();
+							return /^\d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
+						});
+						if (nextNumber) {
+							nextNumber.click();
+							return true;
+						}
+					}
+
+					const iconNext = controls.find((el) => {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+						return hasRightArrowIcon || rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+					});
+
+					if (iconNext) {
+						iconNext.click();
+						return true;
+					}
+
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if old_first_href:
+					try:
+						page.wait_for_function(
+							f"""() => {{
+								const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
+								return a && a.getAttribute('href') !== '{old_first_href}';
+							}}""",
+							timeout=12000,
+						)
+					except Exception:
+						pass
+				else:
+					try:
+						page.wait_for_load_state("domcontentloaded", timeout=12000)
+					except Exception:
+						pass
+				self.pacer.after_page_change()
+				return True
+		except Exception as exc:
+			logger.debug("Numeric/icon pagination fallback failed: %s", exc)
+
+		# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
+		try:
+			clicked = bool(page.evaluate(
+				"""
+				() => {
+					const candidates = Array.from(document.querySelectorAll('a,button,[role="button"]'));
+					for (const el of candidates) {
+						const text = (el.textContent || '').trim().toLowerCase();
+						const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+						const title = (el.getAttribute('title') || '').trim().toLowerCase();
+						const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+						const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+						const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+						const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
+						const looksNext = rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || ['next', '›', '»', '>'].includes(text);
+						if (!disabled && visible && looksNext) {
+							el.click();
+							return true;
+						}
+					}
+					return false;
+				}
+				"""
+			))
+			if clicked:
+				if old_first_href:
+					try:
+						page.wait_for_function(
+							f"""() => {{
+								const a = document.querySelector(\"a[href*='/VehicleDetail/'], a[href*='/vehicledetail/'], a[href*='VehicleDetail'], a[href*='vehicledetail']\");
+								return a && a.getAttribute('href') !== '{old_first_href}';
+							}}""",
+							timeout=8000,
+						)
+					except Exception:
+						pass
+				self.pacer.after_page_change()
+				return True
+		except Exception as exc:
+			logger.debug("JS next-page fallback failed: %s", exc)
+
+		logger.warning("Could not navigate to next page from %s", page.url)
 		return False
 
 	def collect_listing_links(
@@ -287,7 +538,7 @@ class ListingCollector:
 				except Exception:
 					pass
 				try:
-					page.wait_for_selector("a[href*='/VehicleDetail/']", timeout=3000)
+					page.wait_for_selector(VEHICLE_LINK_SELECTOR, timeout=3000)
 				except Exception:
 					pass
 				return True
@@ -297,7 +548,26 @@ class ListingCollector:
 
 	@staticmethod
 	def _has_next_page(page: Page) -> bool:
-		for selector in ["a[aria-label*='Next']", "button[aria-label*='Next']", "a.pagination-next", "button.pagination-next", "a:has-text('Next')", "button:has-text('Next')"]:
+		for selector in ListingCollector._NEXT_PAGE_SELECTORS:
 			if page.locator(selector).count() > 0:
 				return True
+		try:
+			return bool(page.evaluate(
+				"""
+				() => Array.from(document.querySelectorAll('a,button,[role="button"]')).some((el) => {
+					const text = (el.textContent || '').trim().toLowerCase();
+					const aria = (el.getAttribute('aria-label') || '').trim().toLowerCase();
+					const title = (el.getAttribute('title') || '').trim().toLowerCase();
+					const rel = (el.getAttribute('rel') || '').trim().toLowerCase();
+					const cls = (el.getAttribute('class') || '').trim().toLowerCase();
+					const disabled = el.hasAttribute('disabled') || el.getAttribute('aria-disabled') === 'true' || cls.includes('disabled');
+					const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
+					const hasRightArrowIcon = !!el.querySelector('img[src*="icon-arrow-right"], img[src*="arrow-right"]');
+					const looksNumericNext = /^\d+$/.test(text);
+					return !disabled && visible && (rel === 'next' || aria.includes('next') || title.includes('next') || cls.includes('next') || hasRightArrowIcon || looksNumericNext || ['next', '›', '»', '>'].includes(text));
+				})
+				"""
+			))
+		except Exception:
+			return False
 		return False

tests/test_listing.py

@@ -10,6 +10,7 @@ from iaai_scraper.browser.listing import ListingCollector
 class _FakePage:
     def __init__(self, counts: dict[str, int]) -> None:
         self._counts = counts
+        self._evaluate_result = False
 
     class _Locator:
         def __init__(self, count_value: int) -> None:
@@ -21,6 +22,9 @@ class _FakePage:
     def locator(self, selector: str) -> "_FakePage._Locator":
         return _FakePage._Locator(self._counts.get(selector, 0))
 
+    def evaluate(self, _script: str):
+        return self._evaluate_result
+
 
 class TestListingUnit(unittest.TestCase):
     def test_has_next_page_true_for_known_selector(self) -> None:
@@ -31,6 +35,30 @@ class TestListingUnit(unittest.TestCase):
         page = _FakePage({})
         self.assertFalse(ListingCollector._has_next_page(page))
 
+    def test_has_next_page_true_for_numeric_pagination_fallback(self) -> None:
+        page = _FakePage({})
+        page._evaluate_result = True
+        self.assertTrue(ListingCollector._has_next_page(page))
+
+    def test_extract_vehicle_links_from_html_finds_detail_urls(self) -> None:
+        collector = ListingCollector(Settings(), HumanPacer(Settings()))
+        html = """
+        <div>
+            <h4><a href=\"/VehicleDetail/45184893~US\">Car 1</a></h4>
+            <script>window.__data = {\"href\":\"\\/VehicleDetail\\/45171480~US\"}</script>
+        </div>
+        """
+
+        links = collector._extract_vehicle_links_from_html(html)
+
+        self.assertEqual(
+            links,
+            [
+                ("https://www.iaai.com/VehicleDetail/45184893~US", "45184893"),
+                ("https://www.iaai.com/VehicleDetail/45171480~US", "45171480"),
+            ],
+        )
+
 
 if __name__ == "__main__":
     unittest.main()