@@ -13,6 +13,18 @@ from ..core.utils import first_non_empty
logger = logging . getLogger ( " iaai_scraper.listing " )
VEHICLE_HREF_RE = re . compile ( r " /VehicleDetail/( \ d+)(?:~[A-Z] {2} )? " , re . IGNORECASE )
VEHICLE_LINK_SELECTOR = " a[href*= ' /VehicleDetail/ ' ], a[href*= ' /vehicledetail/ ' ], a[href*= ' VehicleDetail ' ], a[href*= ' vehicledetail ' ] "
COOKIE_ACCEPT_SELECTORS : tuple [ str , . . . ] = (
" button:has-text( ' Accept All ' ) " ,
" button:has-text( ' Accept all ' ) " ,
" button:has-text( ' I Agree ' ) " ,
" button:has-text( ' Agree ' ) " ,
" button:has-text( ' Only necessary ' ) " ,
" button:has-text( ' Только необходимые ' ) " ,
" button:has-text( ' Принять все ' ) " ,
" [id*= ' accept ' ] " ,
" [class*= ' accept ' ] " ,
)
@dataclass ( slots = True )
@@ -32,6 +44,35 @@ class ListingPageResult:
class ListingCollector :
_NEXT_PAGE_SELECTORS : tuple [ str , . . . ] = (
" a[aria-label*= ' Next ' ] " ,
" button[aria-label*= ' Next ' ] " ,
" a[aria-label*= ' next ' ] " ,
" button[aria-label*= ' next ' ] " ,
" a[title*= ' Next ' ] " ,
" button[title*= ' Next ' ] " ,
" a[title*= ' next ' ] " ,
" button[title*= ' next ' ] " ,
" a[rel= ' next ' ] " ,
" link[rel= ' next ' ] " ,
" a.pagination-next " ,
" button.pagination-next " ,
" a.next " ,
" button.next " ,
" a:has-text( ' Next ' ) " ,
" button:has-text( ' Next ' ) " ,
" a:has-text( ' NEXT ' ) " ,
" button:has-text( ' NEXT ' ) " ,
" a:has-text( ' › ' ) " ,
" button:has-text( ' › ' ) " ,
" a:has-text( ' » ' ) " ,
" button:has-text( ' » ' ) " ,
" a:has(img[src*= ' icon-arrow-right ' ]) " ,
" button:has(img[src*= ' icon-arrow-right ' ]) " ,
" a:has(img[src*= ' arrow-right ' ]) " ,
" button:has(img[src*= ' arrow-right ' ]) " ,
)
def __init__ ( self , settings : Settings , pacer : HumanPacer ) - > None :
self . settings = settings
self . pacer = pacer
@@ -61,14 +102,50 @@ class ListingCollector:
page . wait_for_load_state ( " domcontentloaded " , timeout = 12_000 )
except Exception :
pass
try :
page . wait_for_selector ( " a[href*= ' /VehicleDetail/ ' ] " , timeout = 4_000 )
except Exception :
# Короткая пауза вместо длинного sleep.
time . sleep ( 0.25 )
self . _accept_cookie_banner ( page )
self . _ wait_for_listing_content ( page )
logger . info ( " Listing page URL: %s " , page . url )
self . pacer . after_listing_open ( )
def _accept_cookie_banner ( self , page : Page ) - > None :
for selector in COOKIE_ACCEPT_SELECTORS :
locator = page . locator ( selector ) . first
try :
if locator . count ( ) == 0 :
continue
if not locator . is_visible ( timeout = 500 ) :
continue
locator . click ( timeout = 2_000 )
logger . info ( " Accepted cookie banner using selector: %s " , selector )
try :
page . wait_for_load_state ( " domcontentloaded " , timeout = 3_000 )
except Exception :
pass
return
except Exception :
continue
def _wait_for_listing_content ( self , page : Page ) - > None :
try :
page . wait_for_selector ( VEHICLE_LINK_SELECTOR , timeout = 12_000 )
return
except Exception :
pass
# Fallback: React/SSR разметка может появиться не сразу, даже если <a> ещё нет в DOM.
try :
page . wait_for_function (
""" () => {
const html = document.documentElement?.innerHTML || ' ' ;
const text = document.body?.innerText || ' ' ;
return html.includes( ' /VehicleDetail/ ' ) || / \\ b \ d+ \ s+VEHICLES \b /i.test(text);
} """ ,
timeout = 12_000 ,
)
except Exception :
# Короткая пауза вместо длинного sleep.
time . sleep ( 1.0 )
def apply_filters ( self , page : Page , make : str | None = None , model : str | None = None ) - > dict [ str , str | None ] :
applied = { " make " : None , " model " : None }
if make and self . _try_fill_filter_input ( page , [ " input[placeholder*= ' Make ' ] " , " input[aria-label*= ' Make ' ] " ] , make ) :
@@ -81,14 +158,20 @@ class ListingCollector:
def collect_current_page ( self , page : Page , page_number : int = 1 ) - > ListingPageResult :
# Считываем ссылки одним проходом по DOM.
self . _accept_cookie_banner ( page )
self . _wait_for_listing_content ( page )
try :
raw_items = page . eval_on_selector_all (
" a[href*= ' /VehicleDetail/ ' " ,
" a[href], [data-href], [href ] " ,
"""
(nodes) => nodes.map((a ) => ( {
href: a.getAttribute( ' href ' ) || ' ' ,
title: a .getAttribute( ' title ' ) || ' ' ,
text: (a.textContent || ' ' ).trim(),
(nodes) => nodes.map((node ) => ( {
href:
node .getAttribute( ' href ' ) ||
node.getAttribute( ' data-href ' ) ||
node.getAttribute( ' data-url ' ) ||
' ' ,
title: node.getAttribute( ' title ' ) || node.getAttribute( ' aria-label ' ) || ' ' ,
text: (node.textContent || ' ' ).trim(),
}))
""" ,
)
@@ -113,21 +196,69 @@ class ListingCollector:
links . append ( ListingVehicleLink ( href = absolute , title = str ( title ) . strip ( ) , lot_number = lot_number ) )
if len ( links ) > = self . settings . listing . max_vehicles_per_run :
break
# Fallback: на IAAI ссылки иногда не рендерятся как <a>,
# но присутствуют в hydration/inline JSON внутри HTML (часто как \/VehicleDetail\/").
if not links :
try :
page . wait_for_timeout ( 1_500 )
except Exception :
pass
try :
html = page . content ( )
except Exception as exc :
logger . debug ( " page.content() failed on page %d : %s " , page_number , exc )
html = " "
for absolute , lot_number in self . _extract_vehicle_links_from_html ( html ) :
if absolute in seen :
continue
seen . add ( absolute )
links . append ( ListingVehicleLink ( href = absolute , title = " " , lot_number = lot_number ) )
if len ( links ) > = self . settings . listing . max_vehicles_per_run :
break
if links :
logger . info (
" Page %d : recovered %d vehicle links from HTML fallback " ,
page_number ,
len ( links ) ,
)
next_page_detected = self . _has_next_page ( page )
return ListingPageResult ( source_url = page . url , page_number = page_number , vehicle_links = links , pagination_available = next_page_detected , next_page_detected = next_page_detected )
def _extract_vehicle_links_from_html ( self , html : str ) - > list [ tuple [ str , str ] ] :
if not html :
return [ ]
# Частый формат в JSON внутри HTML: "\/VehicleDetail\/12345678~US"
normalized = html . replace ( " \\ / " , " / " )
found : list [ tuple [ str , str ] ] = [ ]
seen : set [ str ] = set ( )
for match in VEHICLE_HREF_RE . finditer ( normalized ) :
lot_number = match . group ( 1 )
absolute = urljoin ( self . settings . home_url , match . group ( 0 ) )
if absolute in seen :
continue
seen . add ( absolute )
found . append ( ( absolute , lot_number ) )
if len ( found ) > = self . settings . listing . page_link_limit :
break
return found
def go_to_next_page ( self , page : Page ) - > bool :
# Запоминаем первую ссылку текущей страницы для определения смены контента.
old_first_href = " "
try :
first_link = page . locator ( " a[href*= ' /VehicleDetail/ ' ] " ) . first
first_link = page . locator ( VEHICLE_LINK_SELECTOR ) . first
if first_link . count ( ) > 0 :
old_first_href = first_link . get_attribute ( " href " ) or " "
except Exception :
pass
select ors = [ " a[aria-label*= ' Next ' ] " , " button[aria-label*= ' Next ' ] " , " a.pagination-next " , " button.pagination-next " , " a:has-text( ' Next ' ) " , " button:has-text( ' Next ' ) " ]
for selector in selectors :
f orselector in self . _NEXT_PAGE_SELECTORS :
locator = page . locator ( selector ) . first
if locator . count ( ) == 0 :
continue
@@ -150,7 +281,7 @@ class ListingCollector:
try :
page . wait_for_function (
f """ () => {{
const a = document.querySelector( " a[href*= ' /VehicleDetail/ ' ] " );
const a = document.querySelector( " a[href*= ' /VehicleDetail/ ' ], a[href*= ' /vehicledetail/ ' ], a[href*= ' VehicleDetail ' ], a[href*= ' vehicledetail ' ] " );
return a && a.getAttribute( ' href ' ) !== ' { old_first_href } ' ;
}} """ ,
timeout = 8000 ,
@@ -164,11 +295,131 @@ class ListingCollector:
pass
try :
page . wait_for_selector ( " a[href*= ' /VehicleDetail/ ' ] " , timeout = 3000 )
page . wait_for_selector ( VEHICLE_LINK_SELECTOR , timeout = 3000 )
except Exception :
pass
self . pacer . after_page_change ( )
return True
# Fallback для IAAI: пагинация часто рендерится как набор номеров страниц
# + стрелка с
try :
clicked = bool ( page . evaluate (
"""
() => {
const visible = (el) => !!(el && (el.offsetWidth || el.offsetHeight || el.getClientRects().length));
const disabled = (el) => {
if (!el) return true;
const cls = (el.getAttribute( ' class ' ) || ' ' ).toLowerCase();
const ariaDisabled = (el.getAttribute( ' aria-disabled ' ) || ' ' ).toLowerCase();
return el.hasAttribute( ' disabled ' ) || ariaDisabled === ' true ' || cls.includes( ' disabled ' );
};
const controls = Array.from(document.querySelectorAll( ' a,button,[role= " button " ] ' ))
.filter((el) => visible(el) && !disabled(el));
const current = controls.find((el) => {
const text = (el.textContent || ' ' ).trim();
const cls = (el.getAttribute( ' class ' ) || ' ' ).toLowerCase();
const ariaCurrent = (el.getAttribute( ' aria-current ' ) || ' ' ).toLowerCase();
return /^ \ d+$/.test(text) && (ariaCurrent === ' page ' || cls.includes( ' active ' ) || cls.includes( ' current ' ) || cls.includes( ' selected ' ));
});
if (current) {
const currentPage = parseInt((current.textContent || ' ' ).trim(), 10);
const nextNumber = controls.find((el) => {
const text = (el.textContent || ' ' ).trim();
return /^ \ d+$/.test(text) && parseInt(text, 10) === currentPage + 1;
});
if (nextNumber) {
nextNumber.click();
return true;
}
}
const iconNext = controls.find((el) => {
const text = (el.textContent || ' ' ).trim().toLowerCase();
const aria = (el.getAttribute( ' aria-label ' ) || ' ' ).trim().toLowerCase();
const title = (el.getAttribute( ' title ' ) || ' ' ).trim().toLowerCase();
const rel = (el.getAttribute( ' rel ' ) || ' ' ).trim().toLowerCase();
const cls = (el.getAttribute( ' class ' ) || ' ' ).trim().toLowerCase();
const hasRightArrowIcon = !!el.querySelector( ' img[src*= " icon-arrow-right " ], img[src*= " arrow-right " ] ' );
return hasRightArrowIcon || rel === ' next ' || aria.includes( ' next ' ) || title.includes( ' next ' ) || cls.includes( ' next ' ) || [ ' next ' , ' › ' , ' » ' , ' > ' ].includes(text);
});
if (iconNext) {
iconNext.click();
return true;
}
return false;
}
"""
) )
if clicked :
if old_first_href :
try :
page . wait_for_function (
f """ () => {{
const a = document.querySelector( \" a[href*= ' /VehicleDetail/ ' ], a[href*= ' /vehicledetail/ ' ], a[href*= ' VehicleDetail ' ], a[href*= ' vehicledetail ' ] \" );
return a && a.getAttribute( ' href ' ) !== ' { old_first_href } ' ;
}} """ ,
timeout = 12000 ,
)
except Exception :
pass
else :
try :
page . wait_for_load_state ( " domcontentloaded " , timeout = 12000 )
except Exception :
pass
self . pacer . after_page_change ( )
return True
except Exception as exc :
logger . debug ( " Numeric/icon pagination fallback failed: %s " , exc )
# JS fallback: ищем любой видимый pagination-control «next» по атрибутам/тексту.
try :
clicked = bool ( page . evaluate (
"""
() => {
const candidates = Array.from(document.querySelectorAll( ' a,button,[role= " button " ] ' ));
for (const el of candidates) {
const text = (el.textContent || ' ' ).trim().toLowerCase();
const aria = (el.getAttribute( ' aria-label ' ) || ' ' ).trim().toLowerCase();
const title = (el.getAttribute( ' title ' ) || ' ' ).trim().toLowerCase();
const rel = (el.getAttribute( ' rel ' ) || ' ' ).trim().toLowerCase();
const cls = (el.getAttribute( ' class ' ) || ' ' ).trim().toLowerCase();
const disabled = el.hasAttribute( ' disabled ' ) || el.getAttribute( ' aria-disabled ' ) === ' true ' || cls.includes( ' disabled ' );
const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
const looksNext = rel === ' next ' || aria.includes( ' next ' ) || title.includes( ' next ' ) || cls.includes( ' next ' ) || [ ' next ' , ' › ' , ' » ' , ' > ' ].includes(text);
if (!disabled && visible && looksNext) {
el.click();
return true;
}
}
return false;
}
"""
) )
if clicked :
if old_first_href :
try :
page . wait_for_function (
f """ () => {{
const a = document.querySelector( \" a[href*= ' /VehicleDetail/ ' ], a[href*= ' /vehicledetail/ ' ], a[href*= ' VehicleDetail ' ], a[href*= ' vehicledetail ' ] \" );
return a && a.getAttribute( ' href ' ) !== ' { old_first_href } ' ;
}} """ ,
timeout = 8000 ,
)
except Exception :
pass
self . pacer . after_page_change ( )
return True
except Exception as exc :
logger . debug ( " JS next-page fallback failed: %s " , exc )
logger . warning ( " Could not navigate to next page from %s " , page . url )
return False
def collect_listing_links (
@@ -287,7 +538,7 @@ class ListingCollector:
except Exception :
pass
try :
page . wait_for_selector ( " a[href*= ' /VehicleDetail/ ' ] " , timeout = 3000 )
page . wait_for_selector ( VEHICLE_LINK_SELECTOR , timeout = 3000 )
except Exception :
pass
return True
@@ -297,7 +548,26 @@ class ListingCollector:
@staticmethod
def _has_next_page ( page : Page ) - > bool :
for selector in [ " a[aria-label*= ' Next ' ] " , " button[aria-label*= ' Next ' ] " , " a.pagination-next " , " button.pagination-next " , " a:has-text( ' Next ' ) " , " button:has-text( ' Next ' ) " ] :
for selector in ListingCollector . _NEXT_PAGE_SELECTORS :
if page . locator ( selector ) . count ( ) > 0 :
return True
try :
return bool ( page . evaluate (
"""
() => Array.from(document.querySelectorAll( ' a,button,[role= " button " ] ' )).some((el) => {
const text = (el.textContent || ' ' ).trim().toLowerCase();
const aria = (el.getAttribute( ' aria-label ' ) || ' ' ).trim().toLowerCase();
const title = (el.getAttribute( ' title ' ) || ' ' ).trim().toLowerCase();
const rel = (el.getAttribute( ' rel ' ) || ' ' ).trim().toLowerCase();
const cls = (el.getAttribute( ' class ' ) || ' ' ).trim().toLowerCase();
const disabled = el.hasAttribute( ' disabled ' ) || el.getAttribute( ' aria-disabled ' ) === ' true ' || cls.includes( ' disabled ' );
const visible = !!(el.offsetWidth || el.offsetHeight || el.getClientRects().length);
const hasRightArrowIcon = !!el.querySelector( ' img[src*= " icon-arrow-right " ], img[src*= " arrow-right " ] ' );
const looksNumericNext = /^ \ d+$/.test(text);
return !disabled && visible && (rel === ' next ' || aria.includes( ' next ' ) || title.includes( ' next ' ) || cls.includes( ' next ' ) || hasRightArrowIcon || looksNumericNext || [ ' next ' , ' › ' , ' » ' , ' > ' ].includes(text));
})
"""
) )
except Exception :
return False
return False
qananasikq