118 lines
3.9 KiB
Python
118 lines
3.9 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
|
|
from iaai_scraper.browser.fast_client import (
|
|
DETAIL_MARKER,
|
|
LISTING_MARKER,
|
|
build_resizer_images_from_keys,
|
|
is_challenge_response,
|
|
parse_listing_page,
|
|
parse_product_details_vm,
|
|
)
|
|
from iaai_scraper.parsing.fast_mapper import FastCarMapper
|
|
|
|
|
|
def test_parse_listing_page_extracts_hidden_payloads() -> None:
|
|
gbp = {"CurrentPage": 1, "PageSize": 100}
|
|
vehicles = [
|
|
{
|
|
"Id": "45078011~US",
|
|
"Tenant": "US",
|
|
"InventoryStatus": "RS",
|
|
"Currency": "USD",
|
|
"PreBidIndicator": True,
|
|
}
|
|
]
|
|
html = (
|
|
f'<input id="GBPSearchQuery" value="{json.dumps(gbp).replace(chr(34), """)}" />'
|
|
f'<input id="VehicleDetails" value="{json.dumps(vehicles).replace(chr(34), """)}" />'
|
|
'<input id="ResultCount" value="1" />'
|
|
'<input id="PageSize" value="100" />'
|
|
'<input id="CurrentPage" value="1" />'
|
|
)
|
|
|
|
parsed = parse_listing_page(html)
|
|
|
|
assert parsed.result_count == 1
|
|
assert parsed.page_size == 100
|
|
assert parsed.current_page == 1
|
|
assert parsed.vehicles[0].inventory_id == "45078011~US"
|
|
assert parsed.vehicles[0].inventory_status == "RS"
|
|
|
|
|
|
def test_parse_product_details_vm_and_map_record() -> None:
|
|
payload = {
|
|
"inventoryView": {
|
|
"attributes": {
|
|
"Id": "45078011~US",
|
|
"Year": "2002",
|
|
"Make": "ACURA",
|
|
"Model": "RSX",
|
|
"Series": "BASE",
|
|
"Currency": "USD",
|
|
"ODOValue": "219187",
|
|
"ExteriorColor": "GRAY",
|
|
"DriveLineTypeDesc": "FWD",
|
|
"Transmission": "Automatic",
|
|
"BodyStyleName": "COUPE",
|
|
"EngineSize": "2.0L I-4",
|
|
"VehicleGrade": "50",
|
|
"PrimaryDamageDesc": "NORMAL WEAR & TEAR",
|
|
},
|
|
"imageDimensions": {
|
|
"keys": {
|
|
"$values": [
|
|
{"k": "45078011~US~I1", "w": 1600, "h": 1200, "i": 0},
|
|
]
|
|
}
|
|
},
|
|
},
|
|
"auctionInformation": {
|
|
"prebidInformation": {"decimalHighBidAmount": "600", "highBidAmount": "$600"},
|
|
"biddingInformation": {"buyNowPrice": "$0"},
|
|
},
|
|
}
|
|
html = f'<script id="ProductDetailsVM" type="application/json">{json.dumps(payload)}</script>'
|
|
|
|
parsed = parse_product_details_vm(html)
|
|
record = FastCarMapper().map_payload_to_record(
|
|
detail_payload=parsed,
|
|
vehicle_url="https://www.iaai.com/VehicleDetail/45078011~US",
|
|
)
|
|
|
|
assert record.origin_id == "iaai:45078011~US"
|
|
assert record.brand == "ACURA"
|
|
assert record.model == "RSX BASE"
|
|
assert record.year == 2002
|
|
assert record.price == 600
|
|
assert record.drive == "FWD"
|
|
assert record.gearbox == "AT"
|
|
assert record.body_type == "COUPE"
|
|
assert record.engine_volume == 2000
|
|
assert record.is_damaged is False
|
|
assert len(record.images) == 1
|
|
|
|
|
|
def test_build_resizer_images_from_keys_deduplicates_and_orders() -> None:
|
|
keys = [
|
|
{"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
|
|
{"k": "abc~1", "w": 2000, "h": 1500, "i": 2},
|
|
{"k": "abc~2", "w": 1800, "h": 1200, "i": 1},
|
|
]
|
|
images = build_resizer_images_from_keys(keys)
|
|
|
|
assert len(images) == 2
|
|
assert images[0]["order_index"] == 1
|
|
assert "imageKeys=abc~2" in str(images[0]["fullres_image"])
|
|
|
|
|
|
def test_is_challenge_response_marker_rules() -> None:
|
|
assert is_challenge_response(status_code=403, body_text="", expected_marker=None) is True
|
|
assert is_challenge_response(status_code=200, body_text="<html>blocked</html>", expected_marker=LISTING_MARKER) is True
|
|
assert is_challenge_response(
|
|
status_code=200,
|
|
body_text=f'<html>{DETAIL_MARKER}<script src="/_Incapsula_Resource"></script></html>',
|
|
expected_marker=DETAIL_MARKER,
|
|
) is False
|