stability and vps runtime updates
This commit is contained in:
@@ -314,6 +314,46 @@ class TestScraperSync(unittest.TestCase):
|
||||
self.assertEqual({u for u, _ in fallback_urls}, {urls[1], urls[2]})
|
||||
self.assertNotIn(urls[0], {u for u, _ in fallback_urls})
|
||||
|
||||
def test_sync_listing_marks_failed_on_unfiltered_zero_links_first_page(self) -> None:
|
||||
scraper = self._make_scraper()
|
||||
scraper.persistence.create_tables = MagicMock()
|
||||
scraper.persistence.start_sync_run = MagicMock(return_value=100)
|
||||
scraper.persistence.finish_sync_run = MagicMock()
|
||||
|
||||
scraper._sync_listing_streaming = MagicMock(return_value={
|
||||
"listing": {
|
||||
"status": "ok",
|
||||
"pages_collected": 1,
|
||||
"vehicles_collected": 0,
|
||||
"vehicle_urls": [],
|
||||
"early_stopped": False,
|
||||
"truncated_by_time_budget": False,
|
||||
"pages": [{"page_number": 1, "links_found": 0}],
|
||||
},
|
||||
"total": 0,
|
||||
"skipped_existing": 0,
|
||||
"cars_upserted": 0,
|
||||
"cars_failed": 0,
|
||||
"images_upserted": 0,
|
||||
"protection_events": 0,
|
||||
"failures": [],
|
||||
"all_listing_origin_urls": set(),
|
||||
})
|
||||
|
||||
result = scraper.sync_listing(
|
||||
make=None,
|
||||
model=None,
|
||||
lane="iaai_cars",
|
||||
limit=None,
|
||||
only_new=False,
|
||||
listing_url="https://www.iaai.com/Vehiclelisting/Cars",
|
||||
)
|
||||
|
||||
self.assertEqual(result["status"], "failed")
|
||||
self.assertTrue(result["anti_bot_detected"])
|
||||
self.assertTrue(result["suspected_ip_block"])
|
||||
self.assertTrue(any("IP/proxy block" in f.get("error", "") for f in result["failures"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user