add shard page cap

This commit is contained in:
qananasikq
2026-04-17 23:53:53 +03:00
parent 53eb323287
commit 7f74793812

View File

@@ -1023,11 +1023,21 @@ class EncarScraper:
max_consecutive_errors = 5 max_consecutive_errors = 5
consecutive_zero_new = 0 consecutive_zero_new = 0
max_consecutive_zero_new = 5 max_consecutive_zero_new = 5
# Защита от бесконечной пагинации: Encar API отдаёт максимум
# ~10k записей на query, шарды строятся ≤9500 → не более ~10
# страниц при page_size=1000. Жёсткий cap в 50 — паранойя.
max_pages_per_shard = 50
shard_query = shard_filters.build_query() shard_query = shard_filters.build_query()
shard_collected = 0 shard_collected = 0
while True: while True:
if page >= max_pages_per_shard:
logger.warning(
"Shard %d hit hard page cap (%d), moving to next",
shard_idx, max_pages_per_shard,
)
break
offset = page * page_size offset = page * page_size
try: try:
response = self._fetch_listing_page( response = self._fetch_listing_page(