add shard page cap
This commit is contained in:
@@ -1023,11 +1023,21 @@ class EncarScraper:
|
||||
max_consecutive_errors = 5
|
||||
consecutive_zero_new = 0
|
||||
max_consecutive_zero_new = 5
|
||||
# Защита от бесконечной пагинации: Encar API отдаёт максимум
|
||||
# ~10k записей на query, шарды строятся ≤9500 → не более ~10
|
||||
# страниц при page_size=1000. Жёсткий cap в 50 — паранойя.
|
||||
max_pages_per_shard = 50
|
||||
|
||||
shard_query = shard_filters.build_query()
|
||||
shard_collected = 0
|
||||
|
||||
while True:
|
||||
if page >= max_pages_per_shard:
|
||||
logger.warning(
|
||||
"Shard %d hit hard page cap (%d), moving to next",
|
||||
shard_idx, max_pages_per_shard,
|
||||
)
|
||||
break
|
||||
offset = page * page_size
|
||||
try:
|
||||
response = self._fetch_listing_page(
|
||||
|
||||
Reference in New Issue
Block a user