add shard page cap
This commit is contained in:
@@ -1023,11 +1023,21 @@ class EncarScraper:
|
|||||||
max_consecutive_errors = 5
|
max_consecutive_errors = 5
|
||||||
consecutive_zero_new = 0
|
consecutive_zero_new = 0
|
||||||
max_consecutive_zero_new = 5
|
max_consecutive_zero_new = 5
|
||||||
|
# Защита от бесконечной пагинации: Encar API отдаёт максимум
|
||||||
|
# ~10k записей на query, шарды строятся ≤9500 → не более ~10
|
||||||
|
# страниц при page_size=1000. Жёсткий cap в 50 — паранойя.
|
||||||
|
max_pages_per_shard = 50
|
||||||
|
|
||||||
shard_query = shard_filters.build_query()
|
shard_query = shard_filters.build_query()
|
||||||
shard_collected = 0
|
shard_collected = 0
|
||||||
|
|
||||||
while True:
|
while True:
|
||||||
|
if page >= max_pages_per_shard:
|
||||||
|
logger.warning(
|
||||||
|
"Shard %d hit hard page cap (%d), moving to next",
|
||||||
|
shard_idx, max_pages_per_shard,
|
||||||
|
)
|
||||||
|
break
|
||||||
offset = page * page_size
|
offset = page * page_size
|
||||||
try:
|
try:
|
||||||
response = self._fetch_listing_page(
|
response = self._fetch_listing_page(
|
||||||
|
|||||||
Reference in New Issue
Block a user