fix: eliminate greenlet crash + memory leak protection

- docker-compose: switch worker to --pool=solo --concurrency=1 --max-tasks-per-child=1
- tasks.py: remove ThreadPoolExecutor wrapper from _run_browser_job (greenlet crash)
- scraper.py: browser fallback runs sequentially instead of ThreadPoolExecutor
- scraper.py: add gc.collect() after scraper close to prevent memory leaks
This commit is contained in:
qananasikq
2026-04-20 17:33:51 +03:00
parent 3c71c098cd
commit 133c125e9c
3 changed files with 16 additions and 45 deletions

View File

@@ -1,3 +1,4 @@
import gc
import json
import logging
import os
@@ -201,6 +202,7 @@ class IAAIScraper:
pass
finally:
self._http_pool = None
gc.collect()
def _new_context(self) -> BrowserContext:
if self.browser is None:
@@ -1623,23 +1625,17 @@ class IAAIScraper:
"protection_events": local_protection,
}
# Одна страница — в главном потоке.
if n_pages == 1:
res = _process_slice(slices[0] if slices else [])
# Playwright sync API использует greenlets, привязанные к потоку.
# Запуск в ThreadPoolExecutor вызывает greenlet crash.
# Обрабатываем все слайсы последовательно в текущем потоке.
for s in slices:
if not s:
continue
res = _process_slice(s)
records.extend(res["records"])
failures.extend(res["failures"])
cars_failed += res["cars_failed"]
protection_events += res["protection_events"]
else:
# Несколько страниц — параллельно, каждый поток со своей Page.
with ThreadPoolExecutor(max_workers=n_pages) as executor:
futures = [executor.submit(_process_slice, s) for s in slices if s]
for fut in as_completed(futures):
res = fut.result()
records.extend(res["records"])
failures.extend(res["failures"])
cars_failed += res["cars_failed"]
protection_events += res["protection_events"]
return {
"records": records,