fix: eliminate greenlet crash + memory leak protection
- docker-compose: switch worker to --pool=solo --concurrency=1 --max-tasks-per-child=1 - tasks.py: remove ThreadPoolExecutor wrapper from _run_browser_job (greenlet crash) - scraper.py: browser fallback runs sequentially instead of ThreadPoolExecutor - scraper.py: add gc.collect() after scraper close to prevent memory leaks
This commit is contained in:
@@ -1,3 +1,4 @@
|
||||
import gc
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -201,6 +202,7 @@ class IAAIScraper:
|
||||
pass
|
||||
finally:
|
||||
self._http_pool = None
|
||||
gc.collect()
|
||||
|
||||
def _new_context(self) -> BrowserContext:
|
||||
if self.browser is None:
|
||||
@@ -1623,23 +1625,17 @@ class IAAIScraper:
|
||||
"protection_events": local_protection,
|
||||
}
|
||||
|
||||
# Одна страница — в главном потоке.
|
||||
if n_pages == 1:
|
||||
res = _process_slice(slices[0] if slices else [])
|
||||
# Playwright sync API использует greenlets, привязанные к потоку.
|
||||
# Запуск в ThreadPoolExecutor вызывает greenlet crash.
|
||||
# Обрабатываем все слайсы последовательно в текущем потоке.
|
||||
for s in slices:
|
||||
if not s:
|
||||
continue
|
||||
res = _process_slice(s)
|
||||
records.extend(res["records"])
|
||||
failures.extend(res["failures"])
|
||||
cars_failed += res["cars_failed"]
|
||||
protection_events += res["protection_events"]
|
||||
else:
|
||||
# Несколько страниц — параллельно, каждый поток со своей Page.
|
||||
with ThreadPoolExecutor(max_workers=n_pages) as executor:
|
||||
futures = [executor.submit(_process_slice, s) for s in slices if s]
|
||||
for fut in as_completed(futures):
|
||||
res = fut.result()
|
||||
records.extend(res["records"])
|
||||
failures.extend(res["failures"])
|
||||
cars_failed += res["cars_failed"]
|
||||
protection_events += res["protection_events"]
|
||||
|
||||
return {
|
||||
"records": records,
|
||||
|
||||
Reference in New Issue
Block a user