fix docker scraping
improve batch sync add postgres upsert fix sync locking improve listing sync speed up scraper clean up project prepare for github update docker setup
This commit is contained in:
qananasikq
@@ -1,8 +1,10 @@
|
||||
# Celery-задачи для синхронизации автомобилей и листинга IAAI.
|
||||
# Задачи Celery для синхронизации автомобилей и листинга IAAI.
|
||||
|
||||
from concurrent.futures import ThreadPoolExecutor
|
||||
import json
|
||||
import logging
|
||||
from threading import Event, Thread
|
||||
import time
|
||||
import uuid
|
||||
|
||||
from celery import shared_task
|
||||
from redis import Redis
|
||||
@@ -17,9 +19,7 @@ SYNC_LISTING_LOCK_KEY = "iaai:locks:sync_listing"
|
||||
|
||||
|
||||
def _run_browser_job(func, *args, **kwargs):
|
||||
# Playwright Sync API нельзя запускать в потоке с активным asyncio loop.
|
||||
# Celery/зависимости могут поднимать loop в worker-процессе, поэтому
|
||||
# браузерный код выполняем в отдельном thread без loop.
|
||||
# Браузерный код запускаем в отдельном потоке без активного loop.
|
||||
with ThreadPoolExecutor(max_workers=1, thread_name_prefix="iaai-browser") as executor:
|
||||
future = executor.submit(func, *args, **kwargs)
|
||||
return future.result()
|
||||
@@ -27,7 +27,7 @@ def _run_browser_job(func, *args, **kwargs):
|
||||
|
||||
def _sync_listing_lock_ttl_seconds() -> int:
|
||||
settings = Settings()
|
||||
# Небольшой запас к hard time limit задачи, чтобы lock самоснимался после сбоев.
|
||||
# Небольшой запас к лимиту времени, чтобы lock снимался после сбоев.
|
||||
return max(settings.celery.task_time_limit + 120, 300)
|
||||
|
||||
|
||||
@@ -40,35 +40,70 @@ def _get_redis() -> Redis:
|
||||
return Redis.from_url(settings.redis.url, decode_responses=True)
|
||||
|
||||
|
||||
def _release_lock_if_owner(redis_client: Redis, key: str, owner: str) -> None:
|
||||
def _acquire_lock(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool:
|
||||
try:
|
||||
current_owner = redis_client.get(key)
|
||||
if current_owner == owner:
|
||||
redis_client.delete(key)
|
||||
return bool(redis_client.set(key, owner_token, nx=True, ex=ttl_seconds))
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to release lock %s: %s", key, exc)
|
||||
|
||||
|
||||
def _has_other_active_sync_listing_task(task) -> bool:
|
||||
try:
|
||||
inspector = task.app.control.inspect(timeout=1.0)
|
||||
active_map = inspector.active() or {}
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to inspect active tasks: %s", exc)
|
||||
logger.warning("Failed to acquire lock %s", key, exc_info=True)
|
||||
return False
|
||||
|
||||
current_task_id = task.request.id
|
||||
for worker_tasks in active_map.values():
|
||||
for item in worker_tasks or []:
|
||||
name = str(item.get("name") or "")
|
||||
task_id = str(item.get("id") or "")
|
||||
if (
|
||||
name == "iaai_scraper.worker.tasks.sync_listing_task"
|
||||
and task_id
|
||||
and task_id != current_task_id
|
||||
):
|
||||
return True
|
||||
return False
|
||||
|
||||
def _refresh_lock_if_owner(redis_client: Redis, key: str, owner_token: str, ttl_seconds: int) -> bool | None:
|
||||
try:
|
||||
refreshed = redis_client.eval(
|
||||
"""
|
||||
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
||||
return redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2]))
|
||||
end
|
||||
return 0
|
||||
""",
|
||||
1,
|
||||
key,
|
||||
owner_token,
|
||||
int(ttl_seconds),
|
||||
)
|
||||
return bool(refreshed)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to refresh lock %s", key, exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
def _release_lock_if_owner(redis_client: Redis, key: str, owner_token: str) -> None:
|
||||
try:
|
||||
redis_client.eval(
|
||||
"""
|
||||
if redis.call('GET', KEYS[1]) == ARGV[1] then
|
||||
return redis.call('DEL', KEYS[1])
|
||||
end
|
||||
return 0
|
||||
""",
|
||||
1,
|
||||
key,
|
||||
owner_token,
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to release lock %s", key, exc_info=True)
|
||||
|
||||
|
||||
def _start_lock_heartbeat(
|
||||
redis_client: Redis,
|
||||
key: str,
|
||||
owner_token: str,
|
||||
ttl_seconds: int,
|
||||
) -> tuple[Event, Thread]:
|
||||
stop_event = Event()
|
||||
interval_seconds = max(5.0, min(30.0, ttl_seconds / 3))
|
||||
|
||||
def _heartbeat() -> None:
|
||||
while not stop_event.wait(interval_seconds):
|
||||
refreshed = _refresh_lock_if_owner(redis_client, key, owner_token, ttl_seconds)
|
||||
if refreshed is False:
|
||||
logger.warning("Lost sync_listing lock ownership for %s", owner_token)
|
||||
return
|
||||
|
||||
thread = Thread(target=_heartbeat, name="sync-listing-lock-heartbeat", daemon=True)
|
||||
thread.start()
|
||||
return stop_event, thread
|
||||
|
||||
|
||||
@shared_task(
|
||||
@@ -98,7 +133,7 @@ def sync_vehicle_task(self, vehicle_url: str, lane: str = "iaai"):
|
||||
}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc)
|
||||
logger.error("sync_vehicle_task failed: %s — %s", vehicle_url, exc, exc_info=True)
|
||||
raise self.retry(exc=exc)
|
||||
|
||||
|
||||
@@ -121,44 +156,15 @@ def sync_listing_task(
|
||||
persistence = _get_persistence()
|
||||
persistence.create_tables()
|
||||
task_id = self.request.id or "unknown"
|
||||
owner_token = f"{task_id}:{uuid.uuid4().hex}"
|
||||
redis_client = _get_redis()
|
||||
|
||||
lock_acquired = False
|
||||
lock_ttl = _sync_listing_lock_ttl_seconds()
|
||||
try:
|
||||
lock_acquired = bool(
|
||||
redis_client.set(
|
||||
SYNC_LISTING_LOCK_KEY,
|
||||
task_id,
|
||||
nx=True,
|
||||
ex=lock_ttl,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to acquire sync lock in Redis: %s", exc)
|
||||
heartbeat_stop: Event | None = None
|
||||
heartbeat_thread: Thread | None = None
|
||||
|
||||
if not lock_acquired:
|
||||
# Возможен stale lock после рестарта worker. Если активного sync_listing нет —
|
||||
# снимаем lock и пытаемся взять его заново.
|
||||
if not _has_other_active_sync_listing_task(self):
|
||||
try:
|
||||
stale_owner = redis_client.get(SYNC_LISTING_LOCK_KEY)
|
||||
if stale_owner:
|
||||
logger.warning(
|
||||
"Removing stale sync lock held by task %s",
|
||||
stale_owner,
|
||||
)
|
||||
redis_client.delete(SYNC_LISTING_LOCK_KEY)
|
||||
lock_acquired = bool(
|
||||
redis_client.set(
|
||||
SYNC_LISTING_LOCK_KEY,
|
||||
task_id,
|
||||
nx=True,
|
||||
ex=lock_ttl,
|
||||
)
|
||||
)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to recover stale sync lock: %s", exc)
|
||||
lock_acquired = _acquire_lock(redis_client, SYNC_LISTING_LOCK_KEY, owner_token, lock_ttl)
|
||||
|
||||
if not lock_acquired:
|
||||
logger.info("sync_listing_task skipped: another sync is already running")
|
||||
@@ -169,6 +175,12 @@ def sync_listing_task(
|
||||
}
|
||||
|
||||
try:
|
||||
heartbeat_stop, heartbeat_thread = _start_lock_heartbeat(
|
||||
redis_client,
|
||||
SYNC_LISTING_LOCK_KEY,
|
||||
owner_token,
|
||||
lock_ttl,
|
||||
)
|
||||
self.update_state(state="STARTED", meta={"stage": "sync_listing_started", "task_id": task_id})
|
||||
def _job():
|
||||
with IAAIScraper() as scraper:
|
||||
@@ -198,8 +210,12 @@ def sync_listing_task(
|
||||
return {"status": "success", **summary}
|
||||
|
||||
except Exception as exc:
|
||||
logger.error("sync_listing_task failed: %s", exc)
|
||||
logger.error("sync_listing_task failed: %s", exc, exc_info=True)
|
||||
raise self.retry(exc=exc)
|
||||
finally:
|
||||
if heartbeat_stop is not None:
|
||||
heartbeat_stop.set()
|
||||
if heartbeat_thread is not None:
|
||||
heartbeat_thread.join(timeout=max(1.0, min(5.0, lock_ttl / 10)))
|
||||
if lock_acquired:
|
||||
_release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, task_id)
|
||||
_release_lock_if_owner(redis_client, SYNC_LISTING_LOCK_KEY, owner_token)
|
||||
|
||||
Reference in New Issue
Block a user