179 lines
7.3 KiB
Bash
179 lines
7.3 KiB
Bash
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# VPS monitor for iaai-parser. Run on the VPS from any directory:
|
|
# bash /root/iaai-parser/vps_monitor.sh
|
|
# Live mode:
|
|
# watch -n 30 bash /root/iaai-parser/vps_monitor.sh
|
|
|
|
APP_DIR="${APP_DIR:-/root/iaai-parser}"
|
|
cd "$APP_DIR"
|
|
|
|
LINE='============================================================'
|
|
SMALL='------------------------------------------------------------'
|
|
|
|
section() {
|
|
echo
|
|
echo "$LINE"
|
|
echo "$1"
|
|
echo "$LINE"
|
|
}
|
|
|
|
subsection() {
|
|
echo
|
|
echo "$SMALL"
|
|
echo "$1"
|
|
echo "$SMALL"
|
|
}
|
|
|
|
run_psql() {
|
|
docker compose exec -T postgres psql -U iaai -d iaai_scraper "$@"
|
|
}
|
|
|
|
run_redis() {
|
|
docker compose exec -T redis redis-cli "$@"
|
|
}
|
|
|
|
now_utc="$(date -u '+%Y-%m-%d %H:%M:%S UTC')"
|
|
echo "IAAI VPS monitor | $now_utc | app=$APP_DIR"
|
|
|
|
section "1) Services / restarts"
|
|
docker compose ps
|
|
|
|
subsection "Container restart counters"
|
|
for svc in api worker beat postgres redis; do
|
|
cid="$(docker compose ps -q "$svc" 2>/dev/null || true)"
|
|
if [[ -z "$cid" ]]; then
|
|
printf '%-10s %s\n' "$svc" "not found"
|
|
continue
|
|
fi
|
|
name="$(docker inspect --format '{{.Name}}' "$cid" | sed 's#^/##')"
|
|
restarts="$(docker inspect --format '{{.RestartCount}}' "$cid")"
|
|
started="$(docker inspect --format '{{.State.StartedAt}}' "$cid")"
|
|
status="$(docker inspect --format '{{.State.Status}}' "$cid")"
|
|
health="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}}' "$cid")"
|
|
printf '%-22s restarts=%-3s started=%s status=%s health=%s\n' "$name" "$restarts" "$started" "$status" "$health"
|
|
done
|
|
|
|
subsection "Startup / restart log markers"
|
|
docker compose logs --since '24h' worker beat 2>/dev/null \
|
|
| grep -E 'Worker ready|startup sync|dispatching initial|Scheduler: Sending due task|warm shutdown|Restoring|mingle:|ready\.|ERROR|CRITICAL' \
|
|
| tail -80 || true
|
|
|
|
section "2) Main DB result"
|
|
run_psql -c "
|
|
select
|
|
count(*) as total_cars,
|
|
count(*) filter (where is_sold = false) as active_cars,
|
|
count(*) filter (where is_sold = true) as sold_cars,
|
|
count(*) filter (where last_seen_at > now() - interval '5 minutes') as touched_5m,
|
|
count(*) filter (where last_seen_at > now() - interval '15 minutes') as touched_15m,
|
|
count(*) filter (where last_seen_at > now() - interval '60 minutes') as touched_60m,
|
|
max(last_seen_at) as last_seen
|
|
from iaai_cars;
|
|
"
|
|
|
|
subsection "Writes from sync_runs"
|
|
run_psql -c "
|
|
select
|
|
coalesce(sum(cars_upserted) filter (where started_at > now() - interval '1 hour'), 0) as cars_upserted_1h,
|
|
coalesce(sum(cars_upserted) filter (where started_at > now() - interval '6 hours'), 0) as cars_upserted_6h,
|
|
coalesce(sum(cars_upserted) filter (where started_at::date = now()::date), 0) as cars_upserted_today,
|
|
count(*) filter (where status = 'running') as running_runs,
|
|
count(*) filter (where status = 'success' and started_at > now() - interval '24 hours') as success_runs_24h,
|
|
count(*) filter (where status = 'failed' and started_at > now() - interval '24 hours') as failed_runs_24h
|
|
from iaai_sync_runs;
|
|
"
|
|
|
|
section "3) Segments / passes"
|
|
subsection "Redis checkpoint / active progress"
|
|
checkpoint="$(run_redis GET iaai:state:sync_listing_checkpoint || true)"
|
|
segments_done="$(run_redis SCARD iaai:state:sync_segments_done 2>/dev/null || echo 0)"
|
|
segments_total="$(run_redis GET iaai:state:sync_segments_total || true)"
|
|
full_scan_done="$(run_redis GET iaai:state:sync_full_scan_done || true)"
|
|
queue_len="$(run_redis LLEN iaai_sync || true)"
|
|
lock_value="$(run_redis GET iaai:locks:sync_listing || true)"
|
|
echo "checkpoint_last_completed_segment=${checkpoint:-empty}"
|
|
echo "segments_done_set_count=${segments_done:-0}"
|
|
echo "segments_total=${segments_total:-unknown}"
|
|
echo "full_scan_done=${full_scan_done:-empty}"
|
|
echo "queue_len=${queue_len:-unknown}"
|
|
echo "sync_lock=${lock_value:-empty}"
|
|
|
|
echo
|
|
echo "Active/latest task progress keys:"
|
|
progress_keys="$(run_redis --scan --pattern 'iaai:state:task_progress:*' || true)"
|
|
if [[ -z "$progress_keys" ]]; then
|
|
echo "no task_progress keys"
|
|
else
|
|
while IFS= read -r key; do
|
|
[[ -z "$key" ]] && continue
|
|
echo "$key"
|
|
run_redis GET "$key" || true
|
|
echo
|
|
done <<< "$progress_keys"
|
|
fi
|
|
|
|
subsection "Completed full passes by worker logs"
|
|
passes_24h="$(docker compose logs --since '24h' worker 2>/dev/null | grep -Ec 'Segment 65/65 done|Segment 64/65 done: RIMAC|Bootstrap full scan completed|sync_listing_task completed: status=success' || true)"
|
|
passes_6h="$(docker compose logs --since '6h' worker 2>/dev/null | grep -Ec 'Segment 65/65 done|Segment 64/65 done: RIMAC|Bootstrap full scan completed|sync_listing_task completed: status=success' || true)"
|
|
echo "estimated_completed_passes_6h=$passes_6h"
|
|
echo "estimated_completed_passes_24h=$passes_24h"
|
|
|
|
subsection "Recent segment events"
|
|
docker compose logs --tail=220 worker 2>/dev/null \
|
|
| grep -E 'Segment [0-9]+/[0-9]+|Fast HTTP-first listing collected|Fast HTTP-first DB batch|sync_listing_task completed|Bootstrap full scan completed|Bootstrap continuation stopped|sync_already_running' \
|
|
| tail -90 || true
|
|
|
|
section "4) Errors / warnings"
|
|
subsection "Failed sync_runs last 24h"
|
|
run_psql -c "
|
|
select id, status, cars_upserted, cars_failed, started_at, finished_at, left(coalesce(error_summary, ''), 180) as error_summary
|
|
from iaai_sync_runs
|
|
where status <> 'success' and started_at > now() - interval '24 hours'
|
|
order by id desc
|
|
limit 20;
|
|
"
|
|
|
|
subsection "Recent ERROR/WARNING logs"
|
|
docker compose logs --since '6h' worker beat api 2>/dev/null \
|
|
| grep -E 'ERROR|WARNING|CRITICAL|Traceback|TimeLimitExceeded|SoftTimeLimitExceeded|Read timed out|anti-bot|circuit breaker' \
|
|
| tail -120 || true
|
|
|
|
section "5) Schedule / hourly behavior"
|
|
subsection "Relevant environment"
|
|
docker compose exec -T worker sh -lc 'env | grep -E "IAAI_STARTUP_SYNC_ENABLED|CELERY_BEAT_SYNC_INTERVAL_MINUTES|IAAI_LISTING_SEGMENTS|IAAI_ALWAYS_FULL_SCAN|IAAI_FULL_SCAN_ON_STARTUP|IAAI_HOURLY_MODE|IAAI_HOURLY_REFRESH_BATCH_SIZE|IAAI_SELF_HEAL_STALL_SECONDS|IAAI_DB_IDLE_RESTART_SECONDS|CELERY_TASK_SOFT_TIME_LIMIT|CELERY_TASK_TIME_LIMIT|CELERY_BROKER_VISIBILITY_TIMEOUT" | sort' || true
|
|
|
|
subsection "Beat sends / skipped duplicates"
|
|
docker compose logs --since '12h' beat worker 2>/dev/null \
|
|
| grep -E 'Scheduler: Sending due task periodic-sync-listing|sync_already_running|sync_listing_task hourly|Hourly mode|Worker ready|startup sync dispatch disabled|dispatching initial' \
|
|
| tail -120 || true
|
|
|
|
section "6) Sold cars"
|
|
run_psql -c "
|
|
select
|
|
count(*) filter (where is_sold = true) as sold_total,
|
|
count(*) filter (where is_sold = true and last_seen_at > now() - interval '24 hours') as sold_seen_24h,
|
|
min(last_seen_at) filter (where is_sold = true) as oldest_sold_seen,
|
|
max(last_seen_at) filter (where is_sold = true) as newest_sold_seen
|
|
from iaai_cars;
|
|
"
|
|
|
|
subsection "Top sold brands"
|
|
run_psql -c "
|
|
select brand, count(*) as sold
|
|
from iaai_cars
|
|
where is_sold = true
|
|
group by brand
|
|
order by sold desc
|
|
limit 15;
|
|
"
|
|
|
|
subsection "Sold marking logs"
|
|
docker compose logs --since '24h' worker 2>/dev/null \
|
|
| grep -E 'Marked [0-9]+ cars as sold|sold_marked|Fast sold reconcile failed|mark_sold safety abort' \
|
|
| tail -80 || true
|
|
|
|
section "7) Quick health hints"
|
|
echo "OK if: worker/beat healthy, queue_len is 0 or small, one sync_lock exists during active run, checkpoint grows, failed_runs_24h is low."
|
|
echo "If IAAI_STARTUP_SYNC_ENABLED is empty/true, worker restart can dispatch an immediate sync. Set it to false to rely only on hourly beat." |