#!/usr/bin/env bash
set -euo pipefail

# VPS monitor for iaai-parser. Run on the VPS from any directory:
#   bash /root/iaai-parser/vps_monitor.sh
# Live mode:
#   watch -n 30 bash /root/iaai-parser/vps_monitor.sh

APP_DIR="${APP_DIR:-/root/iaai-parser}"
cd "$APP_DIR"

LINE='============================================================'
SMALL='------------------------------------------------------------'

section() {
  echo
  echo "$LINE"
  echo "$1"
  echo "$LINE"
}

subsection() {
  echo
  echo "$SMALL"
  echo "$1"
  echo "$SMALL"
}

run_psql() {
  docker compose exec -T postgres psql -U iaai -d iaai_scraper "$@"
}

run_redis() {
  docker compose exec -T redis redis-cli "$@"
}

now_utc="$(date -u '+%Y-%m-%d %H:%M:%S UTC')"
echo "IAAI VPS monitor | $now_utc | app=$APP_DIR"

section "1) Services / restarts"
docker compose ps

subsection "Container restart counters"
for svc in api worker beat postgres redis; do
  cid="$(docker compose ps -q "$svc" 2>/dev/null || true)"
  if [[ -z "$cid" ]]; then
    printf '%-10s %s\n' "$svc" "not found"
    continue
  fi
  name="$(docker inspect --format '{{.Name}}' "$cid" | sed 's#^/##')"
  restarts="$(docker inspect --format '{{.RestartCount}}' "$cid")"
  started="$(docker inspect --format '{{.State.StartedAt}}' "$cid")"
  status="$(docker inspect --format '{{.State.Status}}' "$cid")"
  health="$(docker inspect --format '{{if .State.Health}}{{.State.Health.Status}}{{else}}n/a{{end}}' "$cid")"
  printf '%-22s restarts=%-3s started=%s status=%s health=%s\n' "$name" "$restarts" "$started" "$status" "$health"
done

subsection "Startup / restart log markers"
docker compose logs --since '24h' worker beat 2>/dev/null \
  | grep -E 'Worker ready|startup sync|dispatching initial|Scheduler: Sending due task|warm shutdown|Restoring|mingle:|ready\.|ERROR|CRITICAL' \
  | tail -80 || true

section "2) Main DB result"
run_psql -c "
select
  count(*) as total_cars,
  count(*) filter (where is_sold = false) as active_cars,
  count(*) filter (where is_sold = true) as sold_cars,
  count(*) filter (where last_seen_at > now() - interval '5 minutes') as touched_5m,
  count(*) filter (where last_seen_at > now() - interval '15 minutes') as touched_15m,
  count(*) filter (where last_seen_at > now() - interval '60 minutes') as touched_60m,
  max(last_seen_at) as last_seen
from iaai_cars;
"

subsection "Writes from sync_runs"
run_psql -c "
select
  coalesce(sum(cars_upserted) filter (where started_at > now() - interval '1 hour'), 0) as cars_upserted_1h,
  coalesce(sum(cars_upserted) filter (where started_at > now() - interval '6 hours'), 0) as cars_upserted_6h,
  coalesce(sum(cars_upserted) filter (where started_at::date = now()::date), 0) as cars_upserted_today,
  count(*) filter (where status = 'running') as running_runs,
  count(*) filter (where status = 'success' and started_at > now() - interval '24 hours') as success_runs_24h,
  count(*) filter (where status = 'failed' and started_at > now() - interval '24 hours') as failed_runs_24h
from iaai_sync_runs;
"

section "3) Segments / passes"
subsection "Redis checkpoint / active progress"
checkpoint="$(run_redis GET iaai:state:sync_listing_checkpoint || true)"
segments_done="$(run_redis SCARD iaai:state:sync_segments_done 2>/dev/null || echo 0)"
segments_total="$(run_redis GET iaai:state:sync_segments_total || true)"
full_scan_done="$(run_redis GET iaai:state:sync_full_scan_done || true)"
queue_len="$(run_redis LLEN iaai_sync || true)"
lock_value="$(run_redis GET iaai:locks:sync_listing || true)"
echo "checkpoint_last_completed_segment=${checkpoint:-empty}"
echo "segments_done_set_count=${segments_done:-0}"
echo "segments_total=${segments_total:-unknown}"
echo "full_scan_done=${full_scan_done:-empty}"
echo "queue_len=${queue_len:-unknown}"
echo "sync_lock=${lock_value:-empty}"

echo
echo "Active/latest task progress keys:"
progress_keys="$(run_redis --scan --pattern 'iaai:state:task_progress:*' || true)"
if [[ -z "$progress_keys" ]]; then
  echo "no task_progress keys"
else
  while IFS= read -r key; do
    [[ -z "$key" ]] && continue
    echo "$key"
    run_redis GET "$key" || true
    echo
  done <<< "$progress_keys"
fi

subsection "Completed full passes by worker logs"
passes_24h="$(docker compose logs --since '24h' worker 2>/dev/null | grep -Ec 'Segment 65/65 done|Segment 64/65 done: RIMAC|Bootstrap full scan completed|sync_listing_task completed: status=success' || true)"
passes_6h="$(docker compose logs --since '6h' worker 2>/dev/null | grep -Ec 'Segment 65/65 done|Segment 64/65 done: RIMAC|Bootstrap full scan completed|sync_listing_task completed: status=success' || true)"
echo "estimated_completed_passes_6h=$passes_6h"
echo "estimated_completed_passes_24h=$passes_24h"

subsection "Recent segment events"
docker compose logs --tail=220 worker 2>/dev/null \
  | grep -E 'Segment [0-9]+/[0-9]+|Fast HTTP-first listing collected|Fast HTTP-first DB batch|sync_listing_task completed|Bootstrap full scan completed|Bootstrap continuation stopped|sync_already_running' \
  | tail -90 || true

section "4) Errors / warnings"
subsection "Failed sync_runs last 24h"
run_psql -c "
select id, status, cars_upserted, cars_failed, started_at, finished_at, left(coalesce(error_summary, ''), 180) as error_summary
from iaai_sync_runs
where status <> 'success' and started_at > now() - interval '24 hours'
order by id desc
limit 20;
"

subsection "Recent ERROR/WARNING logs"
docker compose logs --since '6h' worker beat api 2>/dev/null \
  | grep -E 'ERROR|WARNING|CRITICAL|Traceback|TimeLimitExceeded|SoftTimeLimitExceeded|Read timed out|anti-bot|circuit breaker' \
  | tail -120 || true

section "5) Schedule / hourly behavior"
subsection "Relevant environment"
docker compose exec -T worker sh -lc 'env | grep -E "IAAI_STARTUP_SYNC_ENABLED|CELERY_BEAT_SYNC_INTERVAL_MINUTES|IAAI_LISTING_SEGMENTS|IAAI_ALWAYS_FULL_SCAN|IAAI_FULL_SCAN_ON_STARTUP|IAAI_HOURLY_MODE|IAAI_HOURLY_REFRESH_BATCH_SIZE|IAAI_SELF_HEAL_STALL_SECONDS|IAAI_DB_IDLE_RESTART_SECONDS|CELERY_TASK_SOFT_TIME_LIMIT|CELERY_TASK_TIME_LIMIT|CELERY_BROKER_VISIBILITY_TIMEOUT" | sort' || true

subsection "Beat sends / skipped duplicates"
docker compose logs --since '12h' beat worker 2>/dev/null \
  | grep -E 'Scheduler: Sending due task periodic-sync-listing|sync_already_running|sync_listing_task hourly|Hourly mode|Worker ready|startup sync dispatch disabled|dispatching initial' \
  | tail -120 || true

section "6) Sold cars"
run_psql -c "
select
  count(*) filter (where is_sold = true) as sold_total,
  count(*) filter (where is_sold = true and last_seen_at > now() - interval '24 hours') as sold_seen_24h,
  min(last_seen_at) filter (where is_sold = true) as oldest_sold_seen,
  max(last_seen_at) filter (where is_sold = true) as newest_sold_seen
from iaai_cars;
"

subsection "Top sold brands"
run_psql -c "
select brand, count(*) as sold
from iaai_cars
where is_sold = true
group by brand
order by sold desc
limit 15;
"

subsection "Sold marking logs"
docker compose logs --since '24h' worker 2>/dev/null \
  | grep -E 'Marked [0-9]+ cars as sold|sold_marked|Fast sold reconcile failed|mark_sold safety abort' \
  | tail -80 || true

section "7) Quick health hints"
echo "OK if: worker/beat healthy, queue_len is 0 or small, one sync_lock exists during active run, checkpoint grows, failed_runs_24h is low."
echo "If IAAI_STARTUP_SYNC_ENABLED is empty/true, worker restart can dispatch an immediate sync. Set it to false to rely only on hourly beat."