From d9c95b1a9f3b98aae3eb3c40700cc08cfba1aa3a Mon Sep 17 00:00:00 2001 From: qananasikq Date: Mon, 27 Apr 2026 20:47:44 +0300 Subject: [PATCH] Add tools --- check_vps.py | 37 +++++++++++++ clean_vps_db.py | 41 +++++++++++++++ deploy_vps.py | 135 ++++++++++++++++++++++++++++++++++++++++++++++++ status_vps.py | 38 ++++++++++++++ 4 files changed, 251 insertions(+) create mode 100644 check_vps.py create mode 100644 clean_vps_db.py create mode 100644 deploy_vps.py create mode 100644 status_vps.py diff --git a/check_vps.py b/check_vps.py new file mode 100644 index 0000000..af1943a --- /dev/null +++ b/check_vps.py @@ -0,0 +1,37 @@ +from __future__ import annotations + +import os +import paramiko + +HOST = "2.26.123.84" +USER = "root" +PASSWORD = os.environ["VPS_PASSWORD"] + +cmds = [ + "cd /root/iaai-parser && docker compose ps", + "docker logs --since 3m iaai-parser-worker-1 2>&1 | tail -n 120", + "docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c \"SELECT now() AS ts, count(*) AS cars FROM iaai_cars; SELECT count(*) AS runs FROM iaai_sync_runs;\"", +] + +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +client.connect( + HOST, + username=USER, + password=PASSWORD, + look_for_keys=False, + allow_agent=False, + timeout=30, + banner_timeout=30, + auth_timeout=30, +) +try: + for cmd in cmds: + print(f"REMOTE_RUN {cmd}", flush=True) + stdin, stdout, stderr = client.exec_command(cmd, timeout=120) + code = stdout.channel.recv_exit_status() + print(stdout.read().decode("utf-8", "replace"), end="") + print(stderr.read().decode("utf-8", "replace"), end="") + print(f"EXIT {code}", flush=True) +finally: + client.close() diff --git a/clean_vps_db.py b/clean_vps_db.py new file mode 100644 index 0000000..fab920c --- /dev/null +++ b/clean_vps_db.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import os +import paramiko + +HOST = "2.26.123.84" +USER = "root" +PASSWORD = os.environ["VPS_PASSWORD"] + +commands = [ + "cd /root/iaai-parser && docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c 'TRUNCATE TABLE iaai_images, iaai_cars, iaai_sync_runs RESTART IDENTITY CASCADE;'", + "docker exec -i iaai-redis redis-cli FLUSHALL", + "docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c 'SELECT count(*) AS cars FROM iaai_cars; SELECT count(*) AS runs FROM iaai_sync_runs;'", + "cd /root/iaai-parser && docker compose ps", + "docker logs --since 2m iaai-parser-worker-1 2>&1 | tail -n 100", +] + +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +client.connect( + HOST, + username=USER, + password=PASSWORD, + look_for_keys=False, + allow_agent=False, + timeout=30, + banner_timeout=30, + auth_timeout=30, +) +try: + for command in commands: + print(f"REMOTE_RUN {command}", flush=True) + stdin, stdout, stderr = client.exec_command(command, timeout=180) + exit_code = stdout.channel.recv_exit_status() + print(stdout.read().decode("utf-8", "replace"), end="") + print(stderr.read().decode("utf-8", "replace"), end="") + print(f"EXIT {exit_code}", flush=True) + if exit_code != 0: + raise SystemExit(exit_code) +finally: + client.close() diff --git a/deploy_vps.py b/deploy_vps.py new file mode 100644 index 0000000..1afe4cb --- /dev/null +++ b/deploy_vps.py @@ -0,0 +1,135 @@ +from __future__ import annotations + +import os +import posixpath +import stat +import tarfile +import time +import socket +from pathlib import Path + +import paramiko + +HOST = "2.26.123.84" +USER = "root" +PASSWORD = os.environ["VPS_PASSWORD"] +LOCAL_ROOT = Path(__file__).resolve().parent +ARCHIVE = LOCAL_ROOT / "iaai_deploy.tar.gz" +REMOTE_DIR = "/root/iaai-parser" +REMOTE_ARCHIVE = "/root/iaai_deploy.tar.gz" + +EXCLUDE_DIRS = { + ".git", + ".venv", + "__pycache__", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", + "iaai_scraper.egg-info", +} +EXCLUDE_FILES = {"iaai_deploy.tar.gz", "deploy_vps.py"} + + +def _include(path: Path) -> bool: + rel = path.relative_to(LOCAL_ROOT) + parts = set(rel.parts) + if parts & EXCLUDE_DIRS: + return False + if path.name in EXCLUDE_FILES: + return False + if path.suffix in {".pyc", ".pyo"}: + return False + return True + + +def make_archive() -> None: + if ARCHIVE.exists(): + ARCHIVE.unlink() + with tarfile.open(ARCHIVE, "w:gz") as tf: + for path in LOCAL_ROOT.rglob("*"): + if not _include(path): + continue + tf.add(path, arcname=str(path.relative_to(LOCAL_ROOT))) + print(f"ARCHIVE_READY {ARCHIVE} {ARCHIVE.stat().st_size} bytes", flush=True) + + +def connect() -> paramiko.SSHClient: + last_exc: BaseException | None = None + for attempt in range(1, 6): + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + print(f"SSH_CONNECT attempt={attempt}", flush=True) + client.connect( + HOST, + username=USER, + password=PASSWORD, + look_for_keys=False, + allow_agent=False, + timeout=45, + banner_timeout=60, + auth_timeout=45, + ) + return client + except (paramiko.SSHException, socket.timeout, OSError) as exc: + last_exc = exc + client.close() + print(f"SSH_CONNECT_RETRY attempt={attempt} error={type(exc).__name__}: {exc}", flush=True) + time.sleep(3 * attempt) + raise SystemExit(f"SSH_CONNECT_FAILED: {last_exc}") + + +def run(client: paramiko.SSHClient, cmd: str, timeout: int | None = None) -> None: + print(f"REMOTE_RUN {cmd}", flush=True) + stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout) + exit_code = stdout.channel.recv_exit_status() + out = stdout.read().decode("utf-8", "replace") + err = stderr.read().decode("utf-8", "replace") + if out: + print(out, end="", flush=True) + if err: + print(err, end="", flush=True) + if exit_code != 0: + raise SystemExit(f"REMOTE_FAILED code={exit_code}: {cmd}") + + +def upload_file(sftp: paramiko.SFTPClient, local: Path, remote: str) -> None: + total = local.stat().st_size + last = 0.0 + + def cb(done: int, _total: int) -> None: + nonlocal last + now = time.time() + if now - last >= 2 or done == total: + print(f"UPLOAD {done}/{total}", flush=True) + last = now + + sftp.put(str(local), remote, callback=cb) + + +def main() -> None: + make_archive() + client = connect() + try: + run(client, "echo VPS_OK && hostname && docker --version && docker compose version") + sftp = client.open_sftp() + try: + upload_file(sftp, ARCHIVE, REMOTE_ARCHIVE) + finally: + sftp.close() + run( + client, + f"mkdir -p {REMOTE_DIR} && cd {REMOTE_DIR} && docker compose down || true && " + f"find {REMOTE_DIR} -mindepth 1 -maxdepth 1 ! -name '.env' -exec rm -rf {{}} + && " + f"tar -xzf {REMOTE_ARCHIVE} -C {REMOTE_DIR} && rm -f {REMOTE_ARCHIVE}", + timeout=300, + ) + run(client, f"cd {REMOTE_DIR} && docker compose up -d --build", timeout=1800) + run(client, f"cd {REMOTE_DIR} && docker compose ps", timeout=120) + run(client, "docker logs --since 2m iaai-parser-worker-1 2>&1 | tail -n 120 || docker compose -f /root/iaai-parser/docker-compose.yml logs --since 2m worker | tail -n 120", timeout=120) + finally: + client.close() + + +if __name__ == "__main__": + main() diff --git a/status_vps.py b/status_vps.py new file mode 100644 index 0000000..cc4b544 --- /dev/null +++ b/status_vps.py @@ -0,0 +1,38 @@ +from __future__ import annotations + +import os +import paramiko + +HOST = "2.26.123.84" +USER = "root" +PASSWORD = os.environ["VPS_PASSWORD"] + +commands = [ + "cd /root/iaai-parser && docker compose ps", + "cd /root/iaai-parser && docker compose exec -T worker celery -A iaai_scraper.worker.celery_app inspect active reserved scheduled", + "docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c \"SELECT now() AS ts, (SELECT count(*) FROM iaai_cars) AS cars, (SELECT count(*) FROM iaai_sync_runs) AS runs, (SELECT status FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_status, (SELECT cars_upserted FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_upserted, (SELECT cars_failed FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_failed, (SELECT started_at FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_started;\"", + "docker logs --since 10m iaai-parser-worker-1 2>&1 | tail -n 160", +] + +client = paramiko.SSHClient() +client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) +client.connect( + HOST, + username=USER, + password=PASSWORD, + look_for_keys=False, + allow_agent=False, + timeout=30, + banner_timeout=30, + auth_timeout=30, +) +try: + for command in commands: + print(f"\n=== REMOTE_RUN {command} ===", flush=True) + stdin, stdout, stderr = client.exec_command(command, timeout=180) + exit_code = stdout.channel.recv_exit_status() + print(stdout.read().decode("utf-8", "replace"), end="") + print(stderr.read().decode("utf-8", "replace"), end="") + print(f"\n=== EXIT {exit_code} ===", flush=True) +finally: + client.close()