qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add tools

Browse Source
This commit is contained in:
qananasikq
2026-04-27 20:47:44 +03:00
parent 9f215dc70a
commit 3299f8f372
4 changed files with 251 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
check_vps.py Normal file
View File

@@ -0,0 +1,37 @@
from __future__ import annotations
import os
import paramiko
HOST = "2.26.123.84"
USER = "root"
PASSWORD = os.environ["VPS_PASSWORD"]
cmds = [
"cd /root/iaai-parser && docker compose ps",
"docker logs --since 3m iaai-parser-worker-1 2>&1 | tail -n 120",
"docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c \"SELECT now() AS ts, count(*) AS cars FROM iaai_cars; SELECT count(*) AS runs FROM iaai_sync_runs;\"",
]
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(
HOST,
username=USER,
password=PASSWORD,
look_for_keys=False,
allow_agent=False,
timeout=30,
banner_timeout=30,
auth_timeout=30,
)
try:
for cmd in cmds:
print(f"REMOTE_RUN {cmd}", flush=True)
stdin, stdout, stderr = client.exec_command(cmd, timeout=120)
code = stdout.channel.recv_exit_status()
print(stdout.read().decode("utf-8", "replace"), end="")
print(stderr.read().decode("utf-8", "replace"), end="")
print(f"EXIT {code}", flush=True)
finally:
client.close()

41
clean_vps_db.py Normal file
View File

@@ -0,0 +1,41 @@
from __future__ import annotations
import os
import paramiko
HOST = "2.26.123.84"
USER = "root"
PASSWORD = os.environ["VPS_PASSWORD"]
commands = [
"cd /root/iaai-parser && docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c 'TRUNCATE TABLE iaai_images, iaai_cars, iaai_sync_runs RESTART IDENTITY CASCADE;'",
"docker exec -i iaai-redis redis-cli FLUSHALL",
"docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c 'SELECT count(*) AS cars FROM iaai_cars; SELECT count(*) AS runs FROM iaai_sync_runs;'",
"cd /root/iaai-parser && docker compose ps",
"docker logs --since 2m iaai-parser-worker-1 2>&1 | tail -n 100",
]
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(
HOST,
username=USER,
password=PASSWORD,
look_for_keys=False,
allow_agent=False,
timeout=30,
banner_timeout=30,
auth_timeout=30,
)
try:
for command in commands:
print(f"REMOTE_RUN {command}", flush=True)
stdin, stdout, stderr = client.exec_command(command, timeout=180)
exit_code = stdout.channel.recv_exit_status()
print(stdout.read().decode("utf-8", "replace"), end="")
print(stderr.read().decode("utf-8", "replace"), end="")
print(f"EXIT {exit_code}", flush=True)
if exit_code != 0:
raise SystemExit(exit_code)
finally:
client.close()

135
deploy_vps.py Normal file
View File

@@ -0,0 +1,135 @@
from __future__ import annotations
import os
import posixpath
import stat
import tarfile
import time
import socket
from pathlib import Path
import paramiko
HOST = "2.26.123.84"
USER = "root"
PASSWORD = os.environ["VPS_PASSWORD"]
LOCAL_ROOT = Path(__file__).resolve().parent
ARCHIVE = LOCAL_ROOT / "iaai_deploy.tar.gz"
REMOTE_DIR = "/root/iaai-parser"
REMOTE_ARCHIVE = "/root/iaai_deploy.tar.gz"
EXCLUDE_DIRS = {
".git",
".venv",
"__pycache__",
".pytest_cache",
".mypy_cache",
".ruff_cache",
"iaai_scraper.egg-info",
}
EXCLUDE_FILES = {"iaai_deploy.tar.gz", "deploy_vps.py"}
def _include(path: Path) -> bool:
rel = path.relative_to(LOCAL_ROOT)
parts = set(rel.parts)
if parts & EXCLUDE_DIRS:
return False
if path.name in EXCLUDE_FILES:
return False
if path.suffix in {".pyc", ".pyo"}:
return False
return True
def make_archive() -> None:
if ARCHIVE.exists():
ARCHIVE.unlink()
with tarfile.open(ARCHIVE, "w:gz") as tf:
for path in LOCAL_ROOT.rglob("*"):
if not _include(path):
continue
tf.add(path, arcname=str(path.relative_to(LOCAL_ROOT)))
print(f"ARCHIVE_READY {ARCHIVE} {ARCHIVE.stat().st_size} bytes", flush=True)
def connect() -> paramiko.SSHClient:
last_exc: BaseException | None = None
for attempt in range(1, 6):
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
try:
print(f"SSH_CONNECT attempt={attempt}", flush=True)
client.connect(
HOST,
username=USER,
password=PASSWORD,
look_for_keys=False,
allow_agent=False,
timeout=45,
banner_timeout=60,
auth_timeout=45,
)
return client
except (paramiko.SSHException, socket.timeout, OSError) as exc:
last_exc = exc
client.close()
print(f"SSH_CONNECT_RETRY attempt={attempt} error={type(exc).__name__}: {exc}", flush=True)
time.sleep(3 * attempt)
raise SystemExit(f"SSH_CONNECT_FAILED: {last_exc}")
def run(client: paramiko.SSHClient, cmd: str, timeout: int | None = None) -> None:
print(f"REMOTE_RUN {cmd}", flush=True)
stdin, stdout, stderr = client.exec_command(cmd, timeout=timeout)
exit_code = stdout.channel.recv_exit_status()
out = stdout.read().decode("utf-8", "replace")
err = stderr.read().decode("utf-8", "replace")
if out:
print(out, end="", flush=True)
if err:
print(err, end="", flush=True)
if exit_code != 0:
raise SystemExit(f"REMOTE_FAILED code={exit_code}: {cmd}")
def upload_file(sftp: paramiko.SFTPClient, local: Path, remote: str) -> None:
total = local.stat().st_size
last = 0.0
def cb(done: int, _total: int) -> None:
nonlocal last
now = time.time()
if now - last >= 2 or done == total:
print(f"UPLOAD {done}/{total}", flush=True)
last = now
sftp.put(str(local), remote, callback=cb)
def main() -> None:
make_archive()
client = connect()
try:
run(client, "echo VPS_OK && hostname && docker --version && docker compose version")
sftp = client.open_sftp()
try:
upload_file(sftp, ARCHIVE, REMOTE_ARCHIVE)
finally:
sftp.close()
run(
client,
f"mkdir -p {REMOTE_DIR} && cd {REMOTE_DIR} && docker compose down || true && "
f"find {REMOTE_DIR} -mindepth 1 -maxdepth 1 ! -name '.env' -exec rm -rf {{}} + && "
f"tar -xzf {REMOTE_ARCHIVE} -C {REMOTE_DIR} && rm -f {REMOTE_ARCHIVE}",
timeout=300,
)
run(client, f"cd {REMOTE_DIR} && docker compose up -d --build", timeout=1800)
run(client, f"cd {REMOTE_DIR} && docker compose ps", timeout=120)
run(client, "docker logs --since 2m iaai-parser-worker-1 2>&1 | tail -n 120 || docker compose -f /root/iaai-parser/docker-compose.yml logs --since 2m worker | tail -n 120", timeout=120)
finally:
client.close()
if __name__ == "__main__":
main()

38
status_vps.py Normal file
View File

@@ -0,0 +1,38 @@
from __future__ import annotations
import os
import paramiko
HOST = "2.26.123.84"
USER = "root"
PASSWORD = os.environ["VPS_PASSWORD"]
commands = [
"cd /root/iaai-parser && docker compose ps",
"cd /root/iaai-parser && docker compose exec -T worker celery -A iaai_scraper.worker.celery_app inspect active reserved scheduled",
"docker exec -i iaai-postgres psql -U iaai -d iaai_scraper -c \"SELECT now() AS ts, (SELECT count(*) FROM iaai_cars) AS cars, (SELECT count(*) FROM iaai_sync_runs) AS runs, (SELECT status FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_status, (SELECT cars_upserted FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_upserted, (SELECT cars_failed FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_failed, (SELECT started_at FROM iaai_sync_runs ORDER BY id DESC LIMIT 1) AS last_started;\"",
"docker logs --since 10m iaai-parser-worker-1 2>&1 | tail -n 160",
]
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
client.connect(
HOST,
username=USER,
password=PASSWORD,
look_for_keys=False,
allow_agent=False,
timeout=30,
banner_timeout=30,
auth_timeout=30,
)
try:
for command in commands:
print(f"\n=== REMOTE_RUN {command} ===", flush=True)
stdin, stdout, stderr = client.exec_command(command, timeout=180)
exit_code = stdout.channel.recv_exit_status()
print(stdout.read().decode("utf-8", "replace"), end="")
print(stderr.read().decode("utf-8", "replace"), end="")
print(f"\n=== EXIT {exit_code} ===", flush=True)
finally:
client.close()