chore: cleanup ad-hoc debug scripts + sitemap test downloads

.gitignore

@@ -20,3 +20,11 @@ artifacts/
 celerybeat-schedule*
 tokens_data/
 tokens.json
+
+# Ad-hoc debug scripts and one-off downloads
+_*.sh
+_*.tar.gz
+burst_*.ps1
+burst_*.txt
+sitemap*.xml
+vd_*.html

_check.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-echo "== LOCK =="
-docker exec iaai-redis redis-cli get iaai:locks:sync_listing
-docker exec iaai-redis redis-cli ttl iaai:locks:sync_listing
-echo "== CHECKPOINT =="
-docker exec iaai-redis redis-cli get iaai:state:sync_listing_checkpoint
-docker exec iaai-redis redis-cli get iaai:state:sync_full_scan_done
-echo "== DB =="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select count(*) as total, count(*) filter (where last_seen_at > now() - interval '3 minutes') as recent from cars;"
-echo "== WORKER TAIL =="
-docker logs iaai-worker --tail 25 2>&1 | tail -25

_check2.sh

@@ -1,26 +0,0 @@
-#!/bin/bash
-echo "=========================================="
-echo "== ТЕКУЩАЯ АКТИВНОСТЬ SYNC (последние логи) =="
-echo "=========================================="
-docker logs iaai-worker --tail 30 2>&1 | grep -E "Segment|HTTP-mode page|upserted|finished|ERROR" | tail -20
-echo ""
-echo "=========================================="
-echo "== DB: всего машин и апсерты за 5 минут =="
-echo "=========================================="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select 'total=' || count(*), 'last_5min=' || count(*) filter (where last_seen_at > now() - interval '5 minutes'), 'last_1min=' || count(*) filter (where last_seen_at > now() - interval '1 minute') from cars;"
-echo ""
-echo "=========================================="
-echo "== Макс timestamp (когда была последняя запись) =="
-echo "=========================================="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select 'max_last_seen=' || max(last_seen_at), 'now=' || now() from cars;"
-echo ""
-echo "=========================================="
-echo "== Топ марок в БД =="
-echo "=========================================="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select make, count(*) from cars group by make order by count(*) desc limit 10;"
-echo ""
-echo "=========================================="
-echo "== Очередь Celery =="
-echo "=========================================="
-docker exec iaai-redis redis-cli llen scraping
-echo "lock TTL: $(docker exec iaai-redis redis-cli ttl iaai:locks:sync_listing)s"

_check3.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-echo "== ERRORS/EXCEPTIONS за последние 5 минут worker =="
-docker logs iaai-worker --since 5m 2>&1 | grep -iE "error|traceback|exception|failed|warning" | grep -v "Segment.*done" | tail -20
-echo ""
-echo "== Последние события (batch/upsert/save) =="
-docker logs iaai-worker --tail 500 2>&1 | grep -iE "batch|upsert|save|Processing|finished|collected" | tail -15
-echo ""
-echo "== DB: cars schema =="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -c "\d cars" 2>&1 | head -40
-echo ""
-echo "== DB: real counts =="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select 'total=' || count(*), 'last_5min=' || count(*) filter (where last_seen_at > now() - interval '5 minutes') from cars;"
-echo ""
-echo "== Текущая активность sync (tail) =="
-docker logs iaai-worker --tail 5 2>&1

_check4.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-echo "== DB: счётчики =="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select 'total=' || count(*), 'last_10min=' || count(*) filter (where last_seen_at > now() - interval '10 minutes'), 'last_2min=' || count(*) filter (where last_seen_at > now() - interval '2 minutes'), 'max_last_seen=' || max(last_seen_at)::text from cars;"
-echo ""
-echo "== Топ brand =="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select brand, count(*) from cars group by brand order by count(*) desc limit 10;"
-echo ""
-echo "== Текущая фаза sync (tail) =="
-docker logs iaai-worker --tail 5 2>&1
-echo ""
-echo "== Последние upserted/batch =="
-docker logs iaai-worker --tail 500 2>&1 | grep -iE "upserted|Processing batch|Sync run.*finished|Segment.*done" | tail -10
-echo ""
-echo "== ERRORS свежие =="
-docker logs iaai-worker --since 10m 2>&1 | grep -iE "ERROR|Traceback|Exception" | tail -10

_db_now.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-cd /root/iaai_scraper_project
-
-echo "== DB now =="
-docker compose exec -T postgres psql -U iaai -d iaai_scraper -c "
-SELECT
-  count(*) AS total,
-  count(*) FILTER (WHERE last_seen_at > now() - interval '1 minute')  AS last_1min,
-  count(*) FILTER (WHERE last_seen_at > now() - interval '5 minutes') AS last_5min,
-  count(*) FILTER (WHERE last_seen_at > now() - interval '15 minutes') AS last_15min,
-  max(last_seen_at) AS max_ts,
-  now()             AS now_ts
-FROM cars;"
-
-echo
-echo "== last 10 upsert / segment-done events =="
-docker compose logs --tail=1000 worker 2>&1 | grep -E 'upserted=|Segment .* done|HTTP-mode batch' | tail -n 15
-
-echo
-echo "== last 5 page-progress =="
-docker compose logs --tail=200 worker 2>&1 | grep -E 'HTTP-mode page' | tail -n 5
-
-echo
-echo "== recent 500s (if any) =="
-docker compose logs --tail=500 worker 2>&1 | grep -E 'status=500|status=403' | tail -n 5 || true

_db_now2.sh

@@ -1,19 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-cd /root/iaai_scraper_project
-
-echo "== DB =="
-docker compose exec -T postgres psql -U iaai -d iaai_scraper -c "
-SELECT
-  count(*) AS total,
-  count(*) FILTER (WHERE last_seen_at > now() - interval '1 minute')  AS l1,
-  count(*) FILTER (WHERE last_seen_at > now() - interval '5 minutes') AS l5,
-  count(*) FILTER (WHERE created_at   > now() - interval '15 minutes') AS created_15m,
-  max(last_seen_at) AS max_seen,
-  max(created_at)   AS max_created,
-  now() AS now_ts
-FROM cars;"
-
-echo
-echo "== last 50 lines of worker log (everything) =="
-docker compose logs --tail=50 worker 2>&1 | tail -n 50

_diag_vps.sh

@@ -1,42 +0,0 @@
-#!/bin/bash
-echo "=========================================="
-echo "== 1. PROXY/HTTP CONFIG в .env =="
-echo "=========================================="
-cd /root/iaai_scraper_project
-grep -iE "PROXY|HTTP_MODE|USER_AGENT|IAAI_BASE|IAAI_.*URL|BRIGHT" .env | grep -v "^#"
-echo ""
-echo "=========================================="
-echo "== 2. ВНЕШНИЙ IP VPS =="
-echo "=========================================="
-curl -s --max-time 10 https://api.ipify.org; echo ""
-echo ""
-echo "=========================================="
-echo "== 3. ПРЯМОЙ ЗАПРОС IAAI (без прокси) =="
-echo "=========================================="
-curl -s -o /dev/null -w "HTTP=%{http_code} time=%{time_total}s size=%{size_download}\n" --max-time 20 "https://www.iaai.com/"
-curl -s -o /tmp/iaai_root.html -w "HTTP=%{http_code} size=%{size_download}\n" --max-time 20 -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" "https://www.iaai.com/"
-echo "--- первые 400 байт ответа ---"
-head -c 400 /tmp/iaai_root.html; echo ""
-echo "--- признак Incapsula/bot-check ---"
-grep -oiE "incapsula|_Incapsula_|captcha|access denied|bot|cloudflare" /tmp/iaai_root.html | sort -u
-echo ""
-echo "=========================================="
-echo "== 4. POST /Search (как это делает парсер) =="
-echo "=========================================="
-curl -s -o /tmp/iaai_search.html -w "HTTP=%{http_code} size=%{size_download}\n" --max-time 20 \
-  -H "Content-Type: application/x-www-form-urlencoded" \
-  -H "Accept: application/json, text/html, */*" \
-  -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" \
-  -X POST "https://www.iaai.com/Search" \
-  --data "Make=TOYOTA&YearMin=1900&YearMax=2012&Page=1"
-head -c 400 /tmp/iaai_search.html; echo ""
-echo ""
-echo "=========================================="
-echo "== 5. PROXY env внутри worker-контейнера =="
-echo "=========================================="
-docker exec iaai-worker env | grep -iE "PROXY|IAAI|HTTP_" | sort
-echo ""
-echo "=========================================="
-echo "== 6. Последние ERROR-логи worker =="
-echo "=========================================="
-docker logs iaai-worker --tail 200 2>&1 | grep -iE "proxy|bright|incapsula|403|500|429|blocked" | tail -15

_dispatch_check.sh

@@ -1,26 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-cd /root/iaai_scraper_project
-
-echo "== dispatch =="
-TID=$(docker compose exec -T worker python -c "from iaai_scraper.worker.tasks import sync_listing_task; print(sync_listing_task.delay().id)")
-echo "dispatched: $TID"
-
-echo "== wait 120s =="
-sleep 120
-
-echo
-echo "== DB =="
-docker compose exec -T postgres psql -U iaai -d iaai_scraper -c "SELECT count(*) AS total, count(*) FILTER (WHERE last_seen_at > now() - interval '5 minutes') AS last_5min, max(last_seen_at) AS max FROM cars;"
-
-echo
-echo "== upsert / batch / segment-done events (last 40) =="
-docker compose logs --tail=800 worker 2>&1 | grep -E 'HTTP-mode batch|upserted=|HTTP-mode listing done|Segment .* done|HTTP fast-path using proxy|Sync run .* finished' | tail -n 40
-
-echo
-echo "== page-progress tail =="
-docker compose logs --tail=400 worker 2>&1 | grep -E 'HTTP-mode page' | tail -n 15
-
-echo
-echo "== errors if any =="
-docker compose logs --tail=400 worker 2>&1 | grep -Ei 'error|exception|403|500|incapsula|pardon' | tail -n 15 || true

_enable_proxy.sh

@@ -1,55 +0,0 @@
-#!/usr/bin/env bash
-set -euo pipefail
-cd /root/iaai_scraper_project
-
-echo "== patch .env =="
-# удалить старые SOCKS5_*/IAAI_PROXY_* строки если есть
-sed -i '/^SOCKS5_PROXY_/d; /^IAAI_PROXY_/d' .env
-
-cat >> .env <<'EOF'
-
-# --- SOCKS5 US residential proxy (BulletProof) ---
-SOCKS5_PROXY_HOST=residential.bpproxy.at
-SOCKS5_PROXY_PORT=1002
-SOCKS5_PROXY_USER=bpuser-S7o3VlC0
-SOCKS5_PROXY_PASS=YKmRzvjvnnvyQdEUQ8hM_country-US_session-q3TGNZZO_lifetime-30
-EOF
-
-grep -E '^(SOCKS5_|IAAI_PROXY_)' .env
-
-echo "== restart worker =="
-docker compose stop worker
-docker compose up -d worker
-
-echo "== wait 8s for worker + bridge =="
-sleep 8
-
-echo "== verify proxy bridge + external IP =="
-docker compose exec -T worker sh -c 'curl -s --max-time 20 --proxy http://127.0.0.1:8899 https://api.ipify.org' || echo "proxy check FAILED"
-echo
-
-echo "== direct check (no proxy) =="
-docker compose exec -T worker sh -c 'curl -s --max-time 10 https://api.ipify.org' || true
-echo
-
-echo "== clear locks/state =="
-docker compose exec -T redis redis-cli DEL iaai:locks:sync_listing iaai:state:startup_sync_dispatched iaai:state:sync_listing_followup_pending
-
-echo "== dispatch fresh task =="
-TID=$(docker compose exec -T worker python -c "from iaai_scraper.worker.tasks import sync_listing; print(sync_listing.delay().id)")
-echo "dispatched: $TID"
-
-echo "== wait 90s =="
-sleep 90
-
-echo
-echo "== DB =="
-docker compose exec -T postgres psql -U iaai -d iaai_scraper -c "SELECT count(*) AS total, count(*) FILTER (WHERE last_seen_at > now() - interval '5 minutes') AS last_5min, max(last_seen_at) AS max FROM cars;"
-
-echo
-echo "== last 30 log lines (batch / upserted / HTTP-mode page) =="
-docker compose logs --tail=500 worker 2>&1 | grep -E 'HTTP-mode batch|upserted=|HTTP-mode listing done|Segment .* done|HTTP fast-path using proxy|proxy_bridge' | tail -n 30
-
-echo
-echo "== sample of recent pagination lines =="
-docker compose logs --tail=300 worker 2>&1 | grep -E 'HTTP-mode page' | tail -n 10

_kick.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-set -e
-echo "== Cleaning stale Redis state =="
-docker exec iaai-redis redis-cli del iaai:locks:sync_listing iaai:state:startup_sync_dispatched iaai:state:sync_listing_followup_pending iaai:state:sync_listing_bootstrap_failure_streak
-echo "== Dispatching fresh sync_listing_task =="
-docker exec iaai-worker python -c "from iaai_scraper.worker.celery_app import celery_app; r=celery_app.send_task('iaai_scraper.worker.tasks.sync_listing_task', kwargs={'only_new': False}, queue='scraping'); print('dispatched:', r.id)"
-sleep 30
-echo "== Worker logs =="
-docker logs iaai-worker --tail 30 2>&1 | grep -E "Segment|sync_listing_task|progress|upserted|ERROR|clamp" | tail -20
-echo "== Lock TTL now =="
-docker exec iaai-redis redis-cli ttl iaai:locks:sync_listing
-echo "== DB recent =="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select count(*) as total, count(*) filter (where last_seen_at > now() - interval '2 minutes') as recent from cars;"

_reduce50.sh

@@ -1,29 +0,0 @@
-#!/bin/bash
-set -e
-cd /root/iaai_scraper_project
-echo "== update .env =="
-sed -i 's/^IAAI_MAX_PAGES_PER_RUN=.*/IAAI_MAX_PAGES_PER_RUN=50/' .env
-grep MAX_PAGES .env
-echo ""
-echo "== restart worker (wait for finish) =="
-docker compose stop worker
-docker compose up -d worker
-sleep 10
-echo ""
-echo "== clear lock =="
-docker exec iaai-redis redis-cli del iaai:locks:sync_listing iaai:state:startup_sync_dispatched iaai:state:sync_listing_followup_pending
-echo ""
-echo "== dispatch =="
-docker exec iaai-worker python -c "from iaai_scraper.worker.celery_app import celery_app; r=celery_app.send_task('iaai_scraper.worker.tasks.sync_listing_task', kwargs={'only_new': False}, queue='scraping'); print('dispatched:', r.id)"
-echo ""
-echo "== wait 90s =="
-sleep 90
-echo ""
-echo "== DB =="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select 'total=' || count(*), 'last_5min=' || count(*) filter (where last_seen_at > now() - interval '5 minutes'), 'max=' || max(last_seen_at)::text from cars;"
-echo ""
-echo "== BATCH + UPSERT EVENTS =="
-docker logs iaai-worker --tail 5000 2>&1 | grep -E "HTTP-mode batch|HTTP-mode listing done|Segment.*done|upserted" | tail -20
-echo ""
-echo "== TAIL =="
-docker logs iaai-worker --tail 8

_rollback_limits.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-set -e
-cd /root/iaai_scraper_project
-echo "== 1. Verify .env =="
-grep -E "MAX_PAGES|MAX_VEHICLES" .env
-echo ""
-echo "== 2. Restart worker + beat (не трогаем redis/postgres) =="
-docker compose restart worker beat
-sleep 5
-echo ""
-echo "== 3. Clear Redis lock =="
-docker exec iaai-redis redis-cli del iaai:locks:sync_listing iaai:state:startup_sync_dispatched iaai:state:sync_listing_followup_pending
-echo ""
-echo "== 4. Dispatch fresh task =="
-docker exec iaai-worker python -c "from iaai_scraper.worker.celery_app import celery_app; r=celery_app.send_task('iaai_scraper.worker.tasks.sync_listing_task', kwargs={'only_new': False}, queue='scraping'); print('dispatched:', r.id)"
-echo ""
-echo "== 5. Wait 45s, then check =="
-sleep 45
-echo ""
-echo "== LOGS =="
-docker logs iaai-worker --tail 250 2>&1 | grep -E "Segment|HTTP-mode page [0-9]+0:|HTTP-mode batch|HTTP-mode listing done|Sync run.*finished|upserted=" | tail -25
-echo ""
-echo "== DB =="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select 'total=' || count(*), 'last_2min=' || count(*) filter (where last_seen_at > now() - interval '2 minutes'), 'max=' || max(last_seen_at)::text from cars;"

_wait_check.sh

@@ -1,10 +0,0 @@
-#!/bin/bash
-sleep 60
-echo "== DB =="
-docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select 'total=' || count(*), 'last_3min=' || count(*) filter (where last_seen_at > now() - interval '3 minutes'), 'max=' || max(last_seen_at)::text from cars;"
-echo ""
-echo "== BATCH + SEGMENT DONE EVENTS =="
-docker logs iaai-worker --tail 3000 2>&1 | grep -E "HTTP-mode batch|HTTP-mode listing done: [1-9]|Segment.*done.*upserted=[1-9]|Sync run.*finished: [1-9]" | tail -15
-echo ""
-echo "== последние 10 строк =="
-docker logs iaai-worker --tail 10

burst_test.ps1

@@ -1,12 +0,0 @@
-$ua="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/135.0.0.0 Safari/537.36"
-$sw=[Diagnostics.Stopwatch]::StartNew()
-$results=@()
-1..15 | ForEach-Object {
-  $id = 45184893 + $_
-  $code = curl.exe -s -o NUL -A $ua -w "%{http_code}" "https://www.iaai.com/VehicleDetail/$id~US"
-  $results += [pscustomobject]@{Idx=$_; Id=$id; Code=$code}
-}
-$sw.Stop()
-$results | Format-Table -AutoSize | Out-String | Set-Content burst_results.txt
-("elapsed_total_s={0}" -f [math]::Round($sw.Elapsed.TotalSeconds,2)) | Add-Content burst_results.txt
-("avg_per_page_s={0}" -f [math]::Round($sw.Elapsed.TotalSeconds/15,2)) | Add-Content burst_results.txt

sitemap_test.xml

@@ -1 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?><!--List of sitemaps created--><sitemapindex xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://www.sitemaps.org/schemas/sitemap/0.9/sitemap.xsd" xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap1.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap2.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemap3.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemapbranches1.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemapauctions1.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemapMake1.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemapMakeModel1.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemapMakeModelYear1.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemapVehicleTypeMake1.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap><sitemap><loc>https://www.iaai.com/Xj9rDOVMEi0hc38S/sitemapVehicleType.xml</loc><lastmod>2026-04-18T04:03:01-05:00</lastmod></sitemap></sitemapindex>