qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

checkpoint: pipeline + 27k/h baseline before sitemap discovery

Browse Source
This commit is contained in:
qananasikq
2026-04-18 16:34:09 +03:00
parent a8c5f8aa41
commit 5999c2e42d
30 changed files with 4993 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

42
_diag_vps.sh Normal file
View File

@@ -0,0 +1,42 @@
#!/bin/bash
echo "=========================================="
echo "== 1. PROXY/HTTP CONFIG в .env =="
echo "=========================================="
cd /root/iaai_scraper_project
grep -iE "PROXY|HTTP_MODE|USER_AGENT|IAAI_BASE|IAAI_.*URL|BRIGHT" .env | grep -v "^#"
echo ""
echo "=========================================="
echo "== 2. ВНЕШНИЙ IP VPS =="
echo "=========================================="
curl -s --max-time 10 https://api.ipify.org; echo ""
echo ""
echo "=========================================="
echo "== 3. ПРЯМОЙ ЗАПРОС IAAI (без прокси) =="
echo "=========================================="
curl -s -o /dev/null -w "HTTP=%{http_code} time=%{time_total}s size=%{size_download}\n" --max-time 20 "https://www.iaai.com/"
curl -s -o /tmp/iaai_root.html -w "HTTP=%{http_code} size=%{size_download}\n" --max-time 20 -A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" "https://www.iaai.com/"
echo "--- первые 400 байт ответа ---"
head -c 400 /tmp/iaai_root.html; echo ""
echo "--- признак Incapsula/bot-check ---"
grep -oiE "incapsula|_Incapsula_|captcha|access denied|bot|cloudflare" /tmp/iaai_root.html | sort -u
echo ""
echo "=========================================="
echo "== 4. POST /Search (как это делает парсер) =="
echo "=========================================="
curl -s -o /tmp/iaai_search.html -w "HTTP=%{http_code} size=%{size_download}\n" --max-time 20 \
-H "Content-Type: application/x-www-form-urlencoded" \
-H "Accept: application/json, text/html, */*" \
-A "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" \
-X POST "https://www.iaai.com/Search" \
--data "Make=TOYOTA&YearMin=1900&YearMax=2012&Page=1"
head -c 400 /tmp/iaai_search.html; echo ""
echo ""
echo "=========================================="
echo "== 5. PROXY env внутри worker-контейнера =="
echo "=========================================="
docker exec iaai-worker env | grep -iE "PROXY|IAAI|HTTP_" | sort
echo ""
echo "=========================================="
echo "== 6. Последние ERROR-логи worker =="
echo "=========================================="
docker logs iaai-worker --tail 200 2>&1 | grep -iE "proxy|bright|incapsula|403|500|429|blocked" | tail -15