qananasikq/iaai-parser
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

checkpoint: pipeline + 27k/h baseline before sitemap discovery

Browse Source
This commit is contained in:
qananasikq
2026-04-18 16:34:09 +03:00
parent a8c5f8aa41
commit 5999c2e42d
30 changed files with 4993 additions and 61 deletions
Download Patch File Download Diff File Expand all files Collapse all files

24
_rollback_limits.sh Normal file
View File

@@ -0,0 +1,24 @@
#!/bin/bash
set -e
cd /root/iaai_scraper_project
echo "== 1. Verify .env =="
grep -E "MAX_PAGES|MAX_VEHICLES" .env
echo ""
echo "== 2. Restart worker + beat (не трогаем redis/postgres) =="
docker compose restart worker beat
sleep 5
echo ""
echo "== 3. Clear Redis lock =="
docker exec iaai-redis redis-cli del iaai:locks:sync_listing iaai:state:startup_sync_dispatched iaai:state:sync_listing_followup_pending
echo ""
echo "== 4. Dispatch fresh task =="
docker exec iaai-worker python -c "from iaai_scraper.worker.celery_app import celery_app; r=celery_app.send_task('iaai_scraper.worker.tasks.sync_listing_task', kwargs={'only_new': False}, queue='scraping'); print('dispatched:', r.id)"
echo ""
echo "== 5. Wait 45s, then check =="
sleep 45
echo ""
echo "== LOGS =="
docker logs iaai-worker --tail 250 2>&1 | grep -E "Segment|HTTP-mode page [0-9]+0:|HTTP-mode batch|HTTP-mode listing done|Sync run.*finished|upserted=" | tail -25
echo ""
echo "== DB =="
docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select 'total=' || count(*), 'last_2min=' || count(*) filter (where last_seen_at > now() - interval '2 minutes'), 'max=' || max(last_seen_at)::text from cars;"