checkpoint: pipeline + 27k/h baseline before sitemap discovery

_reduce50.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+set -e
+cd /root/iaai_scraper_project
+echo "== update .env =="
+sed -i 's/^IAAI_MAX_PAGES_PER_RUN=.*/IAAI_MAX_PAGES_PER_RUN=50/' .env
+grep MAX_PAGES .env
+echo ""
+echo "== restart worker (wait for finish) =="
+docker compose stop worker
+docker compose up -d worker
+sleep 10
+echo ""
+echo "== clear lock =="
+docker exec iaai-redis redis-cli del iaai:locks:sync_listing iaai:state:startup_sync_dispatched iaai:state:sync_listing_followup_pending
+echo ""
+echo "== dispatch =="
+docker exec iaai-worker python -c "from iaai_scraper.worker.celery_app import celery_app; r=celery_app.send_task('iaai_scraper.worker.tasks.sync_listing_task', kwargs={'only_new': False}, queue='scraping'); print('dispatched:', r.id)"
+echo ""
+echo "== wait 90s =="
+sleep 90
+echo ""
+echo "== DB =="
+docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select 'total=' || count(*), 'last_5min=' || count(*) filter (where last_seen_at > now() - interval '5 minutes'), 'max=' || max(last_seen_at)::text from cars;"
+echo ""
+echo "== BATCH + UPSERT EVENTS =="
+docker logs iaai-worker --tail 5000 2>&1 | grep -E "HTTP-mode batch|HTTP-mode listing done|Segment.*done|upserted" | tail -20
+echo ""
+echo "== TAIL =="
+docker logs iaai-worker --tail 8