checkpoint: pipeline + 27k/h baseline before sitemap discovery
This commit is contained in:
13
_kick.sh
Normal file
13
_kick.sh
Normal file
@@ -0,0 +1,13 @@
|
||||
#!/bin/bash
|
||||
set -e
|
||||
echo "== Cleaning stale Redis state =="
|
||||
docker exec iaai-redis redis-cli del iaai:locks:sync_listing iaai:state:startup_sync_dispatched iaai:state:sync_listing_followup_pending iaai:state:sync_listing_bootstrap_failure_streak
|
||||
echo "== Dispatching fresh sync_listing_task =="
|
||||
docker exec iaai-worker python -c "from iaai_scraper.worker.celery_app import celery_app; r=celery_app.send_task('iaai_scraper.worker.tasks.sync_listing_task', kwargs={'only_new': False}, queue='scraping'); print('dispatched:', r.id)"
|
||||
sleep 30
|
||||
echo "== Worker logs =="
|
||||
docker logs iaai-worker --tail 30 2>&1 | grep -E "Segment|sync_listing_task|progress|upserted|ERROR|clamp" | tail -20
|
||||
echo "== Lock TTL now =="
|
||||
docker exec iaai-redis redis-cli ttl iaai:locks:sync_listing
|
||||
echo "== DB recent =="
|
||||
docker exec iaai-postgres psql -U iaai -d iaai_scraper -t -c "select count(*) as total, count(*) filter (where last_seen_at > now() - interval '2 minutes') as recent from cars;"
|
||||
Reference in New Issue
Block a user