Fast IAAI implementation

docker-compose.yml

@@ -0,0 +1,105 @@
+x-app-env: &app-env
+  POSTGRES_HOST: ${POSTGRES_HOST:-postgres}
+  POSTGRES_PORT: ${POSTGRES_PORT:-5432}
+  POSTGRES_USER: ${POSTGRES_USER:-postgres}
+  POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+  POSTGRES_DB: ${POSTGRES_DB:-iaai}
+  DATABASE_URL: ${IAAI_DATABASE_URL:-postgresql+psycopg://${POSTGRES_USER:-postgres}:${POSTGRES_PASSWORD:-postgres}@${POSTGRES_HOST:-postgres}:${POSTGRES_PORT:-5432}/${POSTGRES_DB:-iaai}}
+  CELERY_BROKER_URL: redis://redis:6379/0
+  CELERY_RESULT_BACKEND: ${CELERY_RESULT_BACKEND:-redis://redis:6379/1}
+  IAAI_BASE_URL: ${IAAI_BASE_URL:-https://www.iaai.com}
+  IAAI_LISTING_START_URL: ${IAAI_LISTING_START_URL:-}
+  IAAI_SESSION_COOKIES: ${IAAI_SESSION_COOKIES:-}
+  IAAI_STORAGE_STATE_PATH: /app/iaai_storage_state.json
+  IAAI_LOGIN: ${IAAI_LOGIN:-}
+  IAAI_PASSWORD: ${IAAI_PASSWORD:-}
+  IAAI_HTTP_TIMEOUT: ${IAAI_HTTP_TIMEOUT:-30}
+  IAAI_HTTP_RETRIES: ${IAAI_HTTP_RETRIES:-2}
+  IAAI_HTTP_RETRY_BACKOFF_MS: ${IAAI_HTTP_RETRY_BACKOFF_MS:-700}
+  IAAI_FETCH_CONCURRENCY: ${IAAI_FETCH_CONCURRENCY:-8}
+  IAAI_DB_COMMIT_BATCH_SIZE: ${IAAI_DB_COMMIT_BATCH_SIZE:-20}
+  IAAI_SESSION_KEEPALIVE_ENABLED: ${IAAI_SESSION_KEEPALIVE_ENABLED:-true}
+  IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES: ${IAAI_SESSION_KEEPALIVE_INTERVAL_MINUTES:-15}
+  IAAI_SCHEMA_BOOTSTRAP_ENABLED: ${IAAI_SCHEMA_BOOTSTRAP_ENABLED:-true}
+  SYNC_RUNTIME_CONFIG_FILE: /app/sync_runtime_config.json
+
+services:
+  postgres:
+    image: postgres:latest
+    environment:
+      POSTGRES_DB: ${POSTGRES_DB:-iaai}
+      POSTGRES_USER: ${POSTGRES_USER:-postgres}
+      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres}
+      PGDATA: /var/lib/postgresql/data/pgdata
+    restart: unless-stopped
+    ports:
+      - "127.0.0.1:5432:5432"
+    volumes:
+      - postgres_data:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-postgres} -d ${POSTGRES_DB:-iaai}"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  redis:
+    image: redis:7-alpine
+    restart: unless-stopped
+    ports:
+      - "6379"
+    healthcheck:
+      test: ["CMD", "redis-cli", "ping"]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+
+  migrate:
+    build: .
+    environment: *app-env
+    command: ["alembic", "upgrade", "head"]
+    volumes:
+      - ./sync_runtime_config.json:/app/sync_runtime_config.json:ro
+      - ./iaai_storage_state.json:/app/iaai_storage_state.json
+    depends_on:
+      postgres:
+        condition: service_healthy
+
+  worker:
+    build: .
+    environment: *app-env
+    command:
+      [
+        "celery",
+        "-A",
+        "iaai_sync_service.celery_app:app",
+        "worker",
+        "--queues=iaai_sync",
+        "--concurrency=1",
+        "--loglevel=INFO"
+      ]
+    volumes:
+      - ./sync_runtime_config.json:/app/sync_runtime_config.json:ro
+      - ./iaai_storage_state.json:/app/iaai_storage_state.json
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+  beat:
+    build: .
+    environment: *app-env
+    command: ["celery", "-A", "iaai_sync_service.celery_app:app", "beat", "--loglevel=INFO"]
+    volumes:
+      - ./sync_runtime_config.json:/app/sync_runtime_config.json:ro
+      - ./iaai_storage_state.json:/app/iaai_storage_state.json
+    depends_on:
+      migrate:
+        condition: service_completed_successfully
+      redis:
+        condition: service_healthy
+    restart: unless-stopped
+
+volumes:
+  postgres_data: