tensorzero/.github/workflows/batch-test.yml at main · pycoder49/tensorzero · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
name: Batch tests

env:
  ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
  AWS_REGION: "us-east-1"
  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
  AZURE_OPENAI_API_BASE: ${{secrets.AZURE_OPENAI_API_BASE }}
  AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
  AZURE_OPENAI_DEPLOYMENT_ID: ${{secrets.AZURE_OPENAI_DEPLOYMENT_ID }}
  DEEPSEEK_API_KEY: ${{ secrets.DEEPSEEK_API_KEY }}
  FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
  FIREWORKS_ACCOUNT_ID: ${{ secrets.FIREWORKS_ACCOUNT_ID }}
  FORCE_COLOR: 1
  GCP_VERTEX_CREDENTIALS_PATH: ${{ github.workspace }}/gcp_jwt_key.json
  GOOGLE_APPLICATION_CREDENTIALS: ${{ github.workspace }}/gcp_jwt_key.json
  GOOGLE_AI_STUDIO_API_KEY: ${{ secrets.GOOGLE_AI_STUDIO_API_KEY }}
  GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
  HYPERBOLIC_API_KEY: ${{secrets.HYPERBOLIC_API_KEY}}
  MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
  OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
  OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}
  SGLANG_API_KEY: ${{ secrets.SGLANG_API_KEY }}
  TGI_API_KEY: ${{ secrets.TGI_API_KEY }}
  TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
  VLLM_API_KEY: ${{ secrets.VLLM_API_KEY }}
  VLLM_API_BASE: ${{ secrets.VLLM_API_BASE }}
  VLLM_MODEL_NAME: "microsoft/Phi-3.5-mini-instruct"
  XAI_API_KEY: ${{ secrets.XAI_API_KEY }}

on:
  workflow_dispatch:
  workflow_call:

jobs:
  batch-tests:
    runs-on: ubuntu-latest
    if: github.repository == 'tensorzero/tensorzero'

    timeout-minutes: 60

    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683

      - name: Cleanup disk space
        run: ./ci/free-disk-space.sh

      - name: Install Rust toolchain
        run: |
          for attempt in 1 2 3; do
            if rustup toolchain install stable && rustup default stable; then
              break
            fi
            if [ $attempt -eq 3 ]; then
              echo "Failed to install Rust toolchain after 3 attempts"
              exit 1
            fi
            sleep $((10 * attempt))
          done
        shell: bash

      - name: Login to DockerHub
        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: Wake up ClickHouse cloud (for batch tests)
        run: |
          curl "${{ secrets.CLICKHOUSE_CLOUD_URL }}" --retry 4 --retry-all-errors --data-binary 'SHOW DATABASES'

      - name: Install cargo-nextest
        uses: taiki-e/install-action@d12e869b89167df346dd0ff65da342d1fb1202fb
        with:
          tool: cargo-nextest

      - name: Write GCP JWT key to file
        env:
          GCP_JWT_KEY: ${{ secrets.GCP_JWT_KEY }}
        run: echo "$GCP_JWT_KEY" > $GITHUB_WORKSPACE/gcp_jwt_key.json

      - name: Set up TENSORZERO_CLICKHOUSE_URL for batch tests
        run: |
          echo "TENSORZERO_CLICKHOUSE_URL=${{ secrets.CLICKHOUSE_CLOUD_URL }}" >> $GITHUB_ENV

      - name: Launch the gateway for batch tests
        run: |
          curl "${{ secrets.CLICKHOUSE_CLOUD_URL }}" --retry 4 --retry-all-errors --data-binary 'SHOW DATABASES'
          cargo run-e2e > batch_logs.txt 2>&1 &
          echo "GATEWAY_PID=$!" >> $GITHUB_ENV
          while ! curl -s -f http://localhost:3000/health >/dev/null 2>&1; do
            echo "Waiting for gateway to be healthy..."
            sleep 1
          done

      - name: Run batch tests
        run: |
          cargo test-batch --no-fail-fast

      - name: Print batch logs
        if: always()
        run: cat batch_logs.txt