codeflash-internal/.github/workflows/end-to-end-tests.yaml

name: end-to-end-tests

on:
  pull_request:
  workflow_run:
    workflows: ["django-unit-tests"]
    types:
      - completed
    branches: [main]
  workflow_dispatch:

permissions:
  contents: read
  pull-requests: read

jobs:
  check-changes:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
    outputs:
      should_run: ${{ steps.filter.outputs.relevant == 'true' || github.event_name == 'workflow_dispatch' }}
      aiservice_changed: ${{ steps.filter.outputs.aiservice == 'true' }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Check which projects changed
        uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            relevant:
              - 'django/aiservice/**'
              - 'cli/**'
              - '.github/workflows/end-to-end-tests.yaml'
            aiservice:
              - 'django/aiservice/**'            

  no-changes-detected:
    name: No relevant changes detected
    needs: check-changes
    if: ${{ github.event_name == 'pull_request' && needs.check-changes.outputs.should_run != 'true' }}
    runs-on: ubuntu-latest
    steps:
      - name: Skip test
        run: echo "No relevant changes detected, skipping end-to-end tests"
      - name: Mark as success
        run: exit 0

  prek-check:
    name: Wait for prek checks
    needs: check-changes
    if: ${{ github.event_name == 'pull_request' && needs.check-changes.outputs.should_run == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
      checks: read
    steps:
      - uses: lewagon/wait-on-check-action@v1.3.4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          check-name: prek
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          wait-interval: 10

  unit-tests-check:
    name: Wait for unit tests
    needs: [check-changes, prek-check]
    if: ${{ github.event_name == 'pull_request' && needs.check-changes.outputs.aiservice_changed == 'true' }}
    runs-on: ubuntu-latest
    permissions:
      contents: read
      pull-requests: read
      checks: read
    steps:
      - uses: actions/checkout@v4
      - name: Wait for unit tests to pass
        uses: lewagon/wait-on-check-action@v1.3.4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
          check-name: 'unit-tests'
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          wait-interval: 10

  e2e-test:
    name: ${{ matrix.name }}
    needs: [check-changes, unit-tests-check]
    if: |
      always() &&
      (needs.unit-tests-check.result == 'success' ||
       (github.event_name == 'workflow_run' && github.event.workflow_run.conclusion == 'success') ||
       github.event_name == 'workflow_dispatch')      
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: bubblesort-pytest-no-git
            script: end_to_end_test_bubblesort_pytest.py
            expected_improvement: 70
            remove_git: true
          - name: bubblesort-unittest
            script: end_to_end_test_bubblesort_unittest.py
            expected_improvement: 40
          - name: coverage
            script: end_to_end_test_coverage.py
            expected_improvement: 10
            extra_deps: black
          - name: futurehouse
            script: end_to_end_test_futurehouse.py
            expected_improvement: 10
          - name: init-optimization
            script: end_to_end_test_init_optimization.py
            expected_improvement: 10
          - name: tracer-replay
            script: end_to_end_test_tracer_replay.py
            expected_improvement: 10
          - name: topological-sort
            script: end_to_end_test_topological_sort_worktree.py
            expected_improvement: 5
    env:
      CODEFLASH_AIS_SERVER: local
      AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
      AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
      OPENAI_API_VERSION: ${{ secrets.OPENAI_API_VERSION }}
      ANTHROPIC_FOUNDRY_API_KEY: ${{ secrets.ANTHROPIC_FOUNDRY_API_KEY }}
      ANTHROPIC_FOUNDRY_BASE_URL: ${{ secrets.ANTHROPIC_FOUNDRY_BASE_URL }}
      DATABASE_URL: ${{ secrets.DATABASE_URL }}
      POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
      CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
      COLUMNS: 110
      MAX_RETRIES: 3
      RETRY_DELAY: 5
      EXPECTED_IMPROVEMENT_PCT: ${{ matrix.expected_improvement }}
      CODEFLASH_END_TO_END: 1
    steps:
      - name: Check out codeflash-internal repo
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Python 3.12 for AI Server
        uses: astral-sh/setup-uv@v7
        with:
          python-version: "3.12"

      - name: Install dependencies for Django server
        run: |
          cd ./django/aiservice
          uv sync          

      - name: Start Django server
        run: |
          cd ./django/aiservice
          uv run uvicorn aiservice.asgi:application --host localhost --port 8000 >> server.log 2>&1 &          

      - name: Remove .git
        if: ${{ matrix.remove_git == true }}
        run: |
          if [ -d ".git" ]; then
            echo ".git directory exists!"
            sudo rm -rf .git
            if [ -d ".git" ]; then
              echo ".git directory still exists after removal attempt!"
              exit 1
            else
              echo ".git directory successfully removed."
            fi
          else
            echo ".git directory does not exist. Nothing to remove."
            exit 1
          fi          

      - name: Clone codeflash repo
        run: |
          cd ..
          git clone https://github.com/codeflash-ai/codeflash.git          
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Set up Python 3.11 for CLI
        uses: actions/setup-python@v5
        with:
          python-version: "3.11.6"

      - name: Install dependencies (CLI)
        run: |
          cd ../codeflash
          uv sync
          uv add sqlalchemy ${{ matrix.extra_deps }}          

      - name: Run Codeflash to optimize code
        id: optimize_code
        run: |
          cd ../codeflash
          uv run python tests/scripts/${{ matrix.script }}          

      - name: Display AI Server logs
        if: always()
        run: |
          cd ./django/aiservice
          cat server.log          

  e2e-status:
    name: E2E Tests Status
    runs-on: ubuntu-latest
    needs: [check-changes, no-changes-detected, prek-check, unit-tests-check, e2e-test]
    if: always()
    steps:
      - name: Check all job statuses
        run: |
          if [[ "${{ needs.e2e-test.result }}" == "success" ]] || \
             [[ "${{ needs.e2e-test.result }}" == "skipped" ]] || \
             [[ "${{ needs.no-changes-detected.result }}" == "success" ]]; then
            echo "✓ End-to-end tests workflow completed successfully"
            exit 0
          else
            echo "✗ End-to-end tests workflow failed"
            echo "no-changes-detected: ${{ needs.no-changes-detected.result }}"
            echo "prek-check: ${{ needs.prek-check.result }}"
            echo "unit-tests-check: ${{ needs.unit-tests-check.result }}"
            echo "e2e-test: ${{ needs.e2e-test.result }}"
            exit 1
          fi