Consolidate CI into single workflow with shared actions (#2614)

Replace 7 separate CI workflow files with a unified ci.yaml that uses
shared workflows from codeflash-ai/github-workflows:

- determine-changes: reusable workflow for path-based change detection
- prek-lint: reusable workflow for pre-commit checks
- ci-python-uv: reusable workflow for Python typecheck
- required-checks-gate: composite action for gate job

All downstream jobs use fromJSON(needs.determine-changes.outputs.flags)
for conditional execution. A single required-checks-passed gate job
replaces per-workflow required checks.

Private repos need explicit permissions on reusable workflow calls
(contents:write for prek) since they don't inherit permissive defaults.

.github/workflows/ci.yaml

@@ -0,0 +1,521 @@
+name: CI
+
+on:
+  push:
+    branches: [main]
+    paths:
+      - 'django/aiservice/**'
+      - 'js/cf-api/**'
+      - 'js/cf-webapp/**'
+      - 'js/VSC-Extension/**'
+      - 'cli/**'
+      - '.github/workflows/ci.yaml'
+  pull_request:
+  workflow_dispatch:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref_name }}
+  cancel-in-progress: true
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Change detection — decides which downstream jobs actually run.
+  # On push/workflow_dispatch every flag is true so all jobs execute.
+  # On pull_request we diff against the merge base.
+  # ---------------------------------------------------------------------------
+  determine-changes:
+    uses: codeflash-ai/github-workflows/.github/workflows/determine-changes.yml@main
+    with:
+      path-filters: |
+        {
+          "aiservice": ["django/aiservice/"],
+          "cf_api": ["js/cf-api/"],
+          "cf_webapp": ["js/cf-webapp/"],
+          "vscode_extension": ["js/VSC-Extension/"],
+          "e2e": ["django/aiservice/", "cli/", ".github/workflows/ci.yaml"]
+        }
+
+  # ---------------------------------------------------------------------------
+  # AI Service — typecheck (shared workflow)
+  # ---------------------------------------------------------------------------
+  aiservice-typecheck:
+    needs: determine-changes
+    if: fromJSON(needs.determine-changes.outputs.flags).aiservice == 'true'
+    uses: codeflash-ai/github-workflows/.github/workflows/ci-python-uv.yml@main
+    with:
+      working-directory: "django/aiservice"
+      sync-command: "uv sync"
+      typecheck-command: "uv run mypy --non-interactive --config-file pyproject.toml @mypy_allowlist.txt"
+
+  # ---------------------------------------------------------------------------
+  # AI Service — pytest (needs secrets as env vars)
+  # ---------------------------------------------------------------------------
+  aiservice-test:
+    needs: determine-changes
+    if: fromJSON(needs.determine-changes.outputs.flags).aiservice == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+    defaults:
+      run:
+        working-directory: django/aiservice
+    env:
+      SECRET_KEY: ${{ secrets.SECRET_KEY }}
+      DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+      AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+      OPENAI_API_VERSION: ${{ secrets.OPENAI_API_VERSION }}
+      ANTHROPIC_FOUNDRY_API_KEY: ${{ secrets.ANTHROPIC_FOUNDRY_API_KEY }}
+      ANTHROPIC_FOUNDRY_BASE_URL: ${{ secrets.ANTHROPIC_FOUNDRY_BASE_URL }}
+    steps:
+      - uses: actions/checkout@v6
+      - uses: astral-sh/setup-uv@v8.0.0
+        with:
+          python-version: "3.12"
+          enable-cache: true
+      - run: uv sync
+      - name: Test
+        run: uv run pytest
+
+  # ---------------------------------------------------------------------------
+  # Prek — lint (pull_request only)
+  # Caller must grant contents:write because prek-lint.yml needs it for
+  # auto-fix commits. Private repos don't inherit permissive defaults.
+  # ---------------------------------------------------------------------------
+  prek:
+    if: github.event_name == 'pull_request'
+    uses: codeflash-ai/github-workflows/.github/workflows/prek-lint.yml@main
+    permissions:
+      contents: write
+    with:
+      continue-on-error: true
+
+  # ---------------------------------------------------------------------------
+  # CF-API — JS tests
+  # ---------------------------------------------------------------------------
+  cf-api-test:
+    needs: determine-changes
+    if: fromJSON(needs.determine-changes.outputs.flags).cf_api == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+    env:
+      NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: '20'
+          registry-url: https://npm.pkg.github.com
+          scope: '@codeflash-ai'
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
+      - name: Install dependencies
+        working-directory: js
+        run: pnpm install --frozen-lockfile
+
+      - name: Build common package
+        working-directory: js
+        run: pnpm --filter @codeflash-ai/common build
+
+      - name: Run tests
+        working-directory: js/cf-api
+        run: NODE_OPTIONS=--experimental-vm-modules pnpm jest --ci --config jest.config.cjs
+
+      - name: Build
+        working-directory: js/cf-api
+        run: pnpm build
+
+  # ---------------------------------------------------------------------------
+  # cf-webapp — quality gates (typecheck + tests + build)
+  # ---------------------------------------------------------------------------
+  cf-webapp-quality-gates:
+    needs: determine-changes
+    if: fromJSON(needs.determine-changes.outputs.flags).cf_webapp == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+      pull-requests: write
+    env:
+      NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - uses: actions/checkout@v6
+
+      - uses: actions/setup-node@v6
+        with:
+          node-version: "20"
+          registry-url: https://npm.pkg.github.com
+          scope: "@codeflash-ai"
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
+      - name: Restore WASM artifacts cache
+        uses: actions/cache@v5
+        with:
+          path: |
+            js/cf-webapp/public/web-tree-sitter.wasm
+            js/cf-webapp/public/tree-sitter-python.wasm
+            js/cf-webapp/public/.tree-sitter-python-version
+          key: wasm-${{ runner.os }}-${{ hashFiles('js/pnpm-lock.yaml') }}
+
+      - name: Install dependencies
+        working-directory: js
+        run: pnpm install --frozen-lockfile
+
+      - name: Build common package
+        working-directory: js
+        run: pnpm --filter @codeflash-ai/common build
+
+      - name: Generate Prisma client for cf-webapp
+        working-directory: js/cf-webapp
+        run: pnpm prisma generate
+
+      - name: Restore Next.js build cache
+        uses: actions/cache@v5
+        with:
+          path: js/cf-webapp/.next/cache
+          key: nextjs-${{ runner.os }}-${{ hashFiles('js/pnpm-lock.yaml') }}-${{ hashFiles('js/cf-webapp/src/**') }}
+          restore-keys: |
+            nextjs-${{ runner.os }}-${{ hashFiles('js/pnpm-lock.yaml') }}-
+            nextjs-${{ runner.os }}-
+
+      - name: Type-check
+        id: typecheck
+        working-directory: js/cf-webapp
+        run: pnpm tsc --noEmit
+        continue-on-error: true
+
+      - name: Tests
+        id: tests
+        working-directory: js/cf-webapp
+        run: pnpm vitest run --reporter=verbose 2>&1 | tee test-output.txt
+        continue-on-error: true
+
+      - name: Build
+        id: build
+        working-directory: js/cf-webapp
+        run: pnpm next build 2>&1 | tee build-output.txt
+        continue-on-error: true
+
+      - name: Extract results
+        id: results
+        working-directory: js/cf-webapp
+        run: |
+          if [ "${{ steps.typecheck.outcome }}" = "success" ]; then
+            echo "typecheck_status=Pass" >> "$GITHUB_OUTPUT"
+          else
+            echo "typecheck_status=Fail" >> "$GITHUB_OUTPUT"
+          fi
+
+          if [ "${{ steps.tests.outcome }}" = "success" ]; then
+            TESTS_SUMMARY=$(grep -E "Tests\s+[0-9]+" test-output.txt | tail -1 || echo "passed")
+            echo "tests_status=Pass ${TESTS_SUMMARY}" >> "$GITHUB_OUTPUT"
+          else
+            echo "tests_status=Tests failed" >> "$GITHUB_OUTPUT"
+          fi
+
+          if [ "${{ steps.build.outcome }}" = "success" ]; then
+            echo "build_status=Success" >> "$GITHUB_OUTPUT"
+          else
+            echo "build_status=Fail" >> "$GITHUB_OUTPUT"
+          fi
+
+          ROUTES=$(sed -n '/Route.*Size.*First Load/,/^$/p' build-output.txt | head -30 || echo "No route data")
+          {
+            echo "routes<<ROUTES_EOF"
+            echo "$ROUTES"
+            echo "ROUTES_EOF"
+          } >> "$GITHUB_OUTPUT"
+
+      - name: Post PR comment
+        if: github.event_name == 'pull_request'
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh pr comment ${{ github.event.pull_request.number }} \
+            --repo ${{ github.repository }} \
+            --body "$(cat <<'COMMENT_EOF'
+          ## cf-webapp Quality Report
+
+          | Check | Result |
+          |-------|--------|
+          | Type-check | ${{ steps.results.outputs.typecheck_status }} |
+          | Tests | ${{ steps.results.outputs.tests_status }} |
+          | Build | ${{ steps.results.outputs.build_status }} |
+
+          <details>
+          <summary>Route Sizes</summary>
+
+          ```
+          ${{ steps.results.outputs.routes }}
+          ```
+          </details>
+          COMMENT_EOF
+          )"
+
+      - name: Fail if any check failed
+        if: steps.typecheck.outcome == 'failure' || steps.tests.outcome == 'failure' || steps.build.outcome == 'failure'
+        run: exit 1
+
+  # ---------------------------------------------------------------------------
+  # Next.js build check
+  # ---------------------------------------------------------------------------
+  nextjs-build:
+    needs: determine-changes
+    if: fromJSON(needs.determine-changes.outputs.flags).cf_webapp == 'true'
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: read
+    env:
+      NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v6
+        with:
+          node-version: '20'
+          registry-url: https://npm.pkg.github.com
+          scope: '@codeflash-ai'
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10
+
+      - name: Restore WASM artifacts cache
+        uses: actions/cache@v5
+        with:
+          path: |
+            js/cf-webapp/public/web-tree-sitter.wasm
+            js/cf-webapp/public/tree-sitter-python.wasm
+            js/cf-webapp/public/.tree-sitter-python-version
+          key: wasm-${{ runner.os }}-${{ hashFiles('js/pnpm-lock.yaml') }}
+
+      - name: Install dependencies
+        working-directory: js
+        run: pnpm install --frozen-lockfile
+
+      - name: Restore Next.js build cache
+        uses: actions/cache@v5
+        with:
+          path: js/cf-webapp/.next/cache
+          key: nextjs-${{ runner.os }}-${{ hashFiles('js/pnpm-lock.yaml') }}-${{ hashFiles('js/cf-webapp/src/**') }}
+          restore-keys: |
+            nextjs-${{ runner.os }}-${{ hashFiles('js/pnpm-lock.yaml') }}-
+            nextjs-${{ runner.os }}-
+
+      - name: Build Next.js app
+        working-directory: js
+        run: pnpm --filter cf-webapp build
+
+  # ---------------------------------------------------------------------------
+  # VSCode Extension — version check + build
+  # ---------------------------------------------------------------------------
+  vscode-extension-build:
+    needs: determine-changes
+    if: fromJSON(needs.determine-changes.outputs.flags).vscode_extension == 'true'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+
+      - name: Extract MIN_CODEFLASH_VERSION from constants file
+        id: extract-version
+        run: |
+          FILE="js/VSC-Extension/src/constants/cf_min_version.ts"
+          VERSION=$(grep -oP 'MIN_CODEFLASH_VERSION\s*=\s*"\K[^"]+' "$FILE")
+
+          if [ -z "$VERSION" ]; then
+            echo "Could not find MIN_CODEFLASH_VERSION in $FILE"
+            exit 1
+          fi
+
+          echo "Found MIN_CODEFLASH_VERSION=$VERSION"
+          echo "version=$VERSION" >> $GITHUB_OUTPUT
+
+      - name: Check version exists on PyPI
+        run: |
+          VERSION="${{ steps.extract-version.outputs.version }}"
+          if pip index versions codeflash | grep -q "Available versions: .*$VERSION"; then
+            echo "Version $VERSION exists on PyPI."
+          else
+            echo "Version $VERSION not found on PyPI"
+            exit 1
+          fi
+
+      - name: Use Node.js v20
+        uses: actions/setup-node@v6
+        with:
+          node-version: 20
+          cache: "npm"
+          cache-dependency-path: js/VSC-Extension/package-lock.json
+
+      - name: Install dependencies
+        working-directory: js/VSC-Extension
+        run: npm ci
+
+      - name: Package VSCode Extension
+        working-directory: js/VSC-Extension
+        run: npm run vsce
+
+      - name: Upload VSIX artifact
+        uses: actions/upload-artifact@v7
+        with:
+          name: vscode-extension
+          path: js/VSC-Extension/*.vsix
+
+  # ---------------------------------------------------------------------------
+  # E2E tests — matrix of 7 tests with local Django server
+  # ---------------------------------------------------------------------------
+  e2e-test:
+    needs: determine-changes
+    if: fromJSON(needs.determine-changes.outputs.flags).e2e == 'true'
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - name: bubblesort-pytest-no-git
+            script: end_to_end_test_bubblesort_pytest.py
+            expected_improvement: 70
+            remove_git: true
+          - name: bubblesort-unittest
+            script: end_to_end_test_bubblesort_unittest.py
+            expected_improvement: 40
+          - name: coverage
+            script: end_to_end_test_coverage.py
+            expected_improvement: 10
+            extra_deps: black
+          - name: futurehouse
+            script: end_to_end_test_futurehouse.py
+            expected_improvement: 10
+          - name: init-optimization
+            script: end_to_end_test_init_optimization.py
+            expected_improvement: 10
+          - name: tracer-replay
+            script: end_to_end_test_tracer_replay.py
+            expected_improvement: 10
+          - name: topological-sort
+            script: end_to_end_test_topological_sort_worktree.py
+            expected_improvement: 5
+    env:
+      CODEFLASH_AIS_SERVER: local
+      AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
+      AZURE_OPENAI_ENDPOINT: ${{ secrets.AZURE_OPENAI_ENDPOINT }}
+      OPENAI_API_VERSION: ${{ secrets.OPENAI_API_VERSION }}
+      ANTHROPIC_FOUNDRY_API_KEY: ${{ secrets.ANTHROPIC_FOUNDRY_API_KEY }}
+      ANTHROPIC_FOUNDRY_BASE_URL: ${{ secrets.ANTHROPIC_FOUNDRY_BASE_URL }}
+      DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
+      CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
+      COLUMNS: 110
+      MAX_RETRIES: 3
+      RETRY_DELAY: 5
+      EXPECTED_IMPROVEMENT_PCT: ${{ matrix.expected_improvement }}
+      CODEFLASH_END_TO_END: 1
+    steps:
+      - name: Check out codeflash-internal repo
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python 3.12 for AI Server
+        uses: astral-sh/setup-uv@v7
+        with:
+          python-version: "3.12"
+
+      - name: Install dependencies for Django server
+        run: |
+          cd ./django/aiservice
+          uv sync
+
+      - name: Start Django server
+        run: |
+          cd ./django/aiservice
+          uv run uvicorn aiservice.asgi:application --host localhost --port 8000 >> server.log 2>&1 &
+
+      - name: Remove .git
+        if: ${{ matrix.remove_git == true }}
+        run: |
+          if [ -d ".git" ]; then
+            echo ".git directory exists!"
+            sudo rm -rf .git
+            if [ -d ".git" ]; then
+              echo ".git directory still exists after removal attempt!"
+              exit 1
+            else
+              echo ".git directory successfully removed."
+            fi
+          else
+            echo ".git directory does not exist. Nothing to remove."
+            exit 1
+          fi
+
+      - name: Clone codeflash repo
+        run: |
+          cd ..
+          git clone https://github.com/codeflash-ai/codeflash.git
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Set up Python 3.11 for CLI
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11.6"
+
+      - name: Install dependencies (CLI)
+        run: |
+          cd ../codeflash
+          uv sync
+          uv add sqlalchemy ${{ matrix.extra_deps }}
+
+      - name: Run Codeflash to optimize code
+        id: optimize_code
+        run: |
+          cd ../codeflash
+          uv run python tests/scripts/${{ matrix.script }}
+
+      - name: Display AI Server logs
+        if: always()
+        run: |
+          cd ./django/aiservice
+          cat server.log
+
+  # ---------------------------------------------------------------------------
+  # Gate job — the ONLY required check in the GitHub ruleset.
+  # Accepts "success" and "skipped" (job skipped by change detection).
+  # Rejects "failure" and "cancelled".
+  # ---------------------------------------------------------------------------
+  required-checks-passed:
+    name: required checks passed
+    if: always()
+    needs:
+      - aiservice-typecheck
+      - aiservice-test
+      - prek
+      - cf-api-test
+      - cf-webapp-quality-gates
+      - nextjs-build
+      - vscode-extension-build
+      - e2e-test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: codeflash-ai/github-workflows/.github/actions/required-checks-gate@main
+        with:
+          needs-json: ${{ toJSON(needs) }}