b7bcd0fe2e
The change detection for JS E2E tests was missing the test fixture directory, so PRs that only modify JS test data (like this one) were skipped. Java already had its equivalent path included.
514 lines
18 KiB
YAML
514 lines
18 KiB
YAML
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches: [main]
|
|
paths:
|
|
- 'codeflash/**'
|
|
- 'codeflash-benchmark/**'
|
|
- 'codeflash-java-runtime/**'
|
|
- 'tests/**'
|
|
- 'packages/**'
|
|
- 'pyproject.toml'
|
|
- 'uv.lock'
|
|
- 'mypy_allowlist.txt'
|
|
- '.github/workflows/ci.yaml'
|
|
- '.github/actions/**'
|
|
pull_request:
|
|
workflow_dispatch:
|
|
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ github.ref_name }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
# ---------------------------------------------------------------------------
|
|
# Change detection — decides which downstream jobs actually run.
|
|
# On push/workflow_dispatch every flag is true so all jobs execute.
|
|
# On pull_request we diff against the merge base (same approach as astral-sh/ruff).
|
|
# ---------------------------------------------------------------------------
|
|
determine-changes:
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: read
|
|
outputs:
|
|
unit_tests: ${{ github.event_name != 'pull_request' || steps.check.outputs.unit_tests == 'true' }}
|
|
type_check: ${{ github.event_name != 'pull_request' || steps.check.outputs.type_check == 'true' }}
|
|
e2e: ${{ github.event_name != 'pull_request' || steps.check.outputs.e2e == 'true' }}
|
|
e2e_js: ${{ github.event_name != 'pull_request' || steps.check.outputs.e2e_js == 'true' }}
|
|
e2e_java: ${{ github.event_name != 'pull_request' || steps.check.outputs.e2e_java == 'true' }}
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
if: github.event_name == 'pull_request'
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Determine merge base
|
|
if: github.event_name == 'pull_request'
|
|
id: merge_base
|
|
run: |
|
|
sha=$(git merge-base HEAD "origin/${{ github.event.pull_request.base.ref }}")
|
|
echo "sha=${sha}" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Check changed paths
|
|
if: github.event_name == 'pull_request'
|
|
id: check
|
|
run: |
|
|
check_paths() {
|
|
local name="$1"; shift
|
|
if ! git diff --quiet "$MERGE_BASE...HEAD" -- "$@" 2>/dev/null; then
|
|
echo "${name}=true" >> "$GITHUB_OUTPUT"
|
|
else
|
|
echo "${name}=false" >> "$GITHUB_OUTPUT"
|
|
fi
|
|
}
|
|
|
|
# Unit tests: code + test infra + packages + build config
|
|
check_paths unit_tests \
|
|
'codeflash/' 'codeflash-benchmark/' \
|
|
'tests/' 'packages/' 'pyproject.toml' 'uv.lock'
|
|
|
|
# Type checking: code + build config + mypy config
|
|
check_paths type_check \
|
|
'codeflash/' 'pyproject.toml' 'uv.lock' 'mypy_allowlist.txt'
|
|
|
|
# E2E tests: Python pipeline + tests + build config (excludes java/ and javascript/)
|
|
check_paths e2e \
|
|
'codeflash/*.py' \
|
|
'codeflash/api/' 'codeflash/benchmarking/' 'codeflash/cli_cmds/' \
|
|
'codeflash/code_utils/' 'codeflash/discovery/' 'codeflash/github/' \
|
|
'codeflash/languages/python/' 'codeflash/languages/*.py' \
|
|
'codeflash/lsp/' 'codeflash/models/' 'codeflash/optimization/' \
|
|
'codeflash/picklepatch/' 'codeflash/result/' 'codeflash/setup/' \
|
|
'codeflash/telemetry/' 'codeflash/tracing/' 'codeflash/verification/' \
|
|
'tests/' 'pyproject.toml' 'uv.lock'
|
|
|
|
# JS E2E tests: JS language support + shared pipeline + packages + test fixtures
|
|
check_paths e2e_js \
|
|
'codeflash/languages/javascript/' 'codeflash/languages/base.py' \
|
|
'codeflash/languages/registry.py' 'codeflash/optimization/' \
|
|
'codeflash/verification/' 'packages/' \
|
|
'code_to_optimize/js/' 'tests/scripts/end_to_end_test_js*'
|
|
|
|
# Java E2E tests: Java language support + shared pipeline + runtime
|
|
check_paths e2e_java \
|
|
'codeflash/languages/java/' 'codeflash/languages/base.py' \
|
|
'codeflash/languages/registry.py' 'codeflash/optimization/' \
|
|
'codeflash/verification/' 'codeflash-java-runtime/' \
|
|
'code_to_optimize/java/' 'tests/scripts/end_to_end_test_java*'
|
|
env:
|
|
MERGE_BASE: ${{ steps.merge_base.outputs.sha }}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Unit tests — 6 Linux + 1 Windows matrix
|
|
# ---------------------------------------------------------------------------
|
|
unit-tests:
|
|
needs: determine-changes
|
|
if: needs.determine-changes.outputs.unit_tests == 'true'
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- os: ubuntu-latest
|
|
python-version: "3.9"
|
|
- os: ubuntu-latest
|
|
python-version: "3.10"
|
|
- os: ubuntu-latest
|
|
python-version: "3.11"
|
|
- os: ubuntu-latest
|
|
python-version: "3.12"
|
|
- os: ubuntu-latest
|
|
python-version: "3.13"
|
|
- os: ubuntu-latest
|
|
python-version: "3.14"
|
|
- os: windows-latest
|
|
python-version: "3.13"
|
|
runs-on: ${{ matrix.os }}
|
|
env:
|
|
PYTHONIOENCODING: utf-8
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 1
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v8.0.0
|
|
with:
|
|
python-version: ${{ matrix.python-version }}
|
|
enable-cache: true
|
|
|
|
- name: Install dependencies
|
|
shell: bash
|
|
run: |
|
|
if [[ "${{ matrix.python-version }}" == "3.9" || "${{ matrix.python-version }}" == "3.13" ]]; then
|
|
uv sync --group tests
|
|
else
|
|
uv sync
|
|
fi
|
|
|
|
- name: Unit tests
|
|
run: uv run pytest tests/
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Mypy type checking
|
|
# ---------------------------------------------------------------------------
|
|
type-check:
|
|
needs: determine-changes
|
|
if: needs.determine-changes.outputs.type_check == 'true'
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 1
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v8.0.0
|
|
with:
|
|
enable-cache: true
|
|
|
|
- name: Install dependencies
|
|
run: uv sync
|
|
|
|
- name: Run mypy
|
|
run: uv run mypy --non-interactive --config-file pyproject.toml @mypy_allowlist.txt
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Lint (prek) — pull_request only
|
|
# ---------------------------------------------------------------------------
|
|
prek:
|
|
needs: determine-changes
|
|
if: >-
|
|
github.event_name == 'pull_request'
|
|
&& (needs.determine-changes.outputs.e2e == 'true'
|
|
|| needs.determine-changes.outputs.e2e_js == 'true')
|
|
runs-on: ubuntu-latest
|
|
permissions:
|
|
contents: write
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
ref: ${{ github.head_ref }}
|
|
fetch-depth: 0
|
|
- uses: astral-sh/setup-uv@v8.0.0
|
|
with:
|
|
enable-cache: true
|
|
|
|
- name: Auto-fix formatting
|
|
run: |
|
|
uv run ruff check --fix . || true
|
|
uv run ruff format .
|
|
|
|
- name: Commit and push fixes
|
|
run: |
|
|
git diff --quiet && exit 0
|
|
git config user.name "github-actions[bot]"
|
|
git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
|
|
git add -u
|
|
git commit -m "style: auto-format with ruff"
|
|
git push
|
|
|
|
- uses: j178/prek-action@v2
|
|
with:
|
|
extra-args: '--from-ref origin/${{ github.base_ref }} --to-ref HEAD'
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# E2E tests — only on pull_request and workflow_dispatch (not push to main)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
# --- Standard Python E2Es (9 tests) ---
|
|
e2e-python:
|
|
needs: determine-changes
|
|
if: >-
|
|
needs.determine-changes.outputs.e2e == 'true'
|
|
&& github.event_name != 'push'
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- name: tracer-replay
|
|
script: end_to_end_test_tracer_replay.py
|
|
expected_improvement: 10
|
|
- name: bubble-sort-pytest-nogit
|
|
script: end_to_end_test_bubblesort_pytest.py
|
|
expected_improvement: 70
|
|
remove_git: true
|
|
- name: bubble-sort-unittest
|
|
script: end_to_end_test_bubblesort_unittest.py
|
|
expected_improvement: 40
|
|
- name: futurehouse-structure
|
|
script: end_to_end_test_futurehouse.py
|
|
expected_improvement: 5
|
|
- name: topological-sort
|
|
script: end_to_end_test_topological_sort_worktree.py
|
|
expected_improvement: 5
|
|
- name: async-optimization
|
|
script: end_to_end_test_async.py
|
|
expected_improvement: 10
|
|
- name: benchmark-bubble-sort
|
|
script: end_to_end_test_benchmark_sort.py
|
|
expected_improvement: 5
|
|
- name: coverage-e2e
|
|
script: end_to_end_test_coverage.py
|
|
extra_deps: black
|
|
- name: init-optimization
|
|
script: end_to_end_test_init_optimization.py
|
|
expected_improvement: 10
|
|
environment: ${{ (github.event_name == 'workflow_dispatch' || (contains(toJSON(github.event.pull_request.files.*.filename), '.github/workflows/') && github.event.pull_request.user.login != 'misrasaurabh1' && github.event.pull_request.user.login != 'KRRT7')) && 'external-trusted-contributors' || '' }}
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CODEFLASH_AIS_SERVER: prod
|
|
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
|
|
CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
|
|
MAX_RETRIES: 3
|
|
RETRY_DELAY: 5
|
|
CODEFLASH_END_TO_END: 1
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.ref || '' }}
|
|
repository: ${{ github.event.pull_request.head.repo.full_name || '' }}
|
|
fetch-depth: 0
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Validate PR
|
|
if: github.event_name == 'pull_request'
|
|
uses: ./.github/actions/validate-pr
|
|
with:
|
|
base_sha: ${{ github.event.pull_request.base.sha }}
|
|
head_sha: ${{ github.event.pull_request.head.sha }}
|
|
author: ${{ github.event.pull_request.user.login }}
|
|
pr_state: ${{ github.event.pull_request.state }}
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v8.0.0
|
|
with:
|
|
python-version: 3.11.6
|
|
enable-cache: true
|
|
|
|
- name: Install dependencies
|
|
run: uv sync
|
|
|
|
- name: Install extra dependencies
|
|
if: matrix.extra_deps
|
|
run: uv add ${{ matrix.extra_deps }}
|
|
|
|
- name: Set test configuration
|
|
if: matrix.expected_improvement
|
|
run: |
|
|
echo "COLUMNS=110" >> "$GITHUB_ENV"
|
|
echo "EXPECTED_IMPROVEMENT_PCT=${{ matrix.expected_improvement }}" >> "$GITHUB_ENV"
|
|
|
|
- name: Remove .git
|
|
if: matrix.remove_git
|
|
run: |
|
|
if [ -d ".git" ]; then
|
|
echo ".git directory exists!"
|
|
sudo rm -rf .git
|
|
if [ -d ".git" ]; then
|
|
echo ".git directory still exists after removal attempt!"
|
|
exit 1
|
|
else
|
|
echo ".git directory successfully removed."
|
|
fi
|
|
else
|
|
echo ".git directory does not exist. Nothing to remove."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run E2E test
|
|
run: uv run python tests/scripts/${{ matrix.script }}
|
|
|
|
# --- JS E2Es (3 tests, need Node.js + packages/) ---
|
|
e2e-js:
|
|
needs: determine-changes
|
|
if: >-
|
|
needs.determine-changes.outputs.e2e_js == 'true'
|
|
&& github.event_name != 'push'
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- name: js-cjs-function
|
|
script: end_to_end_test_js_cjs_function.py
|
|
js_project_dir: code_to_optimize/js/code_to_optimize_js
|
|
expected_improvement: 50
|
|
- name: js-esm-async
|
|
script: end_to_end_test_js_esm_async.py
|
|
js_project_dir: code_to_optimize/js/code_to_optimize_js_esm
|
|
expected_improvement: 10
|
|
- name: js-ts-class
|
|
script: end_to_end_test_js_ts_class.py
|
|
js_project_dir: code_to_optimize/js/code_to_optimize_ts
|
|
expected_improvement: 30
|
|
environment: ${{ (github.event_name == 'workflow_dispatch' || (contains(toJSON(github.event.pull_request.files.*.filename), '.github/workflows/') && github.event.pull_request.user.login != 'misrasaurabh1' && github.event.pull_request.user.login != 'KRRT7')) && 'external-trusted-contributors' || '' }}
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CODEFLASH_AIS_SERVER: prod
|
|
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
|
|
CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
|
|
COLUMNS: 110
|
|
MAX_RETRIES: 3
|
|
RETRY_DELAY: 5
|
|
EXPECTED_IMPROVEMENT_PCT: ${{ matrix.expected_improvement }}
|
|
CODEFLASH_END_TO_END: 1
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.ref || '' }}
|
|
repository: ${{ github.event.pull_request.head.repo.full_name || '' }}
|
|
fetch-depth: 0
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Validate PR
|
|
if: github.event_name == 'pull_request'
|
|
uses: ./.github/actions/validate-pr
|
|
with:
|
|
base_sha: ${{ github.event.pull_request.base.sha }}
|
|
head_sha: ${{ github.event.pull_request.head.sha }}
|
|
author: ${{ github.event.pull_request.user.login }}
|
|
pr_state: ${{ github.event.pull_request.state }}
|
|
|
|
- name: Set up Node.js
|
|
uses: actions/setup-node@v6
|
|
with:
|
|
node-version: '20'
|
|
cache: 'npm'
|
|
cache-dependency-path: |
|
|
packages/codeflash/package-lock.json
|
|
code_to_optimize/js/*/package-lock.json
|
|
|
|
- name: Install codeflash npm package dependencies
|
|
run: |
|
|
cd packages/codeflash
|
|
npm install
|
|
|
|
- name: Install JS test project dependencies
|
|
run: |
|
|
cd ${{ matrix.js_project_dir }}
|
|
npm install
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v8.0.0
|
|
with:
|
|
python-version: 3.11.6
|
|
enable-cache: true
|
|
|
|
- name: Install dependencies
|
|
run: uv sync
|
|
|
|
- name: Run E2E test
|
|
run: uv run python tests/scripts/${{ matrix.script }}
|
|
|
|
# --- Java E2Es (3 tests, need JDK + Maven) ---
|
|
e2e-java:
|
|
needs: determine-changes
|
|
if: >-
|
|
needs.determine-changes.outputs.e2e_java == 'true'
|
|
&& github.event_name != 'push'
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- name: java-fibonacci-nogit
|
|
script: end_to_end_test_java_fibonacci.py
|
|
expected_improvement: 70
|
|
remove_git: true
|
|
- name: java-tracer
|
|
script: end_to_end_test_java_tracer.py
|
|
expected_improvement: 10
|
|
- name: java-void-optimization-nogit
|
|
script: end_to_end_test_java_void_optimization.py
|
|
expected_improvement: 70
|
|
remove_git: true
|
|
environment: ${{ (github.event_name == 'workflow_dispatch' || (contains(toJSON(github.event.pull_request.files.*.filename), '.github/workflows/') && github.event.pull_request.user.login != 'misrasaurabh1' && github.event.pull_request.user.login != 'KRRT7')) && 'external-trusted-contributors' || '' }}
|
|
runs-on: ubuntu-latest
|
|
env:
|
|
CODEFLASH_AIS_SERVER: prod
|
|
POSTHOG_API_KEY: ${{ secrets.POSTHOG_API_KEY }}
|
|
CODEFLASH_API_KEY: ${{ secrets.CODEFLASH_API_KEY }}
|
|
COLUMNS: 110
|
|
MAX_RETRIES: 3
|
|
RETRY_DELAY: 5
|
|
EXPECTED_IMPROVEMENT_PCT: ${{ matrix.expected_improvement }}
|
|
CODEFLASH_END_TO_END: 1
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
ref: ${{ github.event.pull_request.head.ref || '' }}
|
|
repository: ${{ github.event.pull_request.head.repo.full_name || '' }}
|
|
fetch-depth: 0
|
|
token: ${{ secrets.GITHUB_TOKEN }}
|
|
|
|
- name: Validate PR
|
|
if: github.event_name == 'pull_request'
|
|
uses: ./.github/actions/validate-pr
|
|
with:
|
|
base_sha: ${{ github.event.pull_request.base.sha }}
|
|
head_sha: ${{ github.event.pull_request.head.sha }}
|
|
author: ${{ github.event.pull_request.user.login }}
|
|
pr_state: ${{ github.event.pull_request.state }}
|
|
|
|
- name: Set up JDK 11
|
|
uses: actions/setup-java@v5
|
|
with:
|
|
java-version: '11'
|
|
distribution: 'temurin'
|
|
cache: maven
|
|
|
|
- name: Install uv
|
|
uses: astral-sh/setup-uv@v8.0.0
|
|
with:
|
|
python-version: 3.11.6
|
|
enable-cache: true
|
|
|
|
- name: Install dependencies
|
|
run: uv sync
|
|
|
|
- name: Build and install codeflash-runtime JAR
|
|
run: |
|
|
cd codeflash-java-runtime
|
|
mvn install -q -DskipTests
|
|
|
|
- name: Remove .git
|
|
if: matrix.remove_git
|
|
run: |
|
|
if [ -d ".git" ]; then
|
|
sudo rm -rf .git
|
|
echo ".git directory removed."
|
|
else
|
|
echo ".git directory does not exist."
|
|
exit 1
|
|
fi
|
|
|
|
- name: Run E2E test
|
|
run: uv run python tests/scripts/${{ matrix.script }}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Gate job — the ONLY required check in the GitHub ruleset.
|
|
# Accepts "success" and "skipped" (job skipped by change detection).
|
|
# Rejects "failure" and "cancelled".
|
|
# ---------------------------------------------------------------------------
|
|
required-checks-passed:
|
|
name: required checks passed
|
|
if: always()
|
|
needs:
|
|
- unit-tests
|
|
- type-check
|
|
- prek
|
|
- e2e-python
|
|
- e2e-js
|
|
- e2e-java
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Verify all required jobs passed
|
|
run: |
|
|
failing=$(echo "$NEEDS_JSON" | jq -r 'to_entries[] | select(.value.result != "success" and .value.result != "skipped") | "\(.key): \(.value.result)"')
|
|
if [ -n "$failing" ]; then
|
|
echo "Required jobs failed or were cancelled:"
|
|
echo "$failing"
|
|
exit 1
|
|
fi
|
|
env:
|
|
NEEDS_JSON: ${{ toJSON(needs) }}
|