codeflash-agent/evals/check-regression.sh

#!/bin/bash
set -euo pipefail

# Eval regression checker for codeflash-agent plugin
#
# Runs a subset of evals, scores them, and compares to checked-in baselines.
# Exits 1 if any score drops below the minimum threshold.
#
# Usage:
#   ./check-regression.sh                      # run all baseline evals
#   ./check-regression.sh ranking              # run specific template(s)
#   ./check-regression.sh --score-only <dir>   # score existing results, skip running

EVAL_DIR="$(cd "$(dirname "$0")" && pwd)"
BASELINE_FILE="$EVAL_DIR/baseline-scores.json"

die() { echo "ERROR: $*" >&2; exit 1; }

[ -f "$BASELINE_FILE" ] || die "Baseline file not found: $BASELINE_FILE"

# --- Parse args ---

SCORE_ONLY=""
RESULTS_DIRS=()
TEMPLATES=()

while [[ $# -gt 0 ]]; do
    case "$1" in
        --score-only)
            SCORE_ONLY=1
            shift
            while [[ $# -gt 0 && ! "$1" =~ ^-- ]]; do
                RESULTS_DIRS+=("$1")
                shift
            done
            ;;
        *)
            TEMPLATES+=("$1")
            shift
            ;;
    esac
done

# If no templates specified, use all from baseline file
if [[ ${#TEMPLATES[@]} -eq 0 && -z "$SCORE_ONLY" ]]; then
    mapfile -t TEMPLATES < <(jq -r '.evals | keys[]' "$BASELINE_FILE")
fi

# --- Score-only mode ---

if [[ -n "$SCORE_ONLY" ]]; then
    [[ ${#RESULTS_DIRS[@]} -gt 0 ]] || die "Usage: $0 --score-only <results-dir> [<results-dir> ...]"

    for dir in "${RESULTS_DIRS[@]}"; do
        [ -d "$dir" ] || die "Results directory not found: $dir"
        echo "Scoring: $dir"
        "$EVAL_DIR/score-eval.sh" "$dir"
    done

    echo ""
    echo "Scored ${#RESULTS_DIRS[@]} result(s). Compare manually or re-run without --score-only."
    exit 0
fi

# --- Run evals ---

echo "=== Eval Regression Check ==="
echo "Templates: ${TEMPLATES[*]}"
echo "Baseline:  $BASELINE_FILE"
echo ""

declare -A RESULT_DIRS

for template in "${TEMPLATES[@]}"; do
    # Verify template exists in baseline
    min=$(jq -r --arg t "$template" '.evals[$t].min // empty' "$BASELINE_FILE")
    [[ -n "$min" ]] || die "Template '$template' not found in baseline file"

    echo "--- Running: $template ---"
    output=$("$EVAL_DIR/run-eval.sh" "$template" --skill-only 2>&1)
    echo "$output"

    # Extract the results directory from run-eval output
    result_dir=$(echo "$output" | grep "^Results:" | head -1 | awk '{print $2}')
    [[ -n "$result_dir" && -d "$result_dir" ]] || die "Could not find results directory for $template"
    RESULT_DIRS[$template]="$result_dir"
    echo ""
done

# --- Score ---

echo "=== Scoring ==="
echo ""

declare -A SCORES

for template in "${TEMPLATES[@]}"; do
    result_dir="${RESULT_DIRS[$template]}"
    echo "--- Scoring: $template ---"
    "$EVAL_DIR/score-eval.sh" "$result_dir"

    # Read the score
    score_file="$result_dir/skill.score.json"
    if [[ -f "$score_file" ]]; then
        score=$(jq -r '.total' "$score_file")
        SCORES[$template]="$score"
    else
        echo "WARNING: No score file for $template"
        SCORES[$template]="0"
    fi
    echo ""
done

# --- Compare to baseline ---

echo "=== Regression Check ==="
echo ""

printf "%-25s %8s %8s %8s %10s\n" "Template" "Score" "Min" "Expected" "Status"
printf "%-25s %8s %8s %8s %10s\n" "--------" "-----" "---" "--------" "------"

FAILED=0

for template in "${TEMPLATES[@]}"; do
    score="${SCORES[$template]}"
    min=$(jq -r --arg t "$template" '.evals[$t].min' "$BASELINE_FILE")
    expected=$(jq -r --arg t "$template" '.evals[$t].expected' "$BASELINE_FILE")
    max=$(jq -r --arg t "$template" '.evals[$t].max' "$BASELINE_FILE")

    if [[ "$score" -lt "$min" ]]; then
        status="FAIL"
        FAILED=1
    elif [[ "$score" -lt "$expected" ]]; then
        status="WARN"
    else
        status="PASS"
    fi

    printf "%-25s %8s %8s %8s %10s\n" "$template" "$score/$max" "$min" "$expected" "$status"
done

echo ""

# --- Write summary for CI ---

SUMMARY_FILE="$EVAL_DIR/results/regression-summary.json"
mkdir -p "$(dirname "$SUMMARY_FILE")"

{
    echo "{"
    echo "  \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\","
    echo "  \"passed\": $([ $FAILED -eq 0 ] && echo true || echo false),"
    echo "  \"results\": {"
    first=1
    for template in "${TEMPLATES[@]}"; do
        [ $first -eq 0 ] && echo ","
        first=0
        score="${SCORES[$template]}"
        min=$(jq -r --arg t "$template" '.evals[$t].min' "$BASELINE_FILE")
        expected=$(jq -r --arg t "$template" '.evals[$t].expected' "$BASELINE_FILE")
        printf '    "%s": { "score": %s, "min": %s, "expected": %s }' "$template" "$score" "$min" "$expected"
    done
    echo ""
    echo "  }"
    echo "}"
} > "$SUMMARY_FILE"

echo "Summary: $SUMMARY_FILE"

if [[ $FAILED -eq 1 ]]; then
    echo ""
    echo "REGRESSION DETECTED: One or more evals scored below minimum threshold."
    exit 1
fi

echo ""
echo "All evals passed regression check."
exit 0