mirror of
https://github.com/codeflash-ai/codeflash-internal.git
synced 2026-05-04 18:25:18 +00:00
927cc99588
## Summary - Fix regexes in `buildBenchmarkInfo` that strip empty improved/degraded benchmark sections from PR comments - The regexes didn't match the actual template text: wrong heading level (`####` vs `###`), wrong emoji (`📉` vs `⚡️`), and wrong wording — causing `{benchmark_info_degraded}` to render literally when there are no degraded benchmarks - Add unit tests for improved-only, degraded-only, and empty benchmark scenarios ## Test plan - [x] All 33 tests in `pr-changes-utils.test.ts` pass - [x] New tests verify template placeholders are fully removed when a section is empty
452 lines
15 KiB
TypeScript
452 lines
15 KiB
TypeScript
import { describe, it, expect } from "@jest/globals"
|
|
|
|
import {
|
|
buildPrCommentBody,
|
|
buildOptimizationMetadata,
|
|
buildMergeBranchMsg,
|
|
buildBenchmarkInfo,
|
|
buildResultHeader,
|
|
buildResultDetails,
|
|
buildResultFooter,
|
|
buildResultTestReport,
|
|
parseAndCreateOptimizationsDict,
|
|
parseOptimizationMetadata,
|
|
buildDependentPrTitle,
|
|
buildPrTitle,
|
|
} from "./pr-changes-utils.js"
|
|
|
|
const PR_COMMENT_FIELDS = {
|
|
best_runtime: "13.0 microseconds",
|
|
original_runtime: "15.1 microseconds",
|
|
function_name: "find_validation_error",
|
|
file_path: "src/backend/base/langflow/api/v1/mcp.py",
|
|
speedup_x: "0.16x",
|
|
speedup_pct: "16%",
|
|
loop_count: "84",
|
|
optimization_explanation: "This is a test explanation",
|
|
report_table: {
|
|
"Existing Unit Tests": {
|
|
passed: 0,
|
|
failed: 0,
|
|
},
|
|
"Generated Regression Tests": {
|
|
passed: 10,
|
|
failed: 0,
|
|
},
|
|
"Replay Tests": {
|
|
passed: 0,
|
|
failed: 0,
|
|
},
|
|
"Concolic Coverage Tests": {
|
|
passed: 0,
|
|
failed: 0,
|
|
},
|
|
},
|
|
}
|
|
const EXISTING_TESTS = "existing tests"
|
|
const GENERATED_TESTS = "generated tests"
|
|
const COVERAGE_MESSAGE = "100.0%"
|
|
const NEW_BRANCH_NAME = "codeflash/optimize-pr144-2025-02-15T00.30.35"
|
|
const REPLAY_TESTS = "replay tests"
|
|
const CONCOLIC_TESTS = "concolic tests"
|
|
const OPTIMIZATION_IMPACT = "low"
|
|
|
|
// Integration tests - use real templates, verify structure and patterns
|
|
describe("buildPrCommentBody (Integration)", () => {
|
|
it("should generate valid PR comment with expected sections", () => {
|
|
const result = buildPrCommentBody(
|
|
PR_COMMENT_FIELDS,
|
|
EXISTING_TESTS,
|
|
GENERATED_TESTS,
|
|
COVERAGE_MESSAGE,
|
|
NEW_BRANCH_NAME,
|
|
REPLAY_TESTS,
|
|
CONCOLIC_TESTS,
|
|
"f47ac10b-58cc-4372-a567-0e02b2c3d478",
|
|
)
|
|
|
|
// Test that all key sections are present
|
|
expect(result).toContain("CODEFLASH_OPTIMIZATION:")
|
|
expect(result).toContain("find_validation_error")
|
|
expect(result).toContain("src/backend/base/langflow/api/v1/mcp.py")
|
|
expect(result).toContain("16%")
|
|
expect(result).toContain("0.16x")
|
|
expect(result).toContain("This is a test explanation")
|
|
expect(result).toContain("Correctness verification report")
|
|
expect(result).toContain("git merge codeflash/optimize-pr144-2025-02-15T00.30.35")
|
|
|
|
// Test that template substitution worked
|
|
expect(result).not.toContain("{function_name}")
|
|
expect(result).not.toContain("{file_path}")
|
|
expect(result).not.toContain("{speedup_pct}")
|
|
})
|
|
|
|
it("should generate collapsed format when requested", () => {
|
|
const result = buildPrCommentBody(
|
|
PR_COMMENT_FIELDS,
|
|
EXISTING_TESTS,
|
|
GENERATED_TESTS,
|
|
COVERAGE_MESSAGE,
|
|
NEW_BRANCH_NAME,
|
|
REPLAY_TESTS,
|
|
CONCOLIC_TESTS,
|
|
"f47ac10b-58cc-4372-a567-0e02b2c3d478",
|
|
{ isCollapsed: true },
|
|
)
|
|
|
|
// Collapsed format should have details section
|
|
expect(result).toContain("<details>")
|
|
expect(result).toContain("<summary>")
|
|
expect(result).toContain("</details>")
|
|
})
|
|
})
|
|
|
|
describe("buildResultHeader (Integration)", () => {
|
|
it("should generate proper header with template substitution", () => {
|
|
const result = buildResultHeader(PR_COMMENT_FIELDS)
|
|
|
|
// Should contain all the expected data
|
|
expect(result).toContain("find_validation_error")
|
|
expect(result).toContain("src/backend/base/langflow/api/v1/mcp.py")
|
|
expect(result).toContain("16%")
|
|
expect(result).toContain("0.16x")
|
|
expect(result).toContain("15.1 microseconds")
|
|
expect(result).toContain("13.0 microseconds")
|
|
|
|
// Should not contain template placeholders
|
|
expect(result).not.toContain("{function_name}")
|
|
expect(result).not.toContain("{speedup_pct}")
|
|
})
|
|
})
|
|
|
|
describe("buildResultTestReport (Integration)", () => {
|
|
it("should generate proper test report", () => {
|
|
const result = buildResultTestReport(
|
|
PR_COMMENT_FIELDS,
|
|
EXISTING_TESTS,
|
|
GENERATED_TESTS,
|
|
COVERAGE_MESSAGE,
|
|
REPLAY_TESTS,
|
|
CONCOLIC_TESTS,
|
|
)
|
|
|
|
// Should contain report elements
|
|
expect(result).toContain("Correctness verification report")
|
|
expect(result).toContain("Generated Regression Tests")
|
|
expect(result).toContain("10 Passed")
|
|
expect(result).toContain("100.0%")
|
|
expect(result).toContain("generated tests")
|
|
})
|
|
})
|
|
|
|
describe("parseAndCreateOptimizationsDict", () => {
|
|
it("should parse PR body and comments correctly", () => {
|
|
const PRBODY = buildPrCommentBody(
|
|
PR_COMMENT_FIELDS,
|
|
EXISTING_TESTS,
|
|
GENERATED_TESTS,
|
|
COVERAGE_MESSAGE,
|
|
NEW_BRANCH_NAME,
|
|
REPLAY_TESTS,
|
|
CONCOLIC_TESTS,
|
|
"f47ac10b-58cc-4372-a567-0e02b2c3d478",
|
|
)
|
|
const result = parseAndCreateOptimizationsDict(PRBODY, [])
|
|
expect(result).toEqual({
|
|
"src/backend/base/langflow/api/v1/mcp.py": new Set(["find_validation_error"]),
|
|
})
|
|
})
|
|
|
|
it("should match codeflash/pull/1396", () => {
|
|
// Test that it parses function/file from legacy format
|
|
const EXPECTED2 = `#### 📄 798,389% (7,983.89x) speedup for ***\`sorter\` in \`cli/codeflash/dupes.py\`***`
|
|
const result = parseAndCreateOptimizationsDict(EXPECTED2, [])
|
|
expect(result).toEqual({
|
|
"cli/codeflash/dupes.py": new Set(["sorter"]),
|
|
})
|
|
})
|
|
})
|
|
|
|
describe("GitHub Utils", () => {
|
|
describe("buildPrTitle", () => {
|
|
it("should build a title for a function", () => {
|
|
const title = buildPrTitle("myFunction", "20%", "100x")
|
|
expect(title).toBe("⚡️ Speed up function `myFunction` by 20%")
|
|
})
|
|
|
|
it("should build a title for a method", () => {
|
|
const title = buildPrTitle("myObject.myMethod", "30%", "100x")
|
|
expect(title).toBe("⚡️ Speed up method `myObject.myMethod` by 30%")
|
|
})
|
|
|
|
// Over 1000 whould return the minimal method
|
|
it("should build a title for a method", () => {
|
|
const title = buildPrTitle("myObject.myMethod", "10000%", "100x")
|
|
expect(title).toBe("⚡️ Speed up method `myObject.myMethod` by 100x")
|
|
})
|
|
|
|
it("should build a title for a method", () => {
|
|
const title = buildPrTitle("myObject.myMethod", "500%", "500x")
|
|
expect(title).toBe("⚡️ Speed up method `myObject.myMethod` by 500%")
|
|
})
|
|
})
|
|
|
|
describe("buildDependentPrTitle", () => {
|
|
it("should build a title for a dependent PR for a function", () => {
|
|
const title = buildDependentPrTitle("myFunction", "30%", "100x", 123, "main")
|
|
expect(title).toBe("⚡️ Speed up function `myFunction` by 30% in PR #123 (`main`)")
|
|
})
|
|
|
|
it("should build a title for a dependent PR with a method", () => {
|
|
const title = buildDependentPrTitle("myObject.myMethod", "30%", "100x", 456, "develop")
|
|
expect(title).toBe("⚡️ Speed up method `myObject.myMethod` by 30% in PR #456 (`develop`)")
|
|
})
|
|
})
|
|
})
|
|
|
|
// Unit tests for individual functions
|
|
describe("buildOptimizationMetadata", () => {
|
|
it("should build metadata JSON comment", () => {
|
|
const result = buildOptimizationMetadata(PR_COMMENT_FIELDS, "")
|
|
expect(result).toContain("<!-- CODEFLASH_OPTIMIZATION:")
|
|
expect(result).toContain('"function":"find_validation_error"')
|
|
expect(result).toContain('"file":"src/backend/base/langflow/api/v1/mcp.py"')
|
|
expect(result).toContain('"speedup_pct":"16%"')
|
|
expect(result).toContain('"speedup_x":"0.16x"')
|
|
expect(result).toContain("-->")
|
|
})
|
|
})
|
|
|
|
describe("buildMergeBranchMsg", () => {
|
|
it("should build merge message with branch name", () => {
|
|
const result = buildMergeBranchMsg("feature/test-branch")
|
|
expect(result).toBe(
|
|
"To test or edit this optimization locally `git merge feature/test-branch`\n\n",
|
|
)
|
|
})
|
|
|
|
it("should return empty string for empty branch name", () => {
|
|
const result = buildMergeBranchMsg("")
|
|
expect(result).toBe("")
|
|
})
|
|
|
|
it("should return empty string for null/undefined branch name", () => {
|
|
expect(buildMergeBranchMsg(null as any)).toBe("")
|
|
expect(buildMergeBranchMsg(undefined as any)).toBe("")
|
|
})
|
|
})
|
|
|
|
describe("buildBenchmarkInfo", () => {
|
|
it("should build benchmark info when benchmark_details exist", () => {
|
|
const fieldsWithBenchmarks = {
|
|
...PR_COMMENT_FIELDS,
|
|
benchmark_details: [
|
|
{
|
|
benchmark_name: "test1",
|
|
test_function: "testFunc1",
|
|
original_timing: 100,
|
|
expected_new_timing: 80,
|
|
speedup_percent: 20,
|
|
},
|
|
{
|
|
benchmark_name: "test2",
|
|
test_function: "testFunc2",
|
|
original_timing: 100,
|
|
expected_new_timing: 105,
|
|
speedup_percent: -5,
|
|
},
|
|
],
|
|
}
|
|
const result = buildBenchmarkInfo(fieldsWithBenchmarks)
|
|
expect(result).toContain("This change will improve the performance")
|
|
expect(result).toContain("This change will degrade the performance")
|
|
})
|
|
|
|
it("should remove degraded section when only improved benchmarks exist", () => {
|
|
const fieldsOnlyImproved = {
|
|
...PR_COMMENT_FIELDS,
|
|
benchmark_details: [
|
|
{
|
|
benchmark_name: "test_bench",
|
|
test_function: "test_benchmark_extract",
|
|
original_timing: "8.34 seconds",
|
|
expected_new_timing: "8.34 seconds",
|
|
speedup_percent: 0.01,
|
|
},
|
|
],
|
|
}
|
|
const result = buildBenchmarkInfo(fieldsOnlyImproved)
|
|
expect(result).toContain("This change will improve the performance")
|
|
expect(result).not.toContain("This change will degrade the performance")
|
|
expect(result).not.toContain("{benchmark_info_degraded}")
|
|
})
|
|
|
|
it("should remove improved section when only degraded benchmarks exist", () => {
|
|
const fieldsOnlyDegraded = {
|
|
...PR_COMMENT_FIELDS,
|
|
benchmark_details: [
|
|
{
|
|
benchmark_name: "test_bench",
|
|
test_function: "test_benchmark_extract",
|
|
original_timing: "8.34 seconds",
|
|
expected_new_timing: "9.00 seconds",
|
|
speedup_percent: -7.9,
|
|
},
|
|
],
|
|
}
|
|
const result = buildBenchmarkInfo(fieldsOnlyDegraded)
|
|
expect(result).toContain("This change will degrade the performance")
|
|
expect(result).not.toContain("This change will improve the performance")
|
|
expect(result).not.toContain("{benchmark_info_improved}")
|
|
})
|
|
|
|
it("should return empty string when no benchmark_details", () => {
|
|
const result = buildBenchmarkInfo(PR_COMMENT_FIELDS)
|
|
expect(result).toBe("")
|
|
})
|
|
})
|
|
|
|
describe("buildResultDetails", () => {
|
|
it("should build expanded details by default", () => {
|
|
const result = buildResultDetails(PR_COMMENT_FIELDS)
|
|
expect(result).toContain("This is a test explanation")
|
|
expect(result).not.toContain("<details>")
|
|
})
|
|
|
|
it("should build collapsed details when requested", () => {
|
|
const result = buildResultDetails(PR_COMMENT_FIELDS, true)
|
|
expect(result).toContain("<details>")
|
|
expect(result).toContain("<summary>")
|
|
expect(result).toContain("This is a test explanation")
|
|
expect(result).toContain("</details>")
|
|
})
|
|
})
|
|
|
|
describe("buildResultFooter", () => {
|
|
it("should build footer with branch name", () => {
|
|
const result = buildResultFooter("test-branch")
|
|
expect(result).toContain("git checkout test-branch")
|
|
})
|
|
})
|
|
|
|
describe("parseOptimizationMetadata", () => {
|
|
it("should parse optimization metadata from comment", () => {
|
|
const comment =
|
|
'<!-- CODEFLASH_OPTIMIZATION: {"function":"testFunc","file":"test.js","speedup_pct":"50%"} -->'
|
|
const result = parseOptimizationMetadata(comment, [])
|
|
expect(result).toEqual([
|
|
{
|
|
function: "testFunc",
|
|
file: "test.js",
|
|
speedup_pct: "50%",
|
|
},
|
|
])
|
|
})
|
|
|
|
it("should return empty object for invalid JSON", () => {
|
|
const comment = "<!-- CODEFLASH_OPTIMIZATION: invalid json -->"
|
|
const result = parseOptimizationMetadata(comment, [])
|
|
expect(result).toEqual([])
|
|
})
|
|
|
|
it("should return empty object when no metadata found", () => {
|
|
const comment = "regular comment without metadata"
|
|
const result = parseOptimizationMetadata(comment, [])
|
|
expect(result).toEqual([])
|
|
})
|
|
})
|
|
|
|
describe("buildPrCommentBody - Extended Coverage", () => {
|
|
it("should handle options.includeHeader = false", () => {
|
|
const result = buildPrCommentBody(
|
|
PR_COMMENT_FIELDS,
|
|
EXISTING_TESTS,
|
|
GENERATED_TESTS,
|
|
COVERAGE_MESSAGE,
|
|
NEW_BRANCH_NAME,
|
|
REPLAY_TESTS,
|
|
CONCOLIC_TESTS,
|
|
"f47ac10b-58cc-4372-a567-0e02b2c3d478",
|
|
{ includeHeader: false },
|
|
)
|
|
expect(result).not.toContain("Codeflash found optimizations for this PR")
|
|
})
|
|
|
|
it("should handle options.isUnifiedReview = true", () => {
|
|
const result = buildPrCommentBody(
|
|
PR_COMMENT_FIELDS,
|
|
EXISTING_TESTS,
|
|
GENERATED_TESTS,
|
|
COVERAGE_MESSAGE,
|
|
NEW_BRANCH_NAME,
|
|
REPLAY_TESTS,
|
|
CONCOLIC_TESTS,
|
|
"f47ac10b-58cc-4372-a567-0e02b2c3d478",
|
|
{ isUnifiedReview: true },
|
|
)
|
|
// Should still contain the core content
|
|
expect(result).toContain("find_validation_error")
|
|
})
|
|
|
|
it("should include benchmark info when present", () => {
|
|
const fieldsWithBenchmarks = {
|
|
...PR_COMMENT_FIELDS,
|
|
benchmark_details: [
|
|
{
|
|
benchmark_name: "test",
|
|
test_function: "testFunc",
|
|
original_timing: 100,
|
|
expected_new_timing: 80,
|
|
speedup_percent: 20,
|
|
},
|
|
],
|
|
}
|
|
const result = buildPrCommentBody(
|
|
fieldsWithBenchmarks,
|
|
EXISTING_TESTS,
|
|
GENERATED_TESTS,
|
|
COVERAGE_MESSAGE,
|
|
NEW_BRANCH_NAME,
|
|
REPLAY_TESTS,
|
|
CONCOLIC_TESTS,
|
|
"f47ac10b-58cc-4372-a567-0e02b2c3d478",
|
|
)
|
|
expect(result).toContain("This change will improve the performance")
|
|
})
|
|
})
|
|
|
|
describe("parseAndCreateOptimizationsDict - Extended Coverage", () => {
|
|
it("should handle empty input", () => {
|
|
const result = parseAndCreateOptimizationsDict("", [])
|
|
expect(result).toEqual({})
|
|
})
|
|
|
|
it("should handle multiple functions in same file", () => {
|
|
const input = `
|
|
#### 📄 20% (1.2x) speedup for ***\`func1\` in \`test.py\`***
|
|
#### 📄 30% (1.3x) speedup for ***\`func2\` in \`test.py\`***
|
|
`
|
|
const result = parseAndCreateOptimizationsDict(input, [])
|
|
expect(result).toEqual({
|
|
"test.py": new Set(["func1", "func2"]),
|
|
})
|
|
})
|
|
|
|
it("should handle comments array", () => {
|
|
const comments = [{ body: "#### 📄 25% (1.25x) speedup for ***`testFunc` in `test.js`***" }]
|
|
const result = parseAndCreateOptimizationsDict("", comments)
|
|
expect(result).toEqual({
|
|
"test.js": new Set(["testFunc"]),
|
|
})
|
|
})
|
|
|
|
it("should combine PR body and comments", () => {
|
|
const prBody = "#### 📄 20% (1.2x) speedup for ***`func1` in `file1.py`***"
|
|
const comments = [{ body: "#### 📄 30% (1.3x) speedup for ***`func2` in `file2.py`***" }]
|
|
const result = parseAndCreateOptimizationsDict(prBody, comments)
|
|
expect(result).toEqual({
|
|
"file1.py": new Set(["func1"]),
|
|
"file2.py": new Set(["func2"]),
|
|
})
|
|
})
|
|
})
|