changed PR message to include benchmark information

js/cf-api/github/create-pr-from-diffcontents.ts

@@ -1,7 +1,12 @@
 import { type FileDiffContent } from "code-suggester/build/src/types.js"
 import { type Octokit } from "octokit"
 import { addLabelToPullRequest } from "./github-utils.js"
-import { buildDependentPrTitle, buildPrTitle, buildResultFooter } from "./pr-changes-utils.js"
+import {
+  buildBenchmarkInfo,
+  buildDependentPrTitle,
+  buildPrTitle,
+  buildResultFooter,
+} from "./pr-changes-utils.js"
 import type { RestEndpointMethodTypes } from "@octokit/rest"
 import { buildResultHeader, buildResultDetails, buildResultTestReport } from "./pr-changes-utils.js"
 import { AnyOctokit, PullRequestCreationResponse } from "../types.js"
@@ -13,16 +18,25 @@ type Response<T> = {
   status: number
   data: T
 }
+interface BenchmarkDetail {
+  benchmark_name: string
+  test_function: string
+  original_timing: number
+  expected_new_timing: number
+  speedup_percent: number
+}
+
 interface PrCommentFields {
+  optimization_explanation: string
   best_runtime: string
   original_runtime: string
   function_name: string
   file_path: string
   speedup_x: string
   speedup_pct: string
-  loop_count: string
-  optimization_explanation: string
-  report_table: string
+  loop_count: number
+  report_table: Record<string, Record<string, number>>
+  benchmark_details?: BenchmarkDetail[]
 }
 
 export async function createNewBranchFromDiffContents(
@@ -158,6 +172,11 @@ export async function createStandalonePullRequest(
 ): Promise<PullRequestCreationResponse> {
   // Open a new standalone Codeflash PR (that doesn't reference an original PR, likely just to main)
   const prCommentHeader = buildResultHeader(prCommentFields)
+  // Build benchmark info if available
+  const benchmarkInfo =
+    prCommentFields.benchmark_details && prCommentFields.benchmark_details.length > 0
+      ? buildBenchmarkInfo(prCommentFields)
+      : ""
   // Open a new PR from the new branch onto the original PR's head branch
   const prCommentBody = buildResultDetails(prCommentFields)
   const prCommentTestReport = buildResultTestReport(
@@ -172,11 +191,9 @@ export async function createStandalonePullRequest(
     prCommentFields.speedup_pct,
     prCommentFields.speedup_x,
   )
-  const body: string =
-    `${prCommentHeader}\n` +
-    `${prCommentBody}\n` +
-    `${prCommentTestReport}\n` +
-    `${prCommentFooter}`
+  const body: string = benchmarkInfo
+    ? `${prCommentHeader}\n${benchmarkInfo}\n${prCommentBody}\n${prCommentTestReport}\n${prCommentFooter}`
+    : `${prCommentHeader}\n${prCommentBody}\n${prCommentTestReport}\n${prCommentFooter}`
 
   return await createNewPullRequest(
     installationOctokit,
@@ -202,6 +219,11 @@ export async function createDependentPullRequest(
   coverage_message: string,
 ): Promise<PullRequestCreationResponse> {
   const prCommentHeader = buildResultHeader(prCommentFields)
+  // Build benchmark info if available
+  const benchmarkInfo =
+    prCommentFields.benchmark_details && prCommentFields.benchmark_details.length > 0
+      ? buildBenchmarkInfo(prCommentFields)
+      : ""
   // Open a new PR from the new branch onto the original PR's head branch
   const prCommentBody = buildResultDetails(prCommentFields)
   const prCommentTestReport = buildResultTestReport(
@@ -218,15 +240,15 @@ export async function createDependentPullRequest(
     origPrNumber,
     baseBranch,
   )
-  const body =
+  const introSection =
     `## ⚡️ This pull request contains optimizations for PR #${origPrNumber}
 If you approve this dependent PR, these changes will be merged into the original PR branch \`${baseBranch}\`.
->This PR will be automatically closed if the original PR is merged.\n` +
-    `----\n` +
-    `${prCommentHeader}\n` +
-    `${prCommentBody}\n` +
-    `${prCommentTestReport}\n` +
-    `${prCommentFooter}`
+>This PR will be automatically closed if the original PR is merged.\n` + `----\n`
+
+  // Conditionally build the body based on whether benchmark info exists
+  const body = benchmarkInfo
+    ? `${introSection}${prCommentHeader}\n${benchmarkInfo}\n${prCommentBody}\n${prCommentTestReport}\n${prCommentFooter}`
+    : `${introSection}${prCommentHeader}\n${prCommentBody}\n${prCommentTestReport}\n${prCommentFooter}`
 
   const newPrData = await createNewPullRequest(
     installationOctokit,

js/cf-api/github/optimization_results_benchmark_info.md

@@ -0,0 +1,7 @@
+### ⚡️ This change will improve the performance of the following benchmarks:
+
+{benchmark_info_improved}
+
+### 🔻 This change will degrade the performance of the following benchmarks:
+
+{benchmark_info_degraded}

js/cf-api/github/pr-changes-utils.ts

@@ -17,6 +17,10 @@ const PR_TEST_REPORT_TEMPLATE = fs.readFileSync(
   path.join(__dirname, "optimization_results_test_report.md"),
   "utf8",
 )
+const PR_BENCHMARK_INFO_TEMPLATE = fs.readFileSync(
+  path.join(__dirname, "optimization_results_benchmark_info.md"),
+  "utf8",
+)
 
 const summaryLinesRegex = new RegExp(
   /### 📄 ([\d,]+(\.\d+)?)% \(([\d,]+(\.\d+)?)x\) speedup for \*\*\*`(.+?)` in `(.+?)`\*\*\*/g,
@@ -54,6 +58,74 @@ export function buildMergeBranchMsg(newBranchName: string): string {
   return ""
 }
 
+export function buildBenchmarkInfo(fields: PrCommentFields): string {
+  if (!fields.benchmark_details || fields.benchmark_details.length === 0) {
+    return ""
+  }
+
+  // Separate benchmarks into improved and degraded categories
+  const improvedBenchmarks = fields.benchmark_details.filter(detail => detail.speedup_percent > 0)
+  const degradedBenchmarks = fields.benchmark_details.filter(detail => detail.speedup_percent <= 0)
+
+  // Build improved benchmarks table if there are any
+  let improvedBenchmarkInfo = ""
+  if (improvedBenchmarks.length > 0) {
+    improvedBenchmarkInfo = "\n\n"
+    improvedBenchmarkInfo +=
+      "| Benchmark File :: Function | Original Runtime | Expected New Runtime | Speedup |\n"
+    improvedBenchmarkInfo += "| :--- | ---: | ---: | ---: |\n"
+
+    for (const detail of improvedBenchmarks) {
+      const benchmarkName = `${detail.benchmark_name}::${detail.test_function}`
+      improvedBenchmarkInfo += `| ${benchmarkName} | ${detail.original_timing} | ${detail.expected_new_timing} | **${detail.speedup_percent.toFixed(2)}%** |\n`
+    }
+  }
+
+  // Build degraded benchmarks table if there are any
+  let degradedBenchmarkInfo = ""
+  if (degradedBenchmarks.length > 0) {
+    degradedBenchmarkInfo = "\n\n"
+    degradedBenchmarkInfo +=
+      "| Benchmark File :: Function | Original Runtime | Expected New Runtime | Slowdown |\n"
+    degradedBenchmarkInfo += "| :--- | ---: | ---: | ---: |\n"
+
+    for (const detail of degradedBenchmarks) {
+      const benchmarkName = `${detail.benchmark_name}::${detail.test_function}`
+      // Use absolute value to show slowdown as a positive percentage
+      const slowdownPercent = Math.abs(detail.speedup_percent)
+      degradedBenchmarkInfo += `| ${benchmarkName} | ${detail.original_timing} | ${detail.expected_new_timing} | **${slowdownPercent.toFixed(2)}%** |\n`
+    }
+  } else {
+    // Add a sample degraded benchmark row for preview
+    degradedBenchmarkInfo = "\n\n"
+    degradedBenchmarkInfo +=
+      "| Benchmark File :: Function | Original Runtime | Expected New Runtime | Slowdown |\n"
+    degradedBenchmarkInfo += "| :--- | ---: | ---: | ---: |\n"
+    degradedBenchmarkInfo +=
+      "| test_sample_benchmark.py::test_sample_function | 5.0 milliseconds | 7.5 milliseconds | **50.00%** |\n"
+  }
+
+  // Replace placeholders in template
+  let result = PR_BENCHMARK_INFO_TEMPLATE
+
+  // Replace improved benchmark info or remove the section if no improvements
+  if (improvedBenchmarks.length > 0) {
+    result = result.replace(/\{benchmark_info_improved\}/g, improvedBenchmarkInfo)
+  } else {
+    // Remove the entire improved section including header
+    result = result.replace(
+      /###\s+📉\s+This change improved performance of the following benchmarks:[\s\S]*?\{benchmark_info_improved\}\s*/g,
+      "",
+    )
+  }
+
+  // Always include the degraded section with our sample
+  result = result.replace(/\{benchmark_info_degraded\}/g, degradedBenchmarkInfo)
+
+  return result
+}
+
+// Update the buildResultHeader function to use the new benchmark info function
 export function buildResultHeader(fields: PrCommentFields): string {
   return PR_HEADER_TEMPLATE.replace(/\{best_runtime}/g, fields.best_runtime)
     .replace(/\{original_runtime}/g, fields.original_runtime)
@@ -61,7 +133,7 @@ export function buildResultHeader(fields: PrCommentFields): string {
     .replace(/\{file_path}/g, fields.file_path)
     .replace(/\{speedup_x}/g, fields.speedup_x)
     .replace(/\{speedup_pct}/g, fields.speedup_pct)
-    .replace(/\{num_runs}/g, fields.loop_count)
+    .replace(/\{num_runs}/g, fields.loop_count.toString())
 }
 
 export function buildResultDetails(fields: PrCommentFields): string {