Merge branch 'main' into skyvern-grace

codeflash/result/explanation.py

@@ -33,6 +33,15 @@ class Explanation:
 
     @property
     def perf_improvement_line(self) -> str:
+        improvement_type = {
+            AcceptanceReason.RUNTIME: "runtime",
+            AcceptanceReason.THROUGHPUT: "throughput",
+            AcceptanceReason.CONCURRENCY: "concurrency",
+            AcceptanceReason.NONE: "",
+        }.get(self.acceptance_reason, "")
+
+        if improvement_type:
+            return f"{self.speedup_pct} {improvement_type} improvement ({self.speedup_x} faster)."
         return f"{self.speedup_pct} improvement ({self.speedup_x} faster)."
 
     @property

tests/scripts/end_to_end_test_async.py

@@ -8,6 +8,7 @@ def run_test(expected_improvement_pct: int) -> bool:
     config = TestConfig(
         file_path="main.py",
         min_improvement_x=0.1,
+        expected_acceptance_reason="concurrency",
         coverage_expectations=[
             CoverageExpectation(
                 function_name="retry_with_backoff",

tests/scripts/end_to_end_test_utilities.py

@@ -37,6 +37,7 @@ class TestConfig:
     benchmarks_root: Optional[pathlib.Path] = None
     use_worktree: bool = False
     no_gen_tests: bool = False
+    expected_acceptance_reason: Optional[str] = None  # "runtime", "throughput", "concurrency"
 
 
 def clear_directory(directory_path: str | pathlib.Path) -> None:
@@ -176,7 +177,7 @@ def validate_output(stdout: str, return_code: int, expected_improvement_pct: int
         logging.error("Failed to find performance improvement message")
         return False
 
-    improvement_match = re.search(r"📈 ([\d,]+)% improvement", stdout)
+    improvement_match = re.search(r"📈 ([\d,]+)% (?:(\w+) )?improvement", stdout)
     if not improvement_match:
         logging.error("Could not find improvement percentage in output")
         return False
@@ -193,6 +194,15 @@ def validate_output(stdout: str, return_code: int, expected_improvement_pct: int
         logging.error(f"Performance improvement rate {improvement_x}x not above {config.min_improvement_x}x")
         return False
 
+    if config.expected_acceptance_reason is not None:
+        actual_reason = improvement_match.group(2)
+        if not actual_reason:
+            logging.error("Could not find acceptance reason type in output")
+            return False
+        if actual_reason != config.expected_acceptance_reason:
+            logging.error(f"Expected acceptance reason '{config.expected_acceptance_reason}', got '{actual_reason}'")
+            return False
+
     if config.expected_unit_tests_count is not None:
         # Match the global test discovery message from optimizer.py which counts test invocations
         # Format: "Discovered X existing unit tests and Y replay tests in Z.Zs at /path/to/tests"