codeflash-ai/codeflash
1
0
Fork 0
Code Issues 18 Pull requests 33 Projects Releases Packages Wiki Activity Actions 4.7k

fix: unit tests

Browse source
This commit is contained in:
mohammed 2025-07-04 15:45:39 +03:00
parent 733de2404a
commit f8255fbfd4
2 changed files with 11 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
codeflash/result/critic.py
View file

@ -26,7 +26,10 @@ def performance_gain(*, original_runtime_ns: int, optimized_runtime_ns: int) ->
def speedup_critic(
candidate_result: OptimizedCandidateResult, original_code_runtime: int, best_runtime_until_now: int
candidate_result: OptimizedCandidateResult,
original_code_runtime: int,
best_runtime_until_now: int,
disable_gh_action_noise: bool | None = None,
) -> bool:
"""Take in a correct optimized Test Result and decide if the optimization should actually be surfaced to the user.
@ -35,10 +38,11 @@ def speedup_critic(
when the original runtime is less than 10 microseconds, and becomes MIN_IMPROVEMENT_THRESHOLD for any higher runtime.
The noise floor is doubled when benchmarking on a (noisy) GitHub Action virtual instance, also we want to be more confident there.
"""
in_github_actions_mode = bool(env_utils.get_pr_number())
noise_floor = 3 * MIN_IMPROVEMENT_THRESHOLD if original_code_runtime < 10000 else MIN_IMPROVEMENT_THRESHOLD
if in_github_actions_mode:
noise_floor = noise_floor * 2 # Increase the noise floor in GitHub Actions mode
if not disable_gh_action_noise:
in_github_actions_mode = bool(env_utils.get_pr_number())
if in_github_actions_mode:
noise_floor = noise_floor * 2 # Increase the noise floor in GitHub Actions mode
perf_gain = performance_gain(
original_runtime_ns=original_code_runtime, optimized_runtime_ns=candidate_result.best_test_runtime

6
tests/test_critic.py
View file

@ -41,7 +41,7 @@ def test_speedup_critic() -> None:
total_candidate_timing=12,
)
assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now) # 20% improvement
assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now, True) # 20% improvement
candidate_result = OptimizedCandidateResult(
max_loop_count=5,
@ -52,7 +52,7 @@ def test_speedup_critic() -> None:
optimization_candidate_index=0,
)
assert not speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now) # 6% improvement
assert not speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now, True) # 6% improvement
original_code_runtime = 100000
best_runtime_until_now = 100000
@ -66,7 +66,7 @@ def test_speedup_critic() -> None:
optimization_candidate_index=0,
)
assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now) # 6% improvement
assert speedup_critic(candidate_result, original_code_runtime, best_runtime_until_now, True) # 6% improvement
def test_generated_test_critic() -> None: