codeflash-ai/codeflash-internal
1
0
Fork 0
Code Issues Pull requests 39 Projects Releases Packages Wiki Activity Actions 2.0k

Catch cases where existing tests change test status

Browse source
This commit is contained in:
Saurabh Misra 2024-12-02 17:09:58 -08:00
parent 6eab41b47b
commit cdbf6424f9
3 changed files with 33 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
cli/codeflash/verification/equivalence.py
View file

@ -1,7 +1,7 @@
import sys
from codeflash.verification.comparator import comparator
from codeflash.verification.test_results import TestResults
from codeflash.verification.test_results import TestResults, TestType
INCREASED_RECURSION_LIMIT = 5000
@ -20,17 +20,22 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR
did_all_timeout: bool = True
for test_id in test_ids_superset:
original_test_result = original_results.get_by_unique_invocation_loop_id(test_id)
cdd_test_results = candidate_results.get_by_unique_invocation_loop_id(test_id)
if cdd_test_results is not None and original_test_result is None:
cdd_test_result = candidate_results.get_by_unique_invocation_loop_id(test_id)
if cdd_test_result is not None and original_test_result is None:
continue
if original_test_result is None or cdd_test_results is None:
if original_test_result is None or cdd_test_result is None:
are_equal = False
break
did_all_timeout = did_all_timeout and original_test_result.timed_out
if original_test_result.timed_out:
continue
if not comparator(original_test_result.return_value, cdd_test_results.return_value):
if not comparator(original_test_result.return_value, cdd_test_result.return_value):
are_equal = False
break
if original_test_result.test_type == TestType.EXISTING_UNIT_TEST and (
cdd_test_result.did_pass != original_test_result.did_pass
):
are_equal = False
break
sys.setrecursionlimit(original_recursion_limit)

3
cli/codeflash/verification/test_results.py
View file

@ -182,9 +182,6 @@ class TestResults(BaseModel):
def __setitem__(self, index: int, value: FunctionTestInvocation) -> None:
self.test_results[index] = value
def __delitem__(self, index: int) -> None:
del self.test_results[index]
def __contains__(self, value: FunctionTestInvocation) -> bool:
return value in self.test_results

23
cli/tests/test_comparator.py
View file

@ -769,4 +769,27 @@ def test_compare_results_fn():
assert compare_test_results(original_results, new_results_3)
new_results_4 = TestResults()
new_results_4.add(
FunctionTestInvocation(
id=InvocationId(
test_module_path="test_module_path",
test_class_name="test_class_name",
test_function_name="test_function_name",
function_getting_tested="function_getting_tested",
iteration_id="0",
),
file_name="file_name",
did_pass=False,
runtime=5,
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=5,
timed_out=False,
loop_index=1,
)
)
assert not compare_test_results(original_results, new_results_4)
assert not compare_test_results(TestResults(), TestResults())