codeflash-ai/codeflash
1
0
Fork 0
mirror of https://github.com/codeflash-ai/codeflash.git synced 2026-05-04 18:25:17 +00:00
Code Issues 18 Projects Releases Packages Wiki Activity Actions

Changed verification type names, improved tests

Browse source
This commit is contained in:
Alvin Ryanputra 2025-01-23 09:10:32 -08:00
parent 7b78513868
commit 741f95ad0a
10 changed files with 56 additions and 50 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
codeflash/code_utils/instrument_existing_tests.py
View file

@ -696,7 +696,7 @@ def create_wrapper_function(mode: TestingMode = TestingMode.BEHAVIOR) -> ast.Fun
ast.Name(id="invocation_id", ctx=ast.Load()),
ast.Name(id="codeflash_duration", ctx=ast.Load()),
ast.Name(id="pickled_return_value", ctx=ast.Load()),
ast.Constant(value=VerificationType.FUNCTION_TO_OPTIMIZE),
ast.Constant(value=VerificationType.FUNCTION_CALL.value),
],
ctx=ast.Load(),
),

2
codeflash/verification/codeflash_capture.py
View file

@ -116,7 +116,7 @@ def codeflash_capture(function_name: str, tmp_dir_path: str, is_fto: bool = Fals
invocation_id,
codeflash_duration,
pickled_return_value,
VerificationType.INSTANCE_STATE_FTO if is_fto else VerificationType.INSTANCE_STATE_HELPER,
VerificationType.INIT_STATE_FTO if is_fto else VerificationType.INIT_STATE_HELPER,
),
)
codeflash_con.commit()

6
codeflash/verification/equivalence.py
View file

@ -26,7 +26,7 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR
# If helper function instance_state verification is not present, that's ok. continue
if (
original_test_result.verification_type
and original_test_result.verification_type == VerificationType.INSTANCE_STATE_HELPER
and original_test_result.verification_type == VerificationType.INIT_STATE_HELPER
and cdd_test_result is None
):
continue
@ -38,8 +38,8 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR
continue
superset_obj = False
if original_test_result.verification_type and (
original_test_result.verification_type == VerificationType.INSTANCE_STATE_HELPER
or original_test_result.verification_type == VerificationType.INSTANCE_STATE_FTO
original_test_result.verification_type
in (VerificationType.INIT_STATE_HELPER, VerificationType.INIT_STATE_FTO)
):
superset_obj = True
if not comparator(original_test_result.return_value, cdd_test_result.return_value, superset_obj=superset_obj):

5
codeflash/verification/parse_test_output.py
View file

@ -92,7 +92,7 @@ def parse_test_return_values_bin(file_location: Path, test_files: TestFiles, tes
test_type=test_type,
return_value=test_pickle,
timed_out=False,
verification_type=VerificationType.FUNCTION_TO_OPTIMIZE,
verification_type=VerificationType.FUNCTION_CALL,
)
)
return test_results
@ -120,7 +120,8 @@ def parse_sqlite_test_results(sqlite_file_path: Path, test_files: TestFiles, tes
test_function_name = val[2] if val[2] else None
function_getting_tested = val[3]
test_file_path = file_path_from_module_name(test_module_path, test_config.tests_project_rootdir)
test_type = test_files.get_test_type_by_instrumented_file_path(test_file_path) # may not be the best way
test_type = (test_files.get_test_type_by_instrumented_file_path(test_file_path) or
test_files.get_test_type_by_original_file_path(test_file_path))
loop_index = val[4]
iteration_id = val[5]
runtime = val[6]

19
codeflash/verification/test_results.py
View file

@ -14,13 +14,18 @@ from codeflash.verification.comparator import comparator
class VerificationType(str, Enum):
FUNCTION_TO_OPTIMIZE = (
"function_to_optimize" # Correctness verification for fto, checks input values and output values
)
INSTANCE_STATE_FTO = "instance_state_fto" # Correctness verification for fto class instance attributes after init
INSTANCE_STATE_HELPER = (
"instance_state_helper" # Correctness verification for helper class instance attributes after init
# FUNCTION_TO_OPTIMIZE = (
# "function_test" # Correctness verification for a test function, checks input values and output values
# )
# INSTANCE_STATE_FTO = "instance_state_fto" # Correctness verification for fto class instance attributes after init
# INSTANCE_STATE_HELPER = (
# "instance_state_helper" # Correctness verification for helper class instance attributes after init
# )
FUNCTION_CALL = (
"function_call" # Correctness verification for a test function, checks input values and output values)
)
INIT_STATE_FTO = "init_state_fto" # Correctness verification for fto class instance attributes after init
INIT_STATE_HELPER = "init_state_helper" # Correctness verification for helper class instance attributes after init
def __new__(cls, value: str) -> VerificationType | None:
obj = str.__new__(cls, value)
@ -89,7 +94,7 @@ class FunctionTestInvocation:
test_type: TestType
return_value: Optional[object] # The return value of the function invocation
timed_out: Optional[bool]
verification_type: str = VerificationType.FUNCTION_TO_OPTIMIZE
verification_type: str = VerificationType.FUNCTION_CALL
@property
def unique_invocation_loop_id(self) -> str:

2
tests/test_code_replacement.py
View file

@ -1565,7 +1565,7 @@ print("Hello world")
class NewClass:
def __init__(self, name):
self.name = name
self.name = str(name)
def __call__(self, value):
return "I am still old"
def new_function2(value):

16
tests/test_codeflash_capture.py
View file

@ -623,13 +623,13 @@ class AnotherHelperClass:
assert len(test_results.test_results) == 4
assert test_results[0].id.test_function_name == "test_helper_classes"
assert test_results[0].id.function_getting_tested == "MyClass.__init__"
assert test_results[0].verification_type == VerificationType.INSTANCE_STATE_FTO
assert test_results[0].verification_type == VerificationType.INIT_STATE_FTO
assert test_results[1].id.function_getting_tested == "HelperClass1.__init__"
assert test_results[1].verification_type == VerificationType.INSTANCE_STATE_HELPER
assert test_results[1].verification_type == VerificationType.INIT_STATE_HELPER
assert test_results[2].id.function_getting_tested == "HelperClass2.__init__"
assert test_results[2].verification_type == VerificationType.INSTANCE_STATE_HELPER
assert test_results[2].verification_type == VerificationType.INIT_STATE_HELPER
assert test_results[3].id.function_getting_tested == "AnotherHelperClass.__init__"
assert test_results[3].verification_type == VerificationType.INSTANCE_STATE_HELPER
assert test_results[3].verification_type == VerificationType.INIT_STATE_HELPER
finally:
test_path.unlink(missing_ok=True)
@ -777,13 +777,13 @@ class AnotherHelperClass:
assert len(test_results.test_results) == 4
assert test_results[0].id.test_function_name == "test_helper_classes"
assert test_results[0].id.function_getting_tested == "MyClass.__init__"
assert test_results[0].verification_type == VerificationType.INSTANCE_STATE_FTO
assert test_results[0].verification_type == VerificationType.INIT_STATE_FTO
assert test_results[1].id.function_getting_tested == "HelperClass1.__init__"
assert test_results[1].verification_type == VerificationType.INSTANCE_STATE_HELPER
assert test_results[1].verification_type == VerificationType.INIT_STATE_HELPER
assert test_results[2].id.function_getting_tested == "HelperClass2.__init__"
assert test_results[2].verification_type == VerificationType.INSTANCE_STATE_HELPER
assert test_results[2].verification_type == VerificationType.INIT_STATE_HELPER
assert test_results[3].id.function_getting_tested == "AnotherHelperClass.__init__"
assert test_results[3].verification_type == VerificationType.INSTANCE_STATE_HELPER
assert test_results[3].verification_type == VerificationType.INIT_STATE_HELPER
# Now, let's say we optimize the code and make changes.
new_fto_code = """

38
tests/test_instrumentation.py
View file

@ -40,7 +40,7 @@ codeflash_wrap_string = """def codeflash_wrap(wrapped, test_module_name, test_cl
exception = e
gc.enable()
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_con.commit()
if exception:
raise exception
@ -132,7 +132,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi
exception = e
gc.enable()
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_con.commit()
if exception:
raise exception
@ -146,7 +146,7 @@ class TestPigLatin(unittest.TestCase):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
input = [5, 4, 3, 2, 1, 0]
output = codeflash_wrap(sorter, '{module_path}', 'TestPigLatin', 'test_sort', 'sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input)
self.assertEqual(output, [0, 1, 2, 3, 4, 5])
@ -233,7 +233,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi
exception = e
gc.enable()
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_con.commit()
if exception:
raise exception
@ -244,7 +244,7 @@ def test_prepare_image_for_yolo():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
"""
if sys.version_info < (3, 11):
expected += """ for (arg_val_pkl, return_val_pkl) in get_next_arg_and_return('/home/saurabh/packagename/traces/first.trace', 3):
@ -307,7 +307,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
input = [5, 4, 3, 2, 1, 0]
output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input)
assert output == [0, 1, 2, 3, 4, 5]
@ -529,7 +529,7 @@ def test_sort_parametrized(input, expected_output):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort_parametrized', 'sorter', '0', codeflash_loop_index, codeflash_cur, codeflash_con, input)
assert output == expected_output
codeflash_con.close()
@ -755,7 +755,7 @@ def test_sort_parametrized_loop(input, expected_output):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
for i in range(2):
output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort_parametrized_loop', 'sorter', '0_0', codeflash_loop_index, codeflash_cur, codeflash_con, input)
assert output == expected_output
@ -1063,7 +1063,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
inputs = [[5, 4, 3, 2, 1, 0], [5.0, 4.0, 3.0, 2.0, 1.0, 0.0], list(reversed(range(50)))]
expected_outputs = [[0, 1, 2, 3, 4, 5], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], list(range(50))]
for i in range(3):
@ -1315,7 +1315,7 @@ class TestPigLatin(unittest.TestCase):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
input = [5, 4, 3, 2, 1, 0]
output = codeflash_wrap(sorter, '{module_path}', 'TestPigLatin', 'test_sort', 'sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input)
self.assertEqual(output, [0, 1, 2, 3, 4, 5])
@ -1586,7 +1586,7 @@ class TestPigLatin(unittest.TestCase):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
output = codeflash_wrap(sorter, '{module_path}', 'TestPigLatin', 'test_sort', 'sorter', '0', codeflash_loop_index, codeflash_cur, codeflash_con, input)
self.assertEqual(output, expected_output)
codeflash_con.close()
@ -1834,7 +1834,7 @@ class TestPigLatin(unittest.TestCase):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
inputs = [[5, 4, 3, 2, 1, 0], [5.0, 4.0, 3.0, 2.0, 1.0, 0.0], list(reversed(range(50)))]
expected_outputs = [[0, 1, 2, 3, 4, 5], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], list(range(50))]
for i in range(3):
@ -2094,7 +2094,7 @@ class TestPigLatin(unittest.TestCase):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
for i in range(2):
output = codeflash_wrap(sorter, '{module_path}', 'TestPigLatin', 'test_sort', 'sorter', '0_0', codeflash_loop_index, codeflash_cur, codeflash_con, input)
self.assertEqual(output, expected_output)
@ -2422,7 +2422,7 @@ def test_class_name_A_function_name():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
ret = codeflash_wrap(class_name_A.function_name, '{module_path}', None, 'test_class_name_A_function_name', 'class_name_A.function_name', '0', codeflash_loop_index, codeflash_cur, codeflash_con, **args)
codeflash_con.close()
"""
@ -2491,7 +2491,7 @@ def test_common_tags_1():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
articles_1 = [1, 2, 3]
assert codeflash_wrap(find_common_tags, '{module_path}', None, 'test_common_tags_1', 'find_common_tags', '1', codeflash_loop_index, codeflash_cur, codeflash_con, articles_1) == set(1, 2)
articles_2 = [1, 2]
@ -2558,7 +2558,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
input = [5, 4, 3, 2, 1, 0]
if len(input) > 0:
assert codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '1_0', codeflash_loop_index, codeflash_cur, codeflash_con, input) == [0, 1, 2, 3, 4, 5]
@ -2625,7 +2625,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
input = [5, 4, 3, 2, 1, 0]
output = codeflash_wrap(BubbleSorter.sorter, 'tests.pytest.test_perfinjector_bubble_sort_results_temp', None, 'test_sort', 'BubbleSorter.sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input)
assert output == [0, 1, 2, 3, 4, 5]
@ -2745,7 +2745,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi
exception = e
gc.enable()
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_con.commit()
if exception:
raise exception
@ -2756,7 +2756,7 @@ def test_code_replacement10() -> None:
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
get_code_output = 'random code'
file_path = Path(__file__).resolve()
opt = Optimizer(Namespace(project_root=str(file_path.parent.resolve()), disable_telemetry=True, tests_root='tests', test_framework='pytest', pytest_cmd='pytest', experiment_id=None))

8
tests/test_instrumentation_results_aiservice.py
View file

@ -317,7 +317,7 @@ def test_single_element_list():
with test_path.open("w") as f:
f.write(instrumented_behavior_test_source)
# Add codeflash capture decorator
instrument_code(function_to_optimize)
instrument_code(function_to_optimize, {})
opt = Optimizer(
Namespace(
project_root=project_root_path,
@ -353,7 +353,7 @@ def test_single_element_list():
testing_time=0.1,
)
# Verify instance_state result, which checks instance state right after __init__, using codeflash_capture
assert test_results[0].id.function_getting_tested == "sorter"
assert test_results[0].id.function_getting_tested == "BubbleSorter.__init__"
assert test_results[0].id.test_function_name == "test_single_element_list"
assert test_results[0].did_pass
assert test_results[0].return_value[0] == {"x": 0}
@ -387,7 +387,7 @@ class BubbleSorter:
return arr
"""
fto_path.write_text(optimized_code_mutated_attr, "utf-8")
instrument_code(function_to_optimize)
instrument_code(function_to_optimize, {})
test_results_mutated_attr, coverage_data = opt.run_and_parse_tests(
testing_type=TestingMode.BEHAVIOR,
test_env=test_env,
@ -418,7 +418,7 @@ class BubbleSorter:
return arr
"""
fto_path.write_text(optimized_code_new_attr, "utf-8")
instrument_code(function_to_optimize)
instrument_code(function_to_optimize, {})
test_results_new_attr, coverage_data = opt.run_and_parse_tests(
testing_type=TestingMode.BEHAVIOR,
test_env=test_env,

8
tests/test_instrumentation_results_cli.py
View file

@ -37,7 +37,7 @@ codeflash_wrap_string = """def codeflash_wrap(wrapped, test_module_name, test_cl
exception = e
gc.enable()
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_con.commit()
if exception:
raise exception
@ -77,7 +77,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
input = [5, 4, 3, 2, 1, 0]
output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input)
assert output == [0, 1, 2, 3, 4, 5]
@ -229,7 +229,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi
exception = e
gc.enable()
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_con.commit()
if exception:
raise exception
@ -241,7 +241,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
input = [5, 4, 3, 2, 1, 0]
sort_class = BubbleSorter()
output = codeflash_wrap(sort_class.sorter, '{module_path}', None, 'test_sort', 'sorter', '2', codeflash_loop_index, codeflash_cur, codeflash_con, input)