From 741f95ad0a58938a9e217da0afcf85b311247cc4 Mon Sep 17 00:00:00 2001 From: Alvin Ryanputra Date: Thu, 23 Jan 2025 09:10:32 -0800 Subject: [PATCH] Changed verification type names, improved tests --- .../code_utils/instrument_existing_tests.py | 2 +- codeflash/verification/codeflash_capture.py | 2 +- codeflash/verification/equivalence.py | 6 +-- codeflash/verification/parse_test_output.py | 5 ++- codeflash/verification/test_results.py | 19 ++++++---- tests/test_code_replacement.py | 2 +- tests/test_codeflash_capture.py | 16 ++++---- tests/test_instrumentation.py | 38 +++++++++---------- .../test_instrumentation_results_aiservice.py | 8 ++-- tests/test_instrumentation_results_cli.py | 8 ++-- 10 files changed, 56 insertions(+), 50 deletions(-) diff --git a/codeflash/code_utils/instrument_existing_tests.py b/codeflash/code_utils/instrument_existing_tests.py index 237004876..032116da7 100644 --- a/codeflash/code_utils/instrument_existing_tests.py +++ b/codeflash/code_utils/instrument_existing_tests.py @@ -696,7 +696,7 @@ def create_wrapper_function(mode: TestingMode = TestingMode.BEHAVIOR) -> ast.Fun ast.Name(id="invocation_id", ctx=ast.Load()), ast.Name(id="codeflash_duration", ctx=ast.Load()), ast.Name(id="pickled_return_value", ctx=ast.Load()), - ast.Constant(value=VerificationType.FUNCTION_TO_OPTIMIZE), + ast.Constant(value=VerificationType.FUNCTION_CALL.value), ], ctx=ast.Load(), ), diff --git a/codeflash/verification/codeflash_capture.py b/codeflash/verification/codeflash_capture.py index 360f781f4..a85d2f9da 100644 --- a/codeflash/verification/codeflash_capture.py +++ b/codeflash/verification/codeflash_capture.py @@ -116,7 +116,7 @@ def codeflash_capture(function_name: str, tmp_dir_path: str, is_fto: bool = Fals invocation_id, codeflash_duration, pickled_return_value, - VerificationType.INSTANCE_STATE_FTO if is_fto else VerificationType.INSTANCE_STATE_HELPER, + VerificationType.INIT_STATE_FTO if is_fto else VerificationType.INIT_STATE_HELPER, ), ) codeflash_con.commit() diff --git a/codeflash/verification/equivalence.py b/codeflash/verification/equivalence.py index 1f8ab9d94..b3f07f9b0 100644 --- a/codeflash/verification/equivalence.py +++ b/codeflash/verification/equivalence.py @@ -26,7 +26,7 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR # If helper function instance_state verification is not present, that's ok. continue if ( original_test_result.verification_type - and original_test_result.verification_type == VerificationType.INSTANCE_STATE_HELPER + and original_test_result.verification_type == VerificationType.INIT_STATE_HELPER and cdd_test_result is None ): continue @@ -38,8 +38,8 @@ def compare_test_results(original_results: TestResults, candidate_results: TestR continue superset_obj = False if original_test_result.verification_type and ( - original_test_result.verification_type == VerificationType.INSTANCE_STATE_HELPER - or original_test_result.verification_type == VerificationType.INSTANCE_STATE_FTO + original_test_result.verification_type + in (VerificationType.INIT_STATE_HELPER, VerificationType.INIT_STATE_FTO) ): superset_obj = True if not comparator(original_test_result.return_value, cdd_test_result.return_value, superset_obj=superset_obj): diff --git a/codeflash/verification/parse_test_output.py b/codeflash/verification/parse_test_output.py index 10fe01d05..be33b9b61 100644 --- a/codeflash/verification/parse_test_output.py +++ b/codeflash/verification/parse_test_output.py @@ -92,7 +92,7 @@ def parse_test_return_values_bin(file_location: Path, test_files: TestFiles, tes test_type=test_type, return_value=test_pickle, timed_out=False, - verification_type=VerificationType.FUNCTION_TO_OPTIMIZE, + verification_type=VerificationType.FUNCTION_CALL, ) ) return test_results @@ -120,7 +120,8 @@ def parse_sqlite_test_results(sqlite_file_path: Path, test_files: TestFiles, tes test_function_name = val[2] if val[2] else None function_getting_tested = val[3] test_file_path = file_path_from_module_name(test_module_path, test_config.tests_project_rootdir) - test_type = test_files.get_test_type_by_instrumented_file_path(test_file_path) # may not be the best way + test_type = (test_files.get_test_type_by_instrumented_file_path(test_file_path) or + test_files.get_test_type_by_original_file_path(test_file_path)) loop_index = val[4] iteration_id = val[5] runtime = val[6] diff --git a/codeflash/verification/test_results.py b/codeflash/verification/test_results.py index d8265dc30..b0e00830d 100644 --- a/codeflash/verification/test_results.py +++ b/codeflash/verification/test_results.py @@ -14,13 +14,18 @@ from codeflash.verification.comparator import comparator class VerificationType(str, Enum): - FUNCTION_TO_OPTIMIZE = ( - "function_to_optimize" # Correctness verification for fto, checks input values and output values - ) - INSTANCE_STATE_FTO = "instance_state_fto" # Correctness verification for fto class instance attributes after init - INSTANCE_STATE_HELPER = ( - "instance_state_helper" # Correctness verification for helper class instance attributes after init + # FUNCTION_TO_OPTIMIZE = ( + # "function_test" # Correctness verification for a test function, checks input values and output values + # ) + # INSTANCE_STATE_FTO = "instance_state_fto" # Correctness verification for fto class instance attributes after init + # INSTANCE_STATE_HELPER = ( + # "instance_state_helper" # Correctness verification for helper class instance attributes after init + # ) + FUNCTION_CALL = ( + "function_call" # Correctness verification for a test function, checks input values and output values) ) + INIT_STATE_FTO = "init_state_fto" # Correctness verification for fto class instance attributes after init + INIT_STATE_HELPER = "init_state_helper" # Correctness verification for helper class instance attributes after init def __new__(cls, value: str) -> VerificationType | None: obj = str.__new__(cls, value) @@ -89,7 +94,7 @@ class FunctionTestInvocation: test_type: TestType return_value: Optional[object] # The return value of the function invocation timed_out: Optional[bool] - verification_type: str = VerificationType.FUNCTION_TO_OPTIMIZE + verification_type: str = VerificationType.FUNCTION_CALL @property def unique_invocation_loop_id(self) -> str: diff --git a/tests/test_code_replacement.py b/tests/test_code_replacement.py index 5d0298da5..f1801fe20 100644 --- a/tests/test_code_replacement.py +++ b/tests/test_code_replacement.py @@ -1565,7 +1565,7 @@ print("Hello world") class NewClass: def __init__(self, name): - self.name = name + self.name = str(name) def __call__(self, value): return "I am still old" def new_function2(value): diff --git a/tests/test_codeflash_capture.py b/tests/test_codeflash_capture.py index d39d79571..7be9e0084 100644 --- a/tests/test_codeflash_capture.py +++ b/tests/test_codeflash_capture.py @@ -623,13 +623,13 @@ class AnotherHelperClass: assert len(test_results.test_results) == 4 assert test_results[0].id.test_function_name == "test_helper_classes" assert test_results[0].id.function_getting_tested == "MyClass.__init__" - assert test_results[0].verification_type == VerificationType.INSTANCE_STATE_FTO + assert test_results[0].verification_type == VerificationType.INIT_STATE_FTO assert test_results[1].id.function_getting_tested == "HelperClass1.__init__" - assert test_results[1].verification_type == VerificationType.INSTANCE_STATE_HELPER + assert test_results[1].verification_type == VerificationType.INIT_STATE_HELPER assert test_results[2].id.function_getting_tested == "HelperClass2.__init__" - assert test_results[2].verification_type == VerificationType.INSTANCE_STATE_HELPER + assert test_results[2].verification_type == VerificationType.INIT_STATE_HELPER assert test_results[3].id.function_getting_tested == "AnotherHelperClass.__init__" - assert test_results[3].verification_type == VerificationType.INSTANCE_STATE_HELPER + assert test_results[3].verification_type == VerificationType.INIT_STATE_HELPER finally: test_path.unlink(missing_ok=True) @@ -777,13 +777,13 @@ class AnotherHelperClass: assert len(test_results.test_results) == 4 assert test_results[0].id.test_function_name == "test_helper_classes" assert test_results[0].id.function_getting_tested == "MyClass.__init__" - assert test_results[0].verification_type == VerificationType.INSTANCE_STATE_FTO + assert test_results[0].verification_type == VerificationType.INIT_STATE_FTO assert test_results[1].id.function_getting_tested == "HelperClass1.__init__" - assert test_results[1].verification_type == VerificationType.INSTANCE_STATE_HELPER + assert test_results[1].verification_type == VerificationType.INIT_STATE_HELPER assert test_results[2].id.function_getting_tested == "HelperClass2.__init__" - assert test_results[2].verification_type == VerificationType.INSTANCE_STATE_HELPER + assert test_results[2].verification_type == VerificationType.INIT_STATE_HELPER assert test_results[3].id.function_getting_tested == "AnotherHelperClass.__init__" - assert test_results[3].verification_type == VerificationType.INSTANCE_STATE_HELPER + assert test_results[3].verification_type == VerificationType.INIT_STATE_HELPER # Now, let's say we optimize the code and make changes. new_fto_code = """ diff --git a/tests/test_instrumentation.py b/tests/test_instrumentation.py index 4a80cd0c7..49439c063 100644 --- a/tests/test_instrumentation.py +++ b/tests/test_instrumentation.py @@ -40,7 +40,7 @@ codeflash_wrap_string = """def codeflash_wrap(wrapped, test_module_name, test_cl exception = e gc.enable() pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) - codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value)) + codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() if exception: raise exception @@ -132,7 +132,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi exception = e gc.enable() pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) - codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value)) + codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() if exception: raise exception @@ -146,7 +146,7 @@ class TestPigLatin(unittest.TestCase): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') input = [5, 4, 3, 2, 1, 0] output = codeflash_wrap(sorter, '{module_path}', 'TestPigLatin', 'test_sort', 'sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input) self.assertEqual(output, [0, 1, 2, 3, 4, 5]) @@ -233,7 +233,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi exception = e gc.enable() pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) - codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value)) + codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() if exception: raise exception @@ -244,7 +244,7 @@ def test_prepare_image_for_yolo(): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') """ if sys.version_info < (3, 11): expected += """ for (arg_val_pkl, return_val_pkl) in get_next_arg_and_return('/home/saurabh/packagename/traces/first.trace', 3): @@ -307,7 +307,7 @@ def test_sort(): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') input = [5, 4, 3, 2, 1, 0] output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input) assert output == [0, 1, 2, 3, 4, 5] @@ -529,7 +529,7 @@ def test_sort_parametrized(input, expected_output): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort_parametrized', 'sorter', '0', codeflash_loop_index, codeflash_cur, codeflash_con, input) assert output == expected_output codeflash_con.close() @@ -755,7 +755,7 @@ def test_sort_parametrized_loop(input, expected_output): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') for i in range(2): output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort_parametrized_loop', 'sorter', '0_0', codeflash_loop_index, codeflash_cur, codeflash_con, input) assert output == expected_output @@ -1063,7 +1063,7 @@ def test_sort(): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') inputs = [[5, 4, 3, 2, 1, 0], [5.0, 4.0, 3.0, 2.0, 1.0, 0.0], list(reversed(range(50)))] expected_outputs = [[0, 1, 2, 3, 4, 5], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], list(range(50))] for i in range(3): @@ -1315,7 +1315,7 @@ class TestPigLatin(unittest.TestCase): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') input = [5, 4, 3, 2, 1, 0] output = codeflash_wrap(sorter, '{module_path}', 'TestPigLatin', 'test_sort', 'sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input) self.assertEqual(output, [0, 1, 2, 3, 4, 5]) @@ -1586,7 +1586,7 @@ class TestPigLatin(unittest.TestCase): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') output = codeflash_wrap(sorter, '{module_path}', 'TestPigLatin', 'test_sort', 'sorter', '0', codeflash_loop_index, codeflash_cur, codeflash_con, input) self.assertEqual(output, expected_output) codeflash_con.close() @@ -1834,7 +1834,7 @@ class TestPigLatin(unittest.TestCase): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') inputs = [[5, 4, 3, 2, 1, 0], [5.0, 4.0, 3.0, 2.0, 1.0, 0.0], list(reversed(range(50)))] expected_outputs = [[0, 1, 2, 3, 4, 5], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], list(range(50))] for i in range(3): @@ -2094,7 +2094,7 @@ class TestPigLatin(unittest.TestCase): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') for i in range(2): output = codeflash_wrap(sorter, '{module_path}', 'TestPigLatin', 'test_sort', 'sorter', '0_0', codeflash_loop_index, codeflash_cur, codeflash_con, input) self.assertEqual(output, expected_output) @@ -2422,7 +2422,7 @@ def test_class_name_A_function_name(): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') ret = codeflash_wrap(class_name_A.function_name, '{module_path}', None, 'test_class_name_A_function_name', 'class_name_A.function_name', '0', codeflash_loop_index, codeflash_cur, codeflash_con, **args) codeflash_con.close() """ @@ -2491,7 +2491,7 @@ def test_common_tags_1(): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') articles_1 = [1, 2, 3] assert codeflash_wrap(find_common_tags, '{module_path}', None, 'test_common_tags_1', 'find_common_tags', '1', codeflash_loop_index, codeflash_cur, codeflash_con, articles_1) == set(1, 2) articles_2 = [1, 2] @@ -2558,7 +2558,7 @@ def test_sort(): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') input = [5, 4, 3, 2, 1, 0] if len(input) > 0: assert codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '1_0', codeflash_loop_index, codeflash_cur, codeflash_con, input) == [0, 1, 2, 3, 4, 5] @@ -2625,7 +2625,7 @@ def test_sort(): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') input = [5, 4, 3, 2, 1, 0] output = codeflash_wrap(BubbleSorter.sorter, 'tests.pytest.test_perfinjector_bubble_sort_results_temp', None, 'test_sort', 'BubbleSorter.sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input) assert output == [0, 1, 2, 3, 4, 5] @@ -2745,7 +2745,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi exception = e gc.enable() pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) - codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value)) + codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() if exception: raise exception @@ -2756,7 +2756,7 @@ def test_code_replacement10() -> None: codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') get_code_output = 'random code' file_path = Path(__file__).resolve() opt = Optimizer(Namespace(project_root=str(file_path.parent.resolve()), disable_telemetry=True, tests_root='tests', test_framework='pytest', pytest_cmd='pytest', experiment_id=None)) diff --git a/tests/test_instrumentation_results_aiservice.py b/tests/test_instrumentation_results_aiservice.py index 32f6adf52..0d473241d 100644 --- a/tests/test_instrumentation_results_aiservice.py +++ b/tests/test_instrumentation_results_aiservice.py @@ -317,7 +317,7 @@ def test_single_element_list(): with test_path.open("w") as f: f.write(instrumented_behavior_test_source) # Add codeflash capture decorator - instrument_code(function_to_optimize) + instrument_code(function_to_optimize, {}) opt = Optimizer( Namespace( project_root=project_root_path, @@ -353,7 +353,7 @@ def test_single_element_list(): testing_time=0.1, ) # Verify instance_state result, which checks instance state right after __init__, using codeflash_capture - assert test_results[0].id.function_getting_tested == "sorter" + assert test_results[0].id.function_getting_tested == "BubbleSorter.__init__" assert test_results[0].id.test_function_name == "test_single_element_list" assert test_results[0].did_pass assert test_results[0].return_value[0] == {"x": 0} @@ -387,7 +387,7 @@ class BubbleSorter: return arr """ fto_path.write_text(optimized_code_mutated_attr, "utf-8") - instrument_code(function_to_optimize) + instrument_code(function_to_optimize, {}) test_results_mutated_attr, coverage_data = opt.run_and_parse_tests( testing_type=TestingMode.BEHAVIOR, test_env=test_env, @@ -418,7 +418,7 @@ class BubbleSorter: return arr """ fto_path.write_text(optimized_code_new_attr, "utf-8") - instrument_code(function_to_optimize) + instrument_code(function_to_optimize, {}) test_results_new_attr, coverage_data = opt.run_and_parse_tests( testing_type=TestingMode.BEHAVIOR, test_env=test_env, diff --git a/tests/test_instrumentation_results_cli.py b/tests/test_instrumentation_results_cli.py index ce0b57b86..41dcbb181 100644 --- a/tests/test_instrumentation_results_cli.py +++ b/tests/test_instrumentation_results_cli.py @@ -37,7 +37,7 @@ codeflash_wrap_string = """def codeflash_wrap(wrapped, test_module_name, test_cl exception = e gc.enable() pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) - codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value)) + codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() if exception: raise exception @@ -77,7 +77,7 @@ def test_sort(): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') input = [5, 4, 3, 2, 1, 0] output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort', 'sorter', '1', codeflash_loop_index, codeflash_cur, codeflash_con, input) assert output == [0, 1, 2, 3, 4, 5] @@ -229,7 +229,7 @@ def codeflash_wrap(wrapped, test_module_name, test_class_name, test_name, functi exception = e gc.enable() pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value) - codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value)) + codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (test_module_name, test_class_name, test_name, function_name, loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call')) codeflash_con.commit() if exception: raise exception @@ -241,7 +241,7 @@ def test_sort(): codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION'] codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite') codeflash_cur = codeflash_con.cursor() - codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB)') + codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)') input = [5, 4, 3, 2, 1, 0] sort_class = BubbleSorter() output = codeflash_wrap(sort_class.sorter, '{module_path}', None, 'test_sort', 'sorter', '2', codeflash_loop_index, codeflash_cur, codeflash_con, input)