codeflash-admin/codeflash-agent
1
0
Fork 0
mirror of https://github.com/codeflash-ai/codeflash-agent.git synced 2026-05-04 18:25:19 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

feat: add dual-clock instrumentation (wall + CPU time) and remove dead binary parser

Browse source 
Measure both wall-clock time (perf_counter_ns) and CPU thread time
(thread_time_ns) in instrumented test code. cpu_runtime is now a required
int field on FunctionTestInvocation, stored in the SQLite test_results
table as a 10th column.

Also fixes the sleeptime.py bug (10e9 → 1e9 divisor) and removes the
binary pickle parser (parse_test_return_values_bin) since no writer
exists in the current codebase — SQLite is the sole data capture path.
This commit is contained in:
Kevin Turcios 2026-04-24 02:21:22 -05:00
parent 0c622ac469
commit eb6a0be717
27 changed files with 626 additions and 165 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
packages/codeflash-python/src/codeflash_python/runtime/_codeflash_capture.py
View file

@ -179,7 +179,7 @@ def codeflash_capture(
and not callable(getattr(instance, attr, None))
}
codeflash_cur.execute(
"CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)"
"CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)"
)
# Write to sqlite
@ -189,7 +189,7 @@ def codeflash_capture(
else PicklePatcher.dumps(instance_state)
)
codeflash_cur.execute(
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
test_module_name,
test_class_name,
@ -202,6 +202,7 @@ def codeflash_capture(
VerificationType.INIT_STATE_FTO
if is_fto
else VerificationType.INIT_STATE_HELPER,
0,
),
)
codeflash_con.commit()

5
packages/codeflash-python/src/codeflash_python/runtime/_codeflash_wrap_decorator.py
View file

@ -91,7 +91,7 @@ def codeflash_behavior_async(func: F) -> F:
codeflash_cur.execute(
"CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, "
"test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, "
"runtime INTEGER, return_value BLOB, verification_type TEXT)"
"runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)"
)
exception = None
@ -119,7 +119,7 @@ def codeflash_behavior_async(func: F) -> F:
else pickle.dumps((args, kwargs, return_value))
)
codeflash_cur.execute(
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
test_module_name,
test_class_name,
@ -130,6 +130,7 @@ def codeflash_behavior_async(func: F) -> F:
codeflash_duration,
pickled_return_value,
VerificationType.FUNCTION_CALL.value,
0,
),
)
codeflash_con.commit()

99
packages/codeflash-python/src/codeflash_python/testing/_data_parsers.py
View file

@ -1,4 +1,4 @@
"""SQLite and binary pickle test result parsing."""
"""SQLite test result parsing."""
from __future__ import annotations
@ -42,7 +42,8 @@ def parse_sqlite_test_results(
" test_function_name,"
" function_getting_tested, loop_index,"
" iteration_id, runtime,"
" return_value, verification_type"
" return_value, verification_type,"
" cpu_runtime"
" FROM test_results"
).fetchall()
except sqlite3.Error:
@ -99,6 +100,7 @@ def _process_sqlite_row_inner(
iteration_id = val[5]
runtime = val[6]
verification_type = val[8]
cpu_runtime = val[9]
test_file_path = file_path_from_module_name(
test_module_path, # type: ignore[arg-type]
@ -156,6 +158,7 @@ def _process_sqlite_row_inner(
test_framework=test_config.test_framework,
test_type=test_type,
return_value=ret_val,
cpu_runtime=cpu_runtime, # type: ignore[arg-type]
timed_out=False,
verification_type=(
VerificationType(verification_type)
@ -164,95 +167,3 @@ def _process_sqlite_row_inner(
),
),
)
def parse_test_return_values_bin(
file_location: Path,
test_files: TestFiles,
test_config: TestConfig,
) -> TestResults:
"""Parse test results from a binary pickle file."""
import dill as pickle # noqa: PLC0415
test_results = TestResults()
if not file_location.exists():
log.debug("No test results for %s found.", file_location)
return test_results
with file_location.open("rb") as fh:
try:
while True:
len_next_bytes = fh.read(4)
if not len_next_bytes:
break
len_next = int.from_bytes(len_next_bytes, byteorder="big")
encoded_test_bytes = fh.read(len_next)
encoded_test_name = encoded_test_bytes.decode("ascii")
duration_bytes = fh.read(8)
duration = int.from_bytes(duration_bytes, byteorder="big")
len_next_bytes = fh.read(4)
len_next = int.from_bytes(len_next_bytes, byteorder="big")
test_pickle_bin = fh.read(len_next)
loop_index_bytes = fh.read(8)
loop_index = int.from_bytes(loop_index_bytes, byteorder="big")
len_next_bytes = fh.read(4)
len_next = int.from_bytes(len_next_bytes, byteorder="big")
invocation_id_bytes = fh.read(len_next)
invocation_id = invocation_id_bytes.decode("ascii")
invocation_id_object = InvocationId.from_str_id(
encoded_test_name, invocation_id
)
test_file_path = file_path_from_module_name(
invocation_id_object.test_module_path,
test_config.tests_project_rootdir,
)
test_type = test_files.get_test_type_by_instrumented_file_path(
test_file_path,
)
try:
test_pickle = (
pickle.loads( # noqa: S301
test_pickle_bin,
)
if loop_index == 1
else None
)
except Exception: # noqa: BLE001
log.debug(
"Failed to deserialize pickle for %s",
encoded_test_name,
exc_info=True,
)
continue
if test_type is None:
log.debug(
"Test type not found for %s, skipping.",
test_file_path,
)
continue
test_results.add(
FunctionTestInvocation(
loop_index=loop_index,
id=invocation_id_object,
file_name=test_file_path,
did_pass=True,
runtime=duration,
test_framework=(test_config.test_framework),
test_type=test_type,
return_value=test_pickle,
timed_out=False,
verification_type=(VerificationType.FUNCTION_CALL),
),
)
except Exception: # noqa: BLE001
log.warning(
"Failed to parse test results from %s.",
file_location,
exc_info=True,
)
return test_results

6
packages/codeflash-python/src/codeflash_python/testing/_instrument_async.py
View file

@ -382,7 +382,8 @@ def codeflash_behavior_async(func):
" iteration_id TEXT,"
" runtime INTEGER,"
" return_value BLOB,"
" verification_type TEXT)"
" verification_type TEXT,"
" cpu_runtime INTEGER)"
)
exception = None
counter = loop.time()
@ -409,7 +410,7 @@ def codeflash_behavior_async(func):
)
)
codeflash_cur.execute(
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
test_module_name,
test_class_name,
@ -420,6 +421,7 @@ def codeflash_behavior_async(func):
codeflash_duration,
pickled_return_value,
"function_call",
0,
),
)
codeflash_con.commit()

77
packages/codeflash-python/src/codeflash_python/testing/_instrument_core.py
View file

@ -874,6 +874,21 @@ def create_wrapper_function(
),
lineno=lineno + 11,
),
ast.Assign(
targets=[
ast.Name(id="cpu_counter", ctx=ast.Store()),
],
value=ast.Call(
func=ast.Attribute(
value=ast.Name(id="time", ctx=ast.Load()),
attr="thread_time_ns",
ctx=ast.Load(),
),
args=[],
keywords=[],
),
lineno=lineno + 11,
),
ast.Assign(
targets=[
ast.Name(
@ -938,6 +953,34 @@ def create_wrapper_function(
),
lineno=lineno + 13,
),
ast.Assign(
targets=[
ast.Name(
id="codeflash_cpu_duration",
ctx=ast.Store(),
)
],
value=ast.BinOp(
left=ast.Call(
func=ast.Attribute(
value=ast.Name(
id="time",
ctx=ast.Load(),
),
attr="thread_time_ns",
ctx=ast.Load(),
),
args=[],
keywords=[],
),
op=ast.Sub(),
right=ast.Name(
id="cpu_counter",
ctx=ast.Load(),
),
),
lineno=lineno + 13,
),
],
handlers=[
ast.ExceptHandler(
@ -972,6 +1015,34 @@ def create_wrapper_function(
),
lineno=lineno + 15,
),
ast.Assign(
targets=[
ast.Name(
id="codeflash_cpu_duration",
ctx=ast.Store(),
)
],
value=ast.BinOp(
left=ast.Call(
func=ast.Attribute(
value=ast.Name(
id="time",
ctx=ast.Load(),
),
attr="thread_time_ns",
ctx=ast.Load(),
),
args=[],
keywords=[],
),
op=ast.Sub(),
right=ast.Name(
id="cpu_counter",
ctx=ast.Load(),
),
),
lineno=lineno + 15,
),
ast.Assign(
targets=[
ast.Name(
@ -1102,7 +1173,7 @@ def create_wrapper_function(
),
args=[
ast.Constant(
value="INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
value="INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"
),
ast.Tuple(
elts=[
@ -1141,6 +1212,10 @@ def create_wrapper_function(
ast.Constant(
value=VerificationType.FUNCTION_CALL.value
),
ast.Name(
id="codeflash_cpu_duration",
ctx=ast.Load(),
),
],
ctx=ast.Load(),
),

3
packages/codeflash-python/src/codeflash_python/testing/_instrumentation.py
View file

@ -587,7 +587,8 @@ class InjectPerfOnly(ast.NodeTransformer):
ast.Constant(
value="CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT,"
" test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT,"
" loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)"
" loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB,"
" verification_type TEXT, cpu_runtime INTEGER)"
)
],
keywords=[],

18
packages/codeflash-python/src/codeflash_python/testing/_parse_results.py
View file

@ -1,6 +1,6 @@
"""Top-level test result orchestrator.
Coordinates XML, SQLite, binary, and merge parsers to produce
Coordinates XML, SQLite, and merge parsers to produce
a unified ``TestResults`` object.
"""
@ -11,10 +11,7 @@ from pathlib import Path
from typing import TYPE_CHECKING
from ..runtime._codeflash_wrap_decorator import get_run_tmp_file
from ._data_parsers import (
parse_sqlite_test_results,
parse_test_return_values_bin,
)
from ._data_parsers import parse_sqlite_test_results
from ._result_merger import merge_test_results
from ._stdout_parsers import parse_test_failures_from_stdout
from ._xml_parser import parse_test_xml
@ -53,18 +50,15 @@ def parse_test_results(
sql_file, test_files, test_config
)
# Parse binary pickle results
# Clean up deprecated binary pickle file if present
bin_file = get_run_tmp_file(
Path(f"test_return_values_{optimization_iteration}.bin"),
)
if bin_file.exists():
bin_results = parse_test_return_values_bin(
bin_file, test_files, test_config
log.debug(
"Found deprecated .bin result file %s, removing.",
bin_file,
)
for result in bin_results:
data_results.add(result)
# Cleanup temp files
bin_file.unlink(missing_ok=True)
sql_file.unlink(missing_ok=True)
get_run_tmp_file(Path("pytest_results.xml")).unlink(

3
packages/codeflash-python/src/codeflash_python/testing/_result_merger.py
View file

@ -85,6 +85,7 @@ def _merge_single_xml(
did_pass=xml_result.did_pass,
test_type=xml_result.test_type,
return_value=data_result.return_value,
cpu_runtime=data_result.cpu_runtime,
timed_out=xml_result.timed_out,
verification_type=(
VerificationType(data_result.verification_type)
@ -120,6 +121,7 @@ def _merge_by_iteration_id(
did_pass=data_result.did_pass,
test_type=xml_result.test_type,
return_value=data_result.return_value,
cpu_runtime=data_result.cpu_runtime,
timed_out=(
xml_result.timed_out if merged_runtime is None else False
),
@ -161,6 +163,7 @@ def _merge_by_index(
did_pass=data_result.did_pass,
test_type=data_result.test_type,
return_value=data_result.return_value,
cpu_runtime=data_result.cpu_runtime,
timed_out=xml_result.timed_out,
verification_type=(
VerificationType(data_result.verification_type)

2
packages/codeflash-python/src/codeflash_python/testing/_xml_parser.py
View file

@ -272,6 +272,7 @@ def parse_test_xml( # noqa: C901, PLR0912, PLR0915
did_pass=result,
test_type=test_type,
return_value=None,
cpu_runtime=0,
timed_out=timed_out,
stdout="",
),
@ -363,6 +364,7 @@ def _parse_begin_matches( # noqa: PLR0913
did_pass=result,
test_type=test_type,
return_value=None,
cpu_runtime=0,
timed_out=timed_out,
stdout=stdout,
),

1
packages/codeflash-python/src/codeflash_python/testing/models.py
View file

@ -142,6 +142,7 @@ class FunctionTestInvocation:
test_framework: str
test_type: TestType
return_value: object | None
cpu_runtime: int
timed_out: bool | None
verification_type: str | None = VerificationType.FUNCTION_CALL
stdout: str | None = None

2
packages/codeflash-python/tests/code_to_optimize/sleeptime.py
View file

@ -5,6 +5,6 @@ def accurate_sleepfunc(t) -> float:
"""T is in seconds"""
start_time = time.perf_counter_ns()
while True:
if (time.perf_counter_ns() - start_time) / 10e9 >= t:
if (time.perf_counter_ns() - start_time) / 1e9 >= t:
break
return t

1
packages/codeflash-python/tests/test_add_runtime_comments.py
View file

@ -61,6 +61,7 @@ class TestAddRuntimeComments:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
verification_type=VerificationType.FUNCTION_CALL,
)

3
packages/codeflash-python/tests/test_baseline.py
View file

@ -391,6 +391,7 @@ def _make_test_results(
test_framework="pytest",
test_type=test_type,
return_value=None,
cpu_runtime=0,
timed_out=False,
),
)
@ -744,6 +745,7 @@ class TestEstablishOriginalCodeBaseline:
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
),
)
@ -763,6 +765,7 @@ class TestEstablishOriginalCodeBaseline:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
),
)

2
packages/codeflash-python/tests/test_benchmark_merge_test_results.py
View file

@ -32,6 +32,7 @@ def generate_test_invocations(count=100):
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=i,
)
@ -53,6 +54,7 @@ def generate_test_invocations(count=100):
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=i,
)

1
packages/codeflash-python/tests/test_candidate_eval.py
View file

@ -60,6 +60,7 @@ def _make_invocation(
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=42,
cpu_runtime=0,
timed_out=False,
)

10
packages/codeflash-python/tests/test_comparator.py
View file

@ -2963,6 +2963,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=5,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -2984,6 +2985,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=5,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -3008,6 +3010,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=[5],
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -3032,6 +3035,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=5,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -3051,6 +3055,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=5,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -3075,6 +3080,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=5,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -3099,6 +3105,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.GENERATED_REGRESSION,
return_value=5,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -3120,6 +3127,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.GENERATED_REGRESSION,
return_value=5,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -3144,6 +3152,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.REPLAY_TEST,
return_value=5,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -3165,6 +3174,7 @@ def test_compare_results_fn():
test_framework="unittest",
test_type=TestType.REPLAY_TEST,
return_value=5,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)

355
packages/codeflash-python/tests/test_cpu_runtime.py Normal file
View file

@ -0,0 +1,355 @@
from __future__ import annotations
import sqlite3
from pathlib import Path
import pytest
from codeflash_python.test_discovery.models import TestType
from codeflash_python.testing._result_merger import (
_merge_by_index,
_merge_by_iteration_id,
_merge_single_xml,
merge_test_results,
)
from codeflash_python.testing.models import (
FunctionTestInvocation,
InvocationId,
TestConfig,
TestFile,
TestFiles,
TestResults,
)
def _make_inv_id(
*,
module: str = "tests.test_example",
cls: str | None = "TestExample",
func: str | None = "test_run",
target: str = "run",
iteration: str | None = "0",
) -> InvocationId:
return InvocationId(
test_module_path=module,
test_class_name=cls,
test_function_name=func,
function_getting_tested=target,
iteration_id=iteration,
)
def _make_invocation( # noqa: PLR0913
*,
loop_index: int = 1,
inv_id: InvocationId | None = None,
did_pass: bool = True,
runtime: int | None = 500,
cpu_runtime: int = 400,
test_type: TestType = TestType.EXISTING_UNIT_TEST,
timed_out: bool = False,
) -> FunctionTestInvocation:
return FunctionTestInvocation(
loop_index=loop_index,
id=inv_id or _make_inv_id(),
file_name=Path("tests/test_example.py"),
did_pass=did_pass,
runtime=runtime,
test_framework="pytest",
test_type=test_type,
return_value=None,
cpu_runtime=cpu_runtime,
timed_out=timed_out,
)
class TestFunctionTestInvocationCpuRuntime:
"""cpu_runtime field on FunctionTestInvocation."""
def test_cpu_runtime_required(self) -> None:
with pytest.raises(TypeError):
FunctionTestInvocation(
loop_index=1,
id=_make_inv_id(),
file_name=Path("tests/test_example.py"),
did_pass=True,
runtime=100,
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
# cpu_runtime omitted
timed_out=False,
)
def test_cpu_runtime_zero_is_valid(self) -> None:
inv = _make_invocation(cpu_runtime=0)
assert 0 == inv.cpu_runtime
def test_cpu_runtime_large_value(self) -> None:
inv = _make_invocation(cpu_runtime=5_000_000_000)
assert 5_000_000_000 == inv.cpu_runtime
class TestMergeSingleXmlCpuRuntime:
"""_merge_single_xml preserves data_result's cpu_runtime."""
def test_data_cpu_runtime_flows_through(self) -> None:
xml_result = _make_invocation(
cpu_runtime=0,
runtime=None,
inv_id=_make_inv_id(iteration=None),
)
data_results = TestResults()
data_results.add(
_make_invocation(cpu_runtime=12345, runtime=800),
)
merged = TestResults()
_merge_single_xml(xml_result, data_results, merged)
assert 1 == len(merged)
assert 12345 == merged[0].cpu_runtime
def test_data_zero_cpu_runtime_preserved(self) -> None:
xml_result = _make_invocation(
cpu_runtime=99999,
runtime=None,
inv_id=_make_inv_id(iteration=None),
)
data_results = TestResults()
data_results.add(
_make_invocation(cpu_runtime=0, runtime=500),
)
merged = TestResults()
_merge_single_xml(xml_result, data_results, merged)
assert 1 == len(merged)
assert 0 == merged[0].cpu_runtime
class TestMergeByIterationIdCpuRuntime:
"""_merge_by_iteration_id preserves data_result's cpu_runtime."""
def test_data_cpu_runtime_flows_through(self) -> None:
inv_id = _make_inv_id(iteration="7")
xml_results = TestResults()
xml_results.add(
_make_invocation(cpu_runtime=0, runtime=None, inv_id=inv_id),
)
data_results = TestResults()
data_results.add(
_make_invocation(cpu_runtime=77777, runtime=600, inv_id=inv_id),
)
merged = TestResults()
_merge_by_iteration_id(xml_results, data_results, merged)
assert 1 == len(merged)
assert 77777 == merged[0].cpu_runtime
class TestMergeByIndexCpuRuntime:
"""_merge_by_index preserves data_result's cpu_runtime."""
def test_data_cpu_runtime_flows_through(self) -> None:
xml_results = TestResults()
xml_results.add(
_make_invocation(
cpu_runtime=0,
runtime=None,
inv_id=_make_inv_id(iteration="a"),
),
)
xml_results.add(
_make_invocation(
cpu_runtime=0,
runtime=None,
inv_id=_make_inv_id(iteration="b"),
),
)
data_results = TestResults()
data_results.add(
_make_invocation(
cpu_runtime=111,
runtime=300,
inv_id=_make_inv_id(iteration="a"),
),
)
data_results.add(
_make_invocation(
cpu_runtime=222,
runtime=400,
inv_id=_make_inv_id(iteration="b"),
),
)
merged = TestResults()
_merge_by_index(xml_results, data_results, merged)
assert 2 == len(merged)
assert 111 == merged[0].cpu_runtime
assert 222 == merged[1].cpu_runtime
class TestSqliteParserCpuRuntime:
"""parse_sqlite_test_results extracts cpu_runtime from the database."""
def test_cpu_runtime_from_sqlite(self, tmp_path: Path) -> None:
from codeflash_python.testing._data_parsers import (
parse_sqlite_test_results,
)
test_file = tmp_path / "tests" / "test_example.py"
test_file.parent.mkdir(parents=True)
test_file.write_text("def test_run(): pass\n")
db_path = tmp_path / "results.sqlite"
db = sqlite3.connect(db_path)
db.execute(
"CREATE TABLE test_results ("
" test_module_path TEXT,"
" test_class_name TEXT,"
" test_function_name TEXT,"
" function_getting_tested TEXT,"
" loop_index INTEGER,"
" iteration_id TEXT,"
" runtime INTEGER,"
" return_value BLOB,"
" verification_type TEXT,"
" cpu_runtime INTEGER"
")"
)
db.execute(
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
"tests.test_example",
None,
"test_run",
"run",
1,
"0",
500,
None,
"function_call",
98765,
),
)
db.commit()
db.close()
test_files = TestFiles(
test_files=[
TestFile(
original_file_path=test_file,
test_type=TestType.EXISTING_UNIT_TEST,
),
],
)
test_config = TestConfig(tests_project_rootdir=tmp_path)
results = parse_sqlite_test_results(db_path, test_files, test_config)
assert 1 == len(results)
assert 98765 == results[0].cpu_runtime
class TestEndToEndCpuRuntime:
"""Full pipeline: SQLite -> parse -> merge with XML -> verify cpu_runtime."""
def test_sqlite_cpu_runtime_survives_xml_merge(
self,
tmp_path: Path,
) -> None:
from codeflash_python.testing._data_parsers import (
parse_sqlite_test_results,
)
test_file = tmp_path / "tests" / "test_example.py"
test_file.parent.mkdir(parents=True)
test_file.write_text("def test_run(): pass\n")
db_path = tmp_path / "results.sqlite"
db = sqlite3.connect(db_path)
db.execute(
"CREATE TABLE test_results ("
" test_module_path TEXT,"
" test_class_name TEXT,"
" test_function_name TEXT,"
" function_getting_tested TEXT,"
" loop_index INTEGER,"
" iteration_id TEXT,"
" runtime INTEGER,"
" return_value BLOB,"
" verification_type TEXT,"
" cpu_runtime INTEGER"
")"
)
db.execute(
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
"tests.test_example",
None,
"test_run",
"run",
1,
"0",
750,
None,
"function_call",
42000,
),
)
db.commit()
db.close()
test_files = TestFiles(
test_files=[
TestFile(
original_file_path=test_file,
test_type=TestType.EXISTING_UNIT_TEST,
),
],
)
test_config = TestConfig(tests_project_rootdir=tmp_path)
data_results = parse_sqlite_test_results(
db_path,
test_files,
test_config,
)
assert 1 == len(data_results)
assert 42000 == data_results[0].cpu_runtime
xml_results = TestResults()
xml_results.add(
FunctionTestInvocation(
loop_index=1,
id=InvocationId(
test_module_path="tests.test_example",
test_class_name=None,
test_function_name="test_run",
function_getting_tested="",
iteration_id=None,
),
file_name=test_file,
did_pass=True,
runtime=None,
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
),
)
merged = merge_test_results(
xml_test_results=xml_results,
bin_test_results=data_results,
)
assert 1 == len(merged)
assert 42000 == merged[0].cpu_runtime

7
packages/codeflash-python/tests/test_critic.py
View file

@ -130,6 +130,7 @@ def test_generated_test_critic() -> None:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -148,6 +149,7 @@ def test_generated_test_critic() -> None:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -166,6 +168,7 @@ def test_generated_test_critic() -> None:
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -184,6 +187,7 @@ def test_generated_test_critic() -> None:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -202,6 +206,7 @@ def test_generated_test_critic() -> None:
test_framework="pytest",
test_type=TestType.REPLAY_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -220,6 +225,7 @@ def test_generated_test_critic() -> None:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=2,
)
@ -238,6 +244,7 @@ def test_generated_test_critic() -> None:
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)

16
packages/codeflash-python/tests/test_file_to_no_of_tests.py
View file

@ -39,6 +39,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -64,6 +65,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -89,6 +91,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -115,6 +118,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -142,6 +146,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -178,6 +183,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=test_type,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -213,6 +219,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -243,6 +250,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -273,6 +281,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -304,6 +313,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=test_type,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -326,6 +336,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -348,6 +359,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -383,6 +395,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -416,6 +429,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -450,6 +464,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -482,6 +497,7 @@ class TestFileToNoOfTests:
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)

85
packages/codeflash-python/tests/test_inject_profiling_used_frameworks.py
View file

@ -66,15 +66,18 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
gc.disable()
try:
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -85,7 +88,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -126,19 +129,22 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
elif _codeflash_should_sync_mps:
torch.mps.synchronize()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_cuda:
torch.cuda.synchronize()
elif _codeflash_should_sync_mps:
torch.mps.synchronize()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -149,7 +155,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -190,19 +196,22 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
elif _codeflash_should_sync_mps:
th.mps.synchronize()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_cuda:
th.cuda.synchronize()
elif _codeflash_should_sync_mps:
th.mps.synchronize()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -213,7 +222,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -255,19 +264,22 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
elif _codeflash_should_sync_mps:
torch.mps.synchronize()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_cuda:
torch.cuda.synchronize()
elif _codeflash_should_sync_mps:
torch.mps.synchronize()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -278,7 +290,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -316,17 +328,20 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -337,7 +352,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -375,17 +390,20 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
if _codeflash_should_sync_tf:
tf.test.experimental.sync_devices()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_tf:
tf.test.experimental.sync_devices()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -396,7 +414,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -432,17 +450,20 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
gc.disable()
try:
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_jax:
jax.block_until_ready(return_value)
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -453,7 +474,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -489,17 +510,20 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
gc.disable()
try:
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_jax:
jnp.block_until_ready(return_value)
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -510,7 +534,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -555,6 +579,7 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_cuda:
torch.cuda.synchronize()
@ -563,13 +588,15 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -580,7 +607,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -627,6 +654,7 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_cuda:
torch.cuda.synchronize()
@ -637,13 +665,15 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -654,7 +684,7 @@ def test_my_function():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{CODEFLASH_DB_PATH}')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(my_function).bind(1, 2)
_call__bound__arguments.apply_defaults()
result = codeflash_wrap(my_function, 'test_example', None, 'test_my_function', 'my_function', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -685,10 +715,13 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
gc.disable()
try:
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}:{codeflash_duration}######!')
@ -732,14 +765,17 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
elif _codeflash_should_sync_mps:
torch.mps.synchronize()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_cuda:
torch.cuda.synchronize()
elif _codeflash_should_sync_mps:
torch.mps.synchronize()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}:{codeflash_duration}######!')
@ -780,12 +816,15 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}:{codeflash_duration}######!')
@ -824,12 +863,15 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
gc.disable()
try:
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_jax:
jax.block_until_ready(return_value)
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}:{codeflash_duration}######!')
@ -879,6 +921,7 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
if _codeflash_should_sync_cuda:
torch.cuda.synchronize()
@ -889,8 +932,10 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
if _codeflash_should_sync_tf:
tensorflow.test.experimental.sync_devices()
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{test_stdout_tag}:{codeflash_duration}######!')

13
packages/codeflash-python/tests/test_instrument_all_and_run.py
View file

@ -42,15 +42,18 @@ codeflash_wrap_string = """def codeflash_wrap(codeflash_wrapped, codeflash_test_
gc.disable()
try:
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f"!######{{test_stdout_tag}}######!")
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -112,7 +115,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
input = [5, 4, 3, 2, 1, 0]
_call__bound__arguments = inspect.signature(sorter).bind(input)
_call__bound__arguments.apply_defaults()
@ -282,7 +285,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
input = [5, 4, 3, 2, 1, 0]
sort_class = BubbleSorter()
_call__bound__arguments = inspect.signature(sort_class.sorter).bind(input)
@ -558,7 +561,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
input = [5, 4, 3, 2, 1, 0]
_call__bound__arguments = inspect.signature(BubbleSorter.sorter_classmethod).bind(input)
_call__bound__arguments.apply_defaults()
@ -729,7 +732,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
input = [5, 4, 3, 2, 1, 0]
_call__bound__arguments = inspect.signature(BubbleSorter.sorter_staticmethod).bind(input)
_call__bound__arguments.apply_defaults()

51
packages/codeflash-python/tests/test_instrument_tests.py
View file

@ -52,15 +52,18 @@ codeflash_wrap_string = """def codeflash_wrap(codeflash_wrapped, codeflash_test_
gc.disable()
try:
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f"!######{{test_stdout_tag}}######!")
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -83,10 +86,13 @@ codeflash_wrap_perfonly_string = """def codeflash_wrap(codeflash_wrapped, codefl
gc.disable()
try:
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f"!######{{test_stdout_tag}}:{{codeflash_duration}}######!")
@ -176,7 +182,7 @@ import dill as pickle"""
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
input = [5, 4, 3, 2, 1, 0]
_call__bound__arguments = inspect.signature(sorter).bind(input)
_call__bound__arguments.apply_defaults()
@ -264,15 +270,18 @@ def codeflash_wrap(codeflash_wrapped, codeflash_test_module_name, codeflash_test
gc.disable()
try:
counter = time.perf_counter_ns()
cpu_counter = time.thread_time_ns()
return_value = codeflash_wrapped(*args, **kwargs)
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
except Exception as e:
codeflash_duration = time.perf_counter_ns() - counter
codeflash_cpu_duration = time.thread_time_ns() - cpu_counter
exception = e
gc.enable()
print(f'!######{{test_stdout_tag}}######!')
pickled_return_value = pickle.dumps(exception) if exception else pickle.dumps(return_value)
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call'))
codeflash_cur.execute('INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)', (codeflash_test_module_name, codeflash_test_class_name, codeflash_test_name, codeflash_function_name, codeflash_loop_index, invocation_id, codeflash_duration, pickled_return_value, 'function_call', codeflash_cpu_duration))
codeflash_con.commit()
if exception:
raise exception
@ -283,7 +292,7 @@ def test_prepare_image_for_yolo():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
"""
if sys.version_info < (3, 11):
expected += """ for (arg_val_pkl, return_val_pkl) in get_next_arg_and_return('/home/saurabh/packagename/traces/first.trace', 3):
@ -358,7 +367,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
input = [5, 4, 3, 2, 1, 0]
print(datetime.datetime.now().isoformat())
_call__bound__arguments = inspect.signature(sorter).bind(input)
@ -639,7 +648,7 @@ def test_sort_parametrized(input, expected_output):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(sorter).bind(input)
_call__bound__arguments.apply_defaults()
output = codeflash_wrap(sorter, '{module_path}', None, 'test_sort_parametrized', 'sorter', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -950,7 +959,7 @@ def test_sort_parametrized_loop(input, expected_output):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
for i in range(2):
_call__bound__arguments = inspect.signature(sorter).bind(input)
_call__bound__arguments.apply_defaults()
@ -1245,7 +1254,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
inputs = [[5, 4, 3, 2, 1, 0], [5.0, 4.0, 3.0, 2.0, 1.0, 0.0], list(reversed(range(50)))]
expected_outputs = [[0, 1, 2, 3, 4, 5], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], list(range(50))]
for i in range(3):
@ -1519,7 +1528,7 @@ class TestPigLatin(unittest.TestCase):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
input = [5, 4, 3, 2, 1, 0]
_call__bound__arguments = inspect.signature(sorter).bind(input)
_call__bound__arguments.apply_defaults()
@ -1813,7 +1822,7 @@ class TestPigLatin(unittest.TestCase):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(sorter).bind(input)
_call__bound__arguments.apply_defaults()
output = codeflash_wrap(sorter, '{module_path}', 'TestPigLatin', 'test_sort', 'sorter', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -2069,7 +2078,7 @@ class TestPigLatin(unittest.TestCase):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
inputs = [[5, 4, 3, 2, 1, 0], [5.0, 4.0, 3.0, 2.0, 1.0, 0.0], list(reversed(range(50)))]
expected_outputs = [[0, 1, 2, 3, 4, 5], [0.0, 1.0, 2.0, 3.0, 4.0, 5.0], list(range(50))]
for i in range(3):
@ -2341,7 +2350,7 @@ class TestPigLatin(unittest.TestCase):
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
for i in range(2):
_call__bound__arguments = inspect.signature(sorter).bind(input)
_call__bound__arguments.apply_defaults()
@ -2656,7 +2665,7 @@ def test_class_name_A_function_name():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
_call__bound__arguments = inspect.signature(class_name_A.function_name).bind(**args)
_call__bound__arguments.apply_defaults()
ret = codeflash_wrap(class_name_A.function_name, '{module_path}', None, 'test_class_name_A_function_name', 'class_name_A.function_name', '0', codeflash_loop_index, codeflash_cur, codeflash_con, *_call__bound__arguments.args, **_call__bound__arguments.kwargs)
@ -2729,7 +2738,7 @@ def test_common_tags_1():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
articles_1 = [1, 2, 3]
_call__bound__arguments = inspect.signature(find_common_tags).bind(articles_1)
_call__bound__arguments.apply_defaults()
@ -2809,7 +2818,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
input = [5, 4, 3, 2, 1, 0]
if len(input) > 0:
_call__bound__arguments = inspect.signature(sorter).bind(input)
@ -2886,7 +2895,7 @@ def test_sort():
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
input = [5, 4, 3, 2, 1, 0]
_call__bound__arguments = inspect.signature(BubbleSorter.sorter).bind(input)
_call__bound__arguments.apply_defaults()
@ -3001,7 +3010,7 @@ def test_code_replacement10() -> None:
codeflash_iteration = os.environ['CODEFLASH_TEST_ITERATION']
codeflash_con = sqlite3.connect(f'{tmp_dir_path}_{{codeflash_iteration}}.sqlite')
codeflash_cur = codeflash_con.cursor()
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT)')
codeflash_cur.execute('CREATE TABLE IF NOT EXISTS test_results (test_module_path TEXT, test_class_name TEXT, test_function_name TEXT, function_getting_tested TEXT, loop_index INTEGER, iteration_id TEXT, runtime INTEGER, return_value BLOB, verification_type TEXT, cpu_runtime INTEGER)')
get_code_output = 'random code'
file_path = Path(__file__).resolve()
opt = Optimizer(Namespace(project_root=str(file_path.parent.resolve()), disable_telemetry=True, tests_root='tests', test_framework='pytest', pytest_cmd='pytest', experiment_id=None))
@ -3176,9 +3185,7 @@ def test_sleepfunc_sequence_short(n, expected_total_sleep_time):
for i, test_result in enumerate(test_results.test_results):
assert test_result.did_pass
expected_ns = ((i % 2) + 1) * 100_000_000
assert math.isclose(
test_result.runtime, expected_ns, rel_tol=1.0
)
assert math.isclose(test_result.runtime, expected_ns, rel_tol=1.0)
finally:
test_path.unlink(missing_ok=True)
@ -3330,9 +3337,7 @@ import unittest
for i, test_result in enumerate(test_results.test_results):
assert test_result.did_pass
expected_ns = ((i % 2) + 1) * 100_000_000
assert math.isclose(
test_result.runtime, expected_ns, rel_tol=1.0
)
assert math.isclose(test_result.runtime, expected_ns, rel_tol=1.0)
finally:
test_path.unlink(missing_ok=True)

13
packages/codeflash-python/tests/test_merge_test_results.py
View file

@ -24,6 +24,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -43,6 +44,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -62,6 +64,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -83,6 +86,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -102,6 +106,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -121,6 +126,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -142,6 +148,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -161,6 +168,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -180,6 +188,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -206,6 +215,7 @@ def test_merge_test_results_1():
test_framework="unittest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -250,6 +260,7 @@ def test_merge_test_results_1():
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=None,
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -271,6 +282,7 @@ def test_merge_test_results_1():
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=[2],
cpu_runtime=0,
timed_out=False,
loop_index=1,
)
@ -290,6 +302,7 @@ def test_merge_test_results_1():
test_framework="pytest",
test_type=TestType.GENERATED_REGRESSION,
return_value=[3],
cpu_runtime=0,
timed_out=False,
loop_index=1,
)

4
packages/codeflash-python/tests/test_model_test_results.py
View file

@ -52,6 +52,7 @@ def make_invocation(
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
)
@ -143,6 +144,7 @@ class TestFunctionTestInvocation:
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
verification_type=VerificationType.INIT_STATE_FTO,
)
@ -348,6 +350,7 @@ def _make_replay_invocation(
test_framework="pytest",
test_type=TestType.REPLAY_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
)
@ -427,6 +430,7 @@ class TestGroupByBenchmarks:
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
)
results.add(unit_inv)

1
packages/codeflash-python/tests/test_parallel_eval_integration.py
View file

@ -65,6 +65,7 @@ def _make_invocation(
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=42,
cpu_runtime=0,
timed_out=False,
)

7
packages/codeflash-python/tests/test_parse_results.py
View file

@ -64,6 +64,7 @@ def make_invocation(
test_framework="pytest",
test_type=TestType.EXISTING_UNIT_TEST,
return_value=None,
cpu_runtime=0,
timed_out=False,
)
@ -273,12 +274,13 @@ class TestParseSqliteTestResults:
" iteration_id TEXT,"
" runtime INTEGER,"
" return_value BLOB,"
" verification_type TEXT"
" verification_type TEXT,"
" cpu_runtime INTEGER"
")"
)
module_name = f"tests{os.sep}test_foo".replace(os.sep, ".")
conn.execute(
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)",
"INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)",
(
module_name,
None,
@ -289,6 +291,7 @@ class TestParseSqliteTestResults:
100,
None,
VerificationType.FUNCTION_CALL.value,
0,
),
)
conn.commit()

1
packages/codeflash-python/tests/test_verification.py
View file

@ -57,6 +57,7 @@ def make_invocation( # noqa: PLR0913
test_framework="pytest",
test_type=test_type,
return_value=return_value,
cpu_runtime=0,
timed_out=timed_out,
verification_type=verification_type,
stdout=stdout,