fix tests & friends

cli/codeflash/code_utils/coverage_utils.py

@@ -10,7 +10,7 @@ from pydantic.dataclasses import dataclass
 
 from codeflash.cli_cmds.console import console, logger
 from codeflash.code_utils.code_utils import get_run_tmp_file
-from codeflash.models.models import CodeOptimizationContext, FunctionCoverage
+from codeflash.models.models import CodeOptimizationContext
 
 
 def extract_dependent_function(main_function: str, code_context: CodeOptimizationContext) -> str | Literal[False]:
@@ -74,6 +74,8 @@ def grab_dependent_function_from_coverage_data(
         except KeyError:
             raise ValueError(msg) from None
 
+    return FunctionCoverage(name=dependent_function_name, coverage=0, executed_lines=[], unexecuted_lines=[])
+
 
 def prepare_coverage_files(project_root: Path) -> tuple[Path, Path]:
     """Prepare coverage configuration and output files."""
@@ -90,6 +92,16 @@ def prepare_coverage_files(project_root: Path) -> tuple[Path, Path]:
     return coverage_out_file, coveragercfile
 
 
+@dataclass
+class FunctionCoverage:
+    """Represents the coverage data for a specific function in a source file."""
+
+    name: str
+    coverage: float
+    executed_lines: list[int]
+    unexecuted_lines: list[int]
+
+
 @dataclass
 class CoverageData:
     """Represents the coverage data for a specific function in a source file, using one or more test files."""

cli/codeflash/models/models.py

@@ -139,13 +139,3 @@ class CodePosition:
 class FunctionParent:
     name: str
     type: str
-
-
-@dataclass
-class FunctionCoverage:
-    """Represents the coverage data for a specific function in a source file."""
-
-    name: str
-    coverage: float
-    executed_lines: list[int]
-    unexecuted_lines: list[int]

cli/codeflash/verification/test_runner.py

@@ -57,34 +57,32 @@ def run_tests(
                 )
             else:
                 test_files.append(str(file.instrumented_file_path))
+        pytest_cmd_list = shlex.split(pytest_cmd, posix=is_posix)
+
+        common_pytest_args = [
+            "--capture=tee-sys",
+            f"--timeout={pytest_timeout}",
+            "-q",
+            f"--codeflash_seconds={pytest_target_runtime_seconds}",
+            "--codeflash_loops_scope=session",
+        ]
+        pytest_test_env = test_env.copy()
+        pytest_test_env["PYTEST_PLUGINS"] = "codeflash.verification.pytest_plugin"
 
         if enable_coverage:
             assert project_root is not None, "project_root must be provided for coverage analysis"
             if not source_file:
-                msg = "source_file must be provided for coverage analysis"
-                raise ValueError(msg)
+                src_file_msg = "source_file must be provided for coverage analysis"
+                raise ValueError(src_file_msg)
             if not function_name:
-                msg = "function_name must be provided for coverage analysis"
-                raise ValueError(msg)
+                function_name_msg = "function_name must be provided for coverage analysis"
+                raise ValueError(function_name_msg)
 
             coverage_out_file, coveragercfile = prepare_coverage_files(project_root)
 
-            pytest_ignore_files = [
-                "--ignore-glob=build/*",
-                "--ignore-glob=dist/*",
-                "--ignore-glob=*.egg-info/*",
-            ]  # --ignore-glob=path, let's use this to ignore leftover build artifacts from setuptools for local installs https://pip.pypa.io/en/stable/topics/local-project-installs/#build-artifacts
+            pytest_ignore_files = ["--ignore-glob=build/*", "--ignore-glob=dist/*", "--ignore-glob=*.egg-info/*"]
 
-            pytest_test_env = test_env.copy()
-
-            pytest_test_env["PYTEST_PLUGINS"] = "codeflash.verification.pytest_plugin"
-            pytest_args = [
-                f"--timeout={pytest_timeout}",
-                f"--codeflash_seconds={pytest_target_runtime_seconds}",
-                "--codeflash_min_loops=1",
-                "--codeflash_max_loops=1",
-                "--codeflash_loops_scope=session",
-            ]
+            coverage_args = ["--codeflash_min_loops=1", "--codeflash_max_loops=1"]
 
             cov_erase = execute_test_subprocess(
                 shlex.split(f"{sys.executable} -m coverage erase"), cwd=cwd, env=pytest_test_env
@@ -96,11 +94,12 @@ def run_tests(
                 for file in test_paths.test_files
                 if file.test_type == TestType.GENERATED_REGRESSION
             ]
-            logger.info(files)
+
             cov_run = execute_test_subprocess(
                 shlex.split(f"{sys.executable} -m coverage run --rcfile={coveragercfile} -m pytest")
                 + files
-                + pytest_args
+                + common_pytest_args
+                + coverage_args
                 + pytest_ignore_files,
                 cwd=cwd,
                 env=pytest_test_env,
@@ -119,37 +118,29 @@ def run_tests(
             coveragepy_coverage.log_coverage()
 
         result_file_path = get_run_tmp_file(Path("pytest_results.xml"))
-        pytest_cmd_list = shlex.split(pytest_cmd, posix=is_posix)
-        pytest_test_env = test_env.copy()
-        pytest_test_env["PYTEST_PLUGINS"] = "codeflash.verification.pytest_plugin"
-        pytest_args = [
-            "--capture=tee-sys",
-            f"--timeout={pytest_timeout}",
-            "-q",
-            f"--junitxml={result_file_path}",
-            "-o",
-            "junit_logging=all",
-            f"--codeflash_seconds={pytest_target_runtime_seconds}",
-            f"--codeflash_min_loops={pytest_min_loops}",
-            f"--codeflash_max_loops={pytest_max_loops}",
-            "--codeflash_loops_scope=session",
-        ]
+        result_args = [f"--junitxml={result_file_path}", "-o", "junit_logging=all"]
 
         results = execute_test_subprocess(
-            pytest_cmd_list + test_files + pytest_args,
+            pytest_cmd_list
+            + test_files
+            + common_pytest_args
+            + result_args
+            + [f"--codeflash_min_loops={pytest_min_loops}", f"--codeflash_max_loops={pytest_max_loops}"],
             cwd=cwd,
             env=pytest_test_env,
             timeout=600,  # TODO: Make this dynamic
         )
     elif test_framework == "unittest":
         result_file_path = get_run_tmp_file(Path("unittest_results.xml"))
-        unittest_cmd = ["python", "-m", "unittest"]
-        verbosity = ["-v"] if verbose else []
-        test_files = [str(file.instrumented_file_path) for file in test_paths.test_files]
-        output_file_command = ["--output-file", str(result_file_path)]
+        unittest_cmd_list = [sys.executable, "-m", "xmlrunner"]
+        log_level = ["-v"] if verbose else []
+        files = [str(file.instrumented_file_path) for file in test_paths.test_files]
+        output_file = ["--output-file", str(result_file_path)]
+
         results = execute_test_subprocess(
-            unittest_cmd + verbosity + test_files + output_file_command, cwd=cwd, env=test_env
+            unittest_cmd_list + log_level + files + output_file, cwd=cwd, env=test_env, timeout=600
         )
+
     else:
         raise ValueError("Invalid test framework -- I only support Pytest and Unittest currently.")
     return result_file_path, results

cli/tests/test_test_runner.py

@@ -2,6 +2,7 @@ import os
 import tempfile
 from pathlib import Path
 
+from codeflash.cli_cmds.console import logger
 from codeflash.models.models import TestFile, TestFiles
 from codeflash.verification.parse_test_output import parse_test_xml
 from codeflash.verification.test_results import TestType
@@ -41,7 +42,12 @@ class TestUnittestRunnerSorter(unittest.TestCase):
         fp.write(code.encode("utf-8"))
         fp.flush()
         result_file, process = run_tests(
-            test_files, test_framework=config.test_framework, cwd=Path(config.project_root_path)
+            test_files,
+            test_framework=config.test_framework,
+            cwd=Path(config.project_root_path),
+            test_env=os.environ.copy(),
+            function_name="test_sort",
+            source_file=Path(fp.name),
         )
         results = parse_test_xml(result_file, test_files, config, process)
     assert results[0].did_pass, "Test did not pass as expected"
@@ -66,13 +72,21 @@ def test_sort():
         test_framework="pytest",
         tests_project_rootdir=cur_dir_path.parent,
     )
+
+    test_env = os.environ.copy()
+    test_env["CODEFLASH_TEST_ITERATION"] = "0"
+    test_env["CODEFLASH_TRACER_DISABLE"] = "1"
+    if "PYTHONPATH" not in test_env:
+        test_env["PYTHONPATH"] = str(config.project_root_path)
+    else:
+        test_env["PYTHONPATH"] += os.pathsep + str(config.project_root_path)
+
     with tempfile.NamedTemporaryFile(prefix="test_xx", suffix=".py", dir=cur_dir_path) as fp:
         test_files = TestFiles(
             test_files=[TestFile(instrumented_file_path=Path(fp.name), test_type=TestType.EXISTING_UNIT_TEST)]
         )
         fp.write(code.encode("utf-8"))
         fp.flush()
-        test_env = os.environ.copy()
         result_file, process = run_tests(
             test_files,
             test_framework=config.test_framework,
@@ -82,6 +96,8 @@ def test_sort():
             pytest_min_loops=1,
             pytest_max_loops=1,
             pytest_target_runtime_seconds=1,
+            function_name="test_sort",
+            source_file=Path(fp.name),
         )
         results = parse_test_xml(
             test_xml_file_path=result_file, test_files=test_files, test_config=config, run_result=process

js/cf-api/endpoints/create-pr.ts

@@ -28,6 +28,7 @@ export async function createPr(req, res, next) {
       prCommentFields,
       existingTests,
       generatedTests,
+      coverage_data_json,
     } = req.body
     const userId = req.userId
     //traceId is optional to allow for backwards compatibility, can make this required in the future