codeflash/tests/test_test_runner.py

import os
import tempfile
from pathlib import Path

from codeflash.code_utils.code_utils import ImportErrorPattern
from codeflash.models.models import TestFile, TestFiles, TestType
from codeflash.verification.parse_test_output import parse_test_xml
from codeflash.verification.test_runner import run_behavioral_tests
from codeflash.verification.verification_utils import TestConfig


def test_unittest_runner():
    code = """import time
import gc
import unittest
def sorter(arr):
    arr.sort()
    return arr
class TestUnittestRunnerSorter(unittest.TestCase):
    def test_sort(self):
        arr = [5, 4, 3, 2, 1, 0]
        gc.disable()
        counter = time.perf_counter_ns()
        output = sorter(arr)
        duration = time.perf_counter_ns() - counter
        gc.enable()
        print(f"#####test_sorter__unit_test_0:TestUnittestRunnerSorter.test_sort:sorter:0#####{duration}^^^^^")
"""
    cur_dir_path = Path(__file__).resolve().parent
    config = TestConfig(
        tests_root=cur_dir_path,
        project_root_path=cur_dir_path,
        test_framework="unittest",
        tests_project_rootdir=cur_dir_path.parent,
    )

    test_env = os.environ.copy()
    test_env["CODEFLASH_TEST_ITERATION"] = "0"
    test_env["CODEFLASH_TRACER_DISABLE"] = "1"
    if "PYTHONPATH" not in test_env:
        test_env["PYTHONPATH"] = str(config.project_root_path)
    else:
        test_env["PYTHONPATH"] += os.pathsep + str(config.project_root_path)

    with tempfile.TemporaryDirectory(dir=cur_dir_path) as temp_dir:
        test_file_path = Path(temp_dir) / "test_xx.py"
        test_files = TestFiles(
            test_files=[TestFile(instrumented_behavior_file_path=test_file_path, test_type=TestType.EXISTING_UNIT_TEST)]
        )
        test_file_path.write_text(code, encoding="utf-8")
        result_file, process, _, _ = run_behavioral_tests(
            test_files, test_framework=config.test_framework, cwd=Path(config.project_root_path), test_env=test_env
        )
        results = parse_test_xml(result_file, test_files, config, process)
    assert results[0].did_pass, "Test did not pass as expected"
    result_file.unlink(missing_ok=True)


def test_pytest_runner():
    code = """
def sorter(arr):
    arr.sort()
    return arr

def test_sort():
    arr = [5, 4, 3, 2, 1, 0]
    output = sorter(arr)
    assert output == [0, 1, 2, 3, 4, 5]
"""
    cur_dir_path = Path(__file__).resolve().parent
    config = TestConfig(
        tests_root=cur_dir_path,
        project_root_path=cur_dir_path,
        test_framework="pytest",
        tests_project_rootdir=cur_dir_path.parent,
    )

    test_env = os.environ.copy()
    test_env["CODEFLASH_TEST_ITERATION"] = "0"
    test_env["CODEFLASH_TRACER_DISABLE"] = "1"
    if "PYTHONPATH" not in test_env:
        test_env["PYTHONPATH"] = str(config.project_root_path)
    else:
        test_env["PYTHONPATH"] += os.pathsep + str(config.project_root_path)

    with tempfile.TemporaryDirectory(dir=cur_dir_path) as temp_dir:
        test_file_path = Path(temp_dir) / "test_xx.py"
        test_files = TestFiles(
            test_files=[TestFile(instrumented_behavior_file_path=test_file_path, test_type=TestType.EXISTING_UNIT_TEST)]
        )
        test_file_path.write_text(code, encoding="utf-8")
        result_file, process, _, _ = run_behavioral_tests(
            test_files,
            test_framework=config.test_framework,
            cwd=Path(config.project_root_path),
            test_env=test_env,
            pytest_timeout=1,
            pytest_target_runtime_seconds=1,
        )
        results = parse_test_xml(
            test_xml_file_path=result_file, test_files=test_files, test_config=config, run_result=process
        )
    assert results[0].did_pass, "Test did not pass as expected"
    result_file.unlink(missing_ok=True)

    code = """import torch_does_not_exist
def sorter(arr):
    print(torch.ones(1))
    arr.sort()
    return arr

def test_sort():
    arr = [5, 4, 3, 2, 1, 0]
    output = sorter(arr)
    assert output == [0, 1, 2, 3, 4, 5]
    """
    cur_dir_path = Path(__file__).resolve().parent
    config = TestConfig(
        tests_root=cur_dir_path,
        project_root_path=cur_dir_path,
        test_framework="pytest",
        tests_project_rootdir=cur_dir_path.parent,
    )

    test_env = os.environ.copy()
    test_env["CODEFLASH_TEST_ITERATION"] = "0"
    test_env["CODEFLASH_TRACER_DISABLE"] = "1"
    if "PYTHONPATH" not in test_env:
        test_env["PYTHONPATH"] = str(config.project_root_path)
    else:
        test_env["PYTHONPATH"] += os.pathsep + str(config.project_root_path)

    with tempfile.TemporaryDirectory(dir=cur_dir_path) as temp_dir:
        test_file_path = Path(temp_dir) / "test_xx.py"
        test_files = TestFiles(
            test_files=[TestFile(instrumented_behavior_file_path=test_file_path, test_type=TestType.EXISTING_UNIT_TEST)]
        )
        test_file_path.write_text(code, encoding="utf-8")
        result_file, process, _, _ = run_behavioral_tests(
            test_files,
            test_framework=config.test_framework,
            cwd=Path(config.project_root_path),
            test_env=test_env,
            pytest_timeout=1,
            pytest_target_runtime_seconds=1,
        )
        results = parse_test_xml(
            test_xml_file_path=result_file, test_files=test_files, test_config=config, run_result=process
        )
    match = ImportErrorPattern.search(process.stdout).group()
    assert match == "ModuleNotFoundError: No module named 'torch_does_not_exist'"
    result_file.unlink(missing_ok=True)