mirror of
https://github.com/codeflash-ai/codeflash.git
synced 2026-05-04 18:25:17 +00:00
19bd6e4bad
34 test files updated with main's refactored tests for new language support protocol, JS/TS improvements, and code context extraction.
506 lines
16 KiB
Python
506 lines
16 KiB
Python
"""End-to-end tests for JavaScript/TypeScript test execution and result parsing.
|
|
|
|
These tests verify the FULL optimization pipeline including:
|
|
- Test instrumentation
|
|
- Running instrumented tests with Vitest/Jest
|
|
- Parsing test results (stdout, timing, return values)
|
|
- Benchmarking with multiple loops
|
|
|
|
This is the JavaScript equivalent of test_instrument_tests.py for Python.
|
|
|
|
NOTE: These tests require:
|
|
- Node.js installed
|
|
- npm packages installed in the test fixture directories
|
|
- The codeflash npm package
|
|
|
|
Tests will be skipped if dependencies are not available.
|
|
"""
|
|
|
|
import subprocess
|
|
from pathlib import Path
|
|
from unittest.mock import MagicMock
|
|
|
|
import pytest
|
|
|
|
from codeflash.discovery.functions_to_optimize import FunctionToOptimize
|
|
from codeflash.languages.base import Language
|
|
from codeflash.models.models import FunctionParent
|
|
from codeflash.verification.verification_utils import TestConfig
|
|
|
|
|
|
def is_node_available():
|
|
"""Check if Node.js is available."""
|
|
try:
|
|
result = subprocess.run(["node", "--version"], capture_output=True, text=True)
|
|
return result.returncode == 0
|
|
except FileNotFoundError:
|
|
return False
|
|
|
|
|
|
def is_npm_available():
|
|
"""Check if npm is available."""
|
|
try:
|
|
result = subprocess.run(["npm", "--version"], capture_output=True, text=True)
|
|
return result.returncode == 0
|
|
except FileNotFoundError:
|
|
return False
|
|
|
|
|
|
def has_node_modules(project_dir: Path) -> bool:
|
|
"""Check if node_modules exists in project directory."""
|
|
return (project_dir / "node_modules").exists()
|
|
|
|
|
|
def install_dependencies(project_dir: Path) -> bool:
|
|
"""Install npm dependencies in project directory."""
|
|
if has_node_modules(project_dir):
|
|
return True
|
|
try:
|
|
result = subprocess.run(["npm", "install"], cwd=project_dir, capture_output=True, text=True, timeout=120)
|
|
return result.returncode == 0
|
|
except Exception:
|
|
return False
|
|
|
|
|
|
def skip_if_js_runtime_not_available():
|
|
"""Skip test if JavaScript runtime is not available."""
|
|
if not is_node_available():
|
|
pytest.skip("Node.js not available")
|
|
if not is_npm_available():
|
|
pytest.skip("npm not available")
|
|
|
|
|
|
def skip_if_js_not_supported():
|
|
"""Skip test if JavaScript/TypeScript languages are not supported."""
|
|
try:
|
|
from codeflash.languages import get_language_support
|
|
|
|
get_language_support(Language.JAVASCRIPT)
|
|
except Exception as e:
|
|
pytest.skip(f"JavaScript/TypeScript language support not available: {e}")
|
|
|
|
|
|
class TestJavaScriptInstrumentation:
|
|
"""Tests for JavaScript test instrumentation."""
|
|
|
|
@pytest.fixture
|
|
def js_project_dir(self, tmp_path):
|
|
"""Create a temporary JavaScript project with Jest."""
|
|
project_dir = tmp_path / "js_project"
|
|
project_dir.mkdir()
|
|
|
|
# Create source file
|
|
src_file = project_dir / "math.js"
|
|
src_file.write_text("""
|
|
function add(a, b) {
|
|
return a + b;
|
|
}
|
|
|
|
function multiply(a, b) {
|
|
return a * b;
|
|
}
|
|
|
|
module.exports = { add, multiply };
|
|
""")
|
|
|
|
# Create test file
|
|
tests_dir = project_dir / "__tests__"
|
|
tests_dir.mkdir()
|
|
test_file = tests_dir / "math.test.js"
|
|
test_file.write_text("""
|
|
const { add, multiply } = require('../math');
|
|
|
|
describe('math functions', () => {
|
|
test('add returns sum', () => {
|
|
expect(add(2, 3)).toBe(5);
|
|
});
|
|
|
|
test('multiply returns product', () => {
|
|
expect(multiply(2, 3)).toBe(6);
|
|
});
|
|
});
|
|
""")
|
|
|
|
# Create package.json
|
|
package_json = project_dir / "package.json"
|
|
package_json.write_text("""{
|
|
"name": "test-project",
|
|
"version": "1.0.0",
|
|
"scripts": {
|
|
"test": "jest"
|
|
},
|
|
"devDependencies": {
|
|
"jest": "^29.0.0",
|
|
"jest-junit": "^16.0.0"
|
|
}
|
|
}""")
|
|
|
|
# Create jest.config.js
|
|
jest_config = project_dir / "jest.config.js"
|
|
jest_config.write_text("""
|
|
module.exports = {
|
|
testEnvironment: 'node',
|
|
reporters: ['default', 'jest-junit'],
|
|
};
|
|
""")
|
|
|
|
return project_dir
|
|
|
|
def test_instrument_javascript_test_file(self, js_project_dir):
|
|
"""Test that JavaScript test instrumentation module can be imported."""
|
|
skip_if_js_not_supported()
|
|
from codeflash.languages import get_language_support
|
|
|
|
# Verify the instrumentation module can be imported
|
|
|
|
# Get JavaScript support
|
|
js_support = get_language_support(Language.JAVASCRIPT)
|
|
|
|
# Create function info
|
|
func_info = FunctionToOptimize(
|
|
function_name="add",
|
|
file_path=js_project_dir / "math.js",
|
|
parents=[],
|
|
starting_line=2,
|
|
ending_line=4,
|
|
language="javascript",
|
|
)
|
|
|
|
# Verify function has correct language
|
|
assert func_info.language == "javascript"
|
|
|
|
# Verify test file exists
|
|
test_file = js_project_dir / "__tests__" / "math.test.js"
|
|
assert test_file.exists()
|
|
|
|
# Note: Full instrumentation test requires call_positions discovery
|
|
# which is done by the FunctionOptimizer. Here we just verify the
|
|
# infrastructure is in place.
|
|
|
|
|
|
class TestTypeScriptInstrumentation:
|
|
"""Tests for TypeScript test instrumentation."""
|
|
|
|
@pytest.fixture
|
|
def ts_project_dir(self, tmp_path):
|
|
"""Create a temporary TypeScript project with Vitest."""
|
|
project_dir = tmp_path / "ts_project"
|
|
project_dir.mkdir()
|
|
|
|
# Create source file
|
|
src_file = project_dir / "math.ts"
|
|
src_file.write_text("""
|
|
export function add(a: number, b: number): number {
|
|
return a + b;
|
|
}
|
|
|
|
export function multiply(a: number, b: number): number {
|
|
return a * b;
|
|
}
|
|
""")
|
|
|
|
# Create test file
|
|
tests_dir = project_dir / "tests"
|
|
tests_dir.mkdir()
|
|
test_file = tests_dir / "math.test.ts"
|
|
test_file.write_text("""
|
|
import { describe, test, expect } from 'vitest';
|
|
import { add, multiply } from '../math';
|
|
|
|
describe('math functions', () => {
|
|
test('add returns sum', () => {
|
|
expect(add(2, 3)).toBe(5);
|
|
});
|
|
|
|
test('multiply returns product', () => {
|
|
expect(multiply(2, 3)).toBe(6);
|
|
});
|
|
});
|
|
""")
|
|
|
|
# Create package.json
|
|
package_json = project_dir / "package.json"
|
|
package_json.write_text("""{
|
|
"name": "test-project",
|
|
"version": "1.0.0",
|
|
"type": "module",
|
|
"scripts": {
|
|
"test": "vitest run"
|
|
},
|
|
"devDependencies": {
|
|
"vitest": "^1.0.0",
|
|
"typescript": "^5.0.0"
|
|
}
|
|
}""")
|
|
|
|
# Create vitest.config.ts
|
|
vitest_config = project_dir / "vitest.config.ts"
|
|
vitest_config.write_text("""
|
|
import { defineConfig } from 'vitest/config';
|
|
|
|
export default defineConfig({
|
|
test: {
|
|
globals: false,
|
|
reporters: ['verbose', 'junit'],
|
|
outputFile: './junit.xml',
|
|
},
|
|
});
|
|
""")
|
|
|
|
# Create tsconfig.json
|
|
tsconfig = project_dir / "tsconfig.json"
|
|
tsconfig.write_text("""{
|
|
"compilerOptions": {
|
|
"target": "ES2020",
|
|
"module": "ESNext",
|
|
"moduleResolution": "node",
|
|
"strict": true,
|
|
"esModuleInterop": true
|
|
}
|
|
}""")
|
|
|
|
return project_dir
|
|
|
|
def test_instrument_typescript_test_file(self, ts_project_dir):
|
|
"""Test that TypeScript test instrumentation module can be imported."""
|
|
skip_if_js_not_supported()
|
|
from codeflash.languages import get_language_support
|
|
|
|
# Verify the instrumentation module can be imported
|
|
|
|
test_file = ts_project_dir / "tests" / "math.test.ts"
|
|
|
|
# Get TypeScript support
|
|
ts_support = get_language_support(Language.TYPESCRIPT)
|
|
|
|
# Create function info
|
|
func_info = FunctionToOptimize(
|
|
function_name="add",
|
|
file_path=ts_project_dir / "math.ts",
|
|
parents=[],
|
|
starting_line=2,
|
|
ending_line=4,
|
|
language="typescript",
|
|
)
|
|
|
|
# Verify function has correct language
|
|
assert func_info.language == "typescript"
|
|
|
|
# Verify test file exists
|
|
assert test_file.exists()
|
|
|
|
# Note: Full instrumentation test requires call_positions discovery
|
|
# which is done by the FunctionOptimizer. Here we just verify the
|
|
# infrastructure is in place.
|
|
|
|
|
|
class TestRunAndParseJavaScriptTests:
|
|
"""Tests for running and parsing JavaScript test results.
|
|
|
|
These tests require actual npm dependencies to be installed.
|
|
They will be skipped if dependencies are not available.
|
|
"""
|
|
|
|
@pytest.fixture
|
|
def vitest_project(self):
|
|
"""Get the Vitest sample project with dependencies installed."""
|
|
project_root = Path(__file__).parent.parent.parent
|
|
vitest_dir = project_root / "code_to_optimize" / "js" / "code_to_optimize_vitest"
|
|
|
|
if not vitest_dir.exists():
|
|
pytest.skip("code_to_optimize_vitest directory not found")
|
|
|
|
skip_if_js_runtime_not_available()
|
|
|
|
# Try to install dependencies if not present
|
|
if not has_node_modules(vitest_dir):
|
|
if not install_dependencies(vitest_dir):
|
|
pytest.skip("Could not install npm dependencies")
|
|
|
|
return vitest_dir
|
|
|
|
def test_run_behavioral_tests_vitest(self, vitest_project):
|
|
"""Test running behavioral tests with Vitest."""
|
|
skip_if_js_not_supported()
|
|
from codeflash.discovery.functions_to_optimize import find_all_functions_in_file
|
|
from codeflash.languages import get_language_support
|
|
|
|
ts_support = get_language_support(Language.TYPESCRIPT)
|
|
|
|
# Find the fibonacci function
|
|
fib_file = vitest_project / "fibonacci.ts"
|
|
functions = find_all_functions_in_file(fib_file)
|
|
fib_func = next(f for f in functions[fib_file] if f.function_name == "fibonacci")
|
|
|
|
# Verify language is correct
|
|
assert fib_func.language == "typescript"
|
|
|
|
# Discover tests
|
|
test_root = vitest_project / "tests"
|
|
tests = ts_support.discover_tests(test_root, [fib_func])
|
|
|
|
# There should be tests for fibonacci
|
|
assert len(tests) > 0 or fib_func.qualified_name in tests
|
|
|
|
def test_function_optimizer_run_and_parse_typescript(self, vitest_project):
|
|
"""Test FunctionOptimizer.run_and_parse_tests for TypeScript.
|
|
|
|
This is the JavaScript equivalent of the Python test in test_instrument_tests.py.
|
|
"""
|
|
skip_if_js_not_supported()
|
|
from codeflash.discovery.functions_to_optimize import find_all_functions_in_file
|
|
from codeflash.languages.javascript.function_optimizer import JavaScriptFunctionOptimizer
|
|
|
|
# Find the fibonacci function
|
|
fib_file = vitest_project / "fibonacci.ts"
|
|
functions = find_all_functions_in_file(fib_file)
|
|
fib_func_info = next(f for f in functions[fib_file] if f.function_name == "fibonacci")
|
|
|
|
# Create FunctionToOptimize
|
|
func = FunctionToOptimize(
|
|
function_name=fib_func_info.function_name,
|
|
file_path=fib_func_info.file_path,
|
|
parents=[FunctionParent(name=p.name, type=p.type) for p in fib_func_info.parents],
|
|
starting_line=fib_func_info.starting_line,
|
|
ending_line=fib_func_info.ending_line,
|
|
language=fib_func_info.language,
|
|
)
|
|
|
|
# Verify language
|
|
assert func.language == "typescript"
|
|
|
|
# Create test config
|
|
test_config = TestConfig(
|
|
tests_root=vitest_project / "tests",
|
|
tests_project_rootdir=vitest_project,
|
|
project_root_path=vitest_project,
|
|
pytest_cmd="vitest",
|
|
test_framework="vitest",
|
|
)
|
|
|
|
# Create optimizer
|
|
func_optimizer = JavaScriptFunctionOptimizer(
|
|
function_to_optimize=func, test_cfg=test_config, aiservice_client=MagicMock()
|
|
)
|
|
|
|
# Get code context - this should work
|
|
result = func_optimizer.get_code_optimization_context()
|
|
context = result.unwrap()
|
|
|
|
assert context is not None
|
|
assert context.read_writable_code.language == "typescript"
|
|
|
|
|
|
class TestTimingMarkerParsing:
|
|
"""Tests for parsing JavaScript timing markers from test output.
|
|
|
|
Note: Timing marker parsing is handled in codeflash/verification/parse_test_output.py,
|
|
which uses a unified parser for all languages. These tests verify the marker format
|
|
is correctly recognized.
|
|
"""
|
|
|
|
def test_timing_marker_format(self):
|
|
"""Test that JavaScript timing markers follow the expected format."""
|
|
skip_if_js_not_supported()
|
|
import re
|
|
|
|
# The marker format used by codeflash for JavaScript
|
|
# Start marker: !$######{tag}######$!
|
|
# End marker: !######{tag}:{duration}######!
|
|
start_pattern = r"!\$######(.+?)######\$!"
|
|
end_pattern = r"!######(.+?):(\d+)######!"
|
|
|
|
start_marker = "!$######test/math.test.ts:TestMath.test_add:add:1:0_0######$!"
|
|
end_marker = "!######test/math.test.ts:TestMath.test_add:add:1:0_0:12345######!"
|
|
|
|
start_match = re.match(start_pattern, start_marker)
|
|
end_match = re.match(end_pattern, end_marker)
|
|
|
|
assert start_match is not None
|
|
assert end_match is not None
|
|
assert start_match.group(1) == "test/math.test.ts:TestMath.test_add:add:1:0_0"
|
|
assert end_match.group(1) == "test/math.test.ts:TestMath.test_add:add:1:0_0"
|
|
assert end_match.group(2) == "12345"
|
|
|
|
def test_timing_marker_components(self):
|
|
"""Test parsing components from timing marker tag."""
|
|
skip_if_js_not_supported()
|
|
|
|
# Tag format: {module}:{class}.{test}:{function}:{loop_index}:{invocation_id}
|
|
tag = "test/math.test.ts:TestMath.test_add:add:1:0_0"
|
|
parts = tag.split(":")
|
|
|
|
assert len(parts) == 5
|
|
assert parts[0] == "test/math.test.ts" # module/file
|
|
assert parts[1] == "TestMath.test_add" # class.test
|
|
assert parts[2] == "add" # function being tested
|
|
assert parts[3] == "1" # loop index
|
|
assert parts[4] == "0_0" # invocation id
|
|
|
|
|
|
class TestJavaScriptTestResultParsing:
|
|
"""Tests for parsing JavaScript test results from JUnit XML."""
|
|
|
|
def test_parse_vitest_junit_xml(self, tmp_path):
|
|
"""Test parsing Vitest JUnit XML output."""
|
|
skip_if_js_not_supported()
|
|
|
|
# Create sample JUnit XML
|
|
junit_xml = tmp_path / "junit.xml"
|
|
junit_xml.write_text("""<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites name="vitest tests" tests="2" failures="0" errors="0" time="0.5">
|
|
<testsuite name="tests/math.test.ts" tests="2" failures="0" errors="0" time="0.5">
|
|
<testcase classname="tests/math.test.ts" name="add returns sum" time="0.1">
|
|
</testcase>
|
|
<testcase classname="tests/math.test.ts" name="multiply returns product" time="0.2">
|
|
</testcase>
|
|
</testsuite>
|
|
</testsuites>
|
|
""")
|
|
|
|
# Parse the XML
|
|
import xml.etree.ElementTree as ET
|
|
|
|
tree = ET.parse(junit_xml)
|
|
root = tree.getroot()
|
|
|
|
# Verify structure
|
|
testsuites = root if root.tag == "testsuites" else root.find("testsuites")
|
|
assert testsuites is not None
|
|
|
|
testsuite = testsuites.find("testsuite") if testsuites is not None else root.find("testsuite")
|
|
assert testsuite is not None
|
|
|
|
testcases = testsuite.findall("testcase")
|
|
assert len(testcases) == 2
|
|
|
|
def test_parse_jest_junit_xml(self, tmp_path):
|
|
"""Test parsing Jest JUnit XML output."""
|
|
skip_if_js_not_supported()
|
|
|
|
# Create sample JUnit XML from jest-junit
|
|
junit_xml = tmp_path / "junit.xml"
|
|
junit_xml.write_text("""<?xml version="1.0" encoding="UTF-8"?>
|
|
<testsuites name="jest tests" tests="2" failures="0" time="0.789">
|
|
<testsuite name="math functions" tests="2" failures="0" time="0.456" timestamp="2024-01-01T00:00:00">
|
|
<testcase classname="__tests__/math.test.js" name="add returns sum" time="0.123">
|
|
</testcase>
|
|
<testcase classname="__tests__/math.test.js" name="multiply returns product" time="0.234">
|
|
</testcase>
|
|
</testsuite>
|
|
</testsuites>
|
|
""")
|
|
|
|
# Parse the XML
|
|
import xml.etree.ElementTree as ET
|
|
|
|
tree = ET.parse(junit_xml)
|
|
root = tree.getroot()
|
|
|
|
# Verify structure
|
|
testsuites = root if root.tag == "testsuites" else root.find("testsuites")
|
|
testsuite = testsuites.find("testsuite") if testsuites is not None else root.find("testsuite")
|
|
assert testsuite is not None
|
|
|
|
testcases = testsuite.findall("testcase")
|
|
assert len(testcases) == 2
|