fix: Improve error messaging for files excluded from Vitest coverage
## Problem When a file is excluded from coverage by vitest.config.ts (e.g., via `coverage.exclude: ["src/agents/**"]`), Codeflash reports misleading "Test coverage is 0.0%" messages even though tests run successfully. This happens because: - Vitest doesn't include excluded files in coverage-final.json - Codeflash detects this (status = NOT_FOUND) but shows generic 0% message - Users don't know the file is excluded from coverage collection ## Solution Detect when coverage status is NOT_FOUND and provide a clear, actionable error message explaining: 1. No coverage data was found for the file 2. It may be excluded by test framework configuration 3. Where to check (coverage.exclude in vitest.config.ts, etc.) ## Changes - function_optimizer.py: Check CoverageStatus.NOT_FOUND before reporting 0% - Added clear warning log and user-facing error message - New test file: test_vitest_coverage_exclusions.py ## Testing - All existing JavaScript tests pass - New tests verify NOT_FOUND status is returned correctly - Manual verification with openclaw logs (trace: 2a84fe6b-9871-4916-96da-bdd79bca508a) Fixes #BUG-1 (from autoresearch:debug workflow) Trace IDs affected: All 10 log files showing 0% coverage in /workspace/logs Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
Codeflash Bot
parent
f86fe2d4ed
commit
e329d52c34
2 changed files with 144 additions and 0 deletions
|
|
@ -2787,6 +2787,25 @@ class FunctionOptimizer:
|
|||
did_pass_all_tests = all(result.did_pass for result in behavioral_results)
|
||||
if not did_pass_all_tests:
|
||||
return Failure("Tests failed to pass for the original code.")
|
||||
|
||||
# Check if coverage data was not found (file excluded from coverage)
|
||||
from codeflash.models.models import CoverageStatus
|
||||
|
||||
if coverage_results and coverage_results.status == CoverageStatus.NOT_FOUND:
|
||||
# File was not found in coverage data - likely excluded by test framework config
|
||||
logger.warning(
|
||||
f"No coverage data found for {self.function_to_optimize.source_file_path}. "
|
||||
f"This file may be excluded from coverage collection by your test framework configuration "
|
||||
f"(e.g., coverage.exclude in vitest.config.ts for Vitest, or testMatch/coveragePathIgnorePatterns "
|
||||
f"for Jest). Tests ran successfully but coverage cannot be measured."
|
||||
)
|
||||
return Failure(
|
||||
f"Coverage data not found for {self.function_to_optimize.source_file_path}. "
|
||||
f"The file may be excluded from coverage by your test framework config. "
|
||||
f"Check coverage.exclude patterns in vitest.config.ts or jest.config.js."
|
||||
)
|
||||
|
||||
# Normal coverage failure (tests ran but coverage below threshold)
|
||||
coverage_pct = coverage_results.coverage if coverage_results else 0
|
||||
return Failure(
|
||||
f"Test coverage is {coverage_pct}%, which is below the required threshold of {COVERAGE_THRESHOLD}%."
|
||||
|
|
|
|||
125
tests/languages/javascript/test_vitest_coverage_exclusions.py
Normal file
125
tests/languages/javascript/test_vitest_coverage_exclusions.py
Normal file
|
|
@ -0,0 +1,125 @@
|
|||
"""Tests for handling Vitest coverage exclusions.
|
||||
|
||||
These tests verify that Codeflash correctly detects and handles files
|
||||
that are excluded from coverage by vitest.config.ts, preventing false
|
||||
0% coverage reports.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from codeflash.models.models import CodeOptimizationContext, CoverageStatus
|
||||
from codeflash.verification.coverage_utils import JestCoverageUtils
|
||||
|
||||
|
||||
class TestVitestCoverageExclusions:
|
||||
"""Tests for Vitest coverage exclusion handling."""
|
||||
|
||||
def test_missing_coverage_returns_not_found_status(self) -> None:
|
||||
"""Should return NOT_FOUND status when file is not in coverage data.
|
||||
|
||||
When a file is excluded from Vitest coverage (via coverage.exclude),
|
||||
it won't appear in coverage-final.json. Codeflash should return
|
||||
NOT_FOUND status (not PARSED_SUCCESSFULLY).
|
||||
|
||||
This test verifies the current behavior is correct at the coverage
|
||||
parsing level. The issue is at a higher level (function_optimizer.py)
|
||||
where NOT_FOUND status needs better handling.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp_path = Path(tmp_dir)
|
||||
|
||||
# Create mock coverage-final.json that's missing the target file
|
||||
coverage_file = tmp_path / "coverage-final.json"
|
||||
coverage_data = {
|
||||
"/workspace/project/src/utils/helpers.ts": {
|
||||
"fnMap": {},
|
||||
"s": {},
|
||||
},
|
||||
# src/agents/sandbox/fs-paths.ts is NOT here (excluded by Vitest)
|
||||
}
|
||||
with coverage_file.open("w") as f:
|
||||
json.dump(coverage_data, f)
|
||||
|
||||
# Try to load coverage for a missing file
|
||||
missing_file = Path("/workspace/project/src/agents/sandbox/fs-paths.ts")
|
||||
from codeflash.models.models import CodeStringsMarkdown
|
||||
|
||||
mock_context = CodeOptimizationContext(
|
||||
testgen_context=CodeStringsMarkdown(language="typescript"),
|
||||
read_writable_code=CodeStringsMarkdown(language="typescript"),
|
||||
helper_functions=[],
|
||||
preexisting_objects=set(),
|
||||
)
|
||||
|
||||
result = JestCoverageUtils.load_from_jest_json(
|
||||
coverage_json_path=coverage_file,
|
||||
function_name="parseSandboxBindMount",
|
||||
code_context=mock_context,
|
||||
source_code_path=missing_file,
|
||||
)
|
||||
|
||||
# Should return NOT_FOUND when file not in coverage
|
||||
assert result.status == CoverageStatus.NOT_FOUND, (
|
||||
f"Expected NOT_FOUND for missing file, got {result.status}"
|
||||
)
|
||||
assert result.coverage == 0.0
|
||||
|
||||
def test_handles_included_file_normally(self) -> None:
|
||||
"""Should handle files that ARE included in coverage normally.
|
||||
|
||||
This test verifies that the fix doesn't break normal coverage parsing
|
||||
for files that are NOT excluded.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
tmp_path = Path(tmp_dir)
|
||||
|
||||
# Create mock coverage-final.json with a valid file
|
||||
coverage_file = tmp_path / "coverage-final.json"
|
||||
test_file = "/workspace/project/src/utils/helpers.ts"
|
||||
coverage_data = {
|
||||
test_file: {
|
||||
"fnMap": {
|
||||
"0": {"name": "someHelper", "loc": {"start": {"line": 1}, "end": {"line": 5}}}
|
||||
},
|
||||
"statementMap": {
|
||||
"0": {"start": {"line": 2}, "end": {"line": 2}},
|
||||
"1": {"start": {"line": 3}, "end": {"line": 3}},
|
||||
},
|
||||
"s": {"0": 5, "1": 5}, # Both statements executed
|
||||
"branchMap": {},
|
||||
"b": {},
|
||||
}
|
||||
}
|
||||
with coverage_file.open("w") as f:
|
||||
json.dump(coverage_data, f)
|
||||
|
||||
source_file = Path(test_file)
|
||||
from codeflash.models.models import CodeStringsMarkdown
|
||||
|
||||
mock_context = CodeOptimizationContext(
|
||||
testgen_context=CodeStringsMarkdown(language="typescript"),
|
||||
read_writable_code=CodeStringsMarkdown(language="typescript"),
|
||||
helper_functions=[],
|
||||
preexisting_objects=set(),
|
||||
)
|
||||
|
||||
result = JestCoverageUtils.load_from_jest_json(
|
||||
coverage_json_path=coverage_file,
|
||||
function_name="someHelper",
|
||||
code_context=mock_context,
|
||||
source_code_path=source_file,
|
||||
)
|
||||
|
||||
# Should parse successfully for non-excluded files
|
||||
assert result.status == CoverageStatus.PARSED_SUCCESSFULLY
|
||||
assert result.coverage > 0.0 # Should have actual coverage
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v"])
|
||||
Loading…
Reference in a new issue