From ec14860d29e6e6f4d1655f67e6d0c89608696050 Mon Sep 17 00:00:00 2001 From: Kevin Turcios Date: Fri, 10 Apr 2026 07:56:02 -0500 Subject: [PATCH] Move benchmarks to .codeflash/benchmarks/ and auto-discover Move codeflash's own benchmarks to .codeflash/benchmarks/. Add auto-discovery of .codeflash/benchmarks/ in codeflash compare and benchmark mode -- when benchmarks-root is not explicitly configured, the CLI checks for .codeflash/benchmarks/ before erroring. Backwards compatible: users with existing benchmarks-root config are unaffected. Docs continue to show tests/benchmarks as the example path. --- {tests => .codeflash}/benchmarks/__init__.py | 0 .../test_benchmark_code_extract_code_context.py | 0 .../benchmarks/test_benchmark_comparator.py | 0 .../benchmarks/test_benchmark_discover_unit_tests.py | 0 .../benchmarks/test_benchmark_libcst_multi_file.py | 0 .../benchmarks/test_benchmark_libcst_pipeline.py | 0 .../benchmarks/test_benchmark_merge_test_results.py | 0 codeflash.code-workspace | 2 +- codeflash/cli_cmds/cli.py | 9 ++++++++- codeflash/cli_cmds/cmd_compare.py | 10 ++++++++-- pyproject.toml | 2 +- 11 files changed, 18 insertions(+), 5 deletions(-) rename {tests => .codeflash}/benchmarks/__init__.py (100%) rename {tests => .codeflash}/benchmarks/test_benchmark_code_extract_code_context.py (100%) rename {tests => .codeflash}/benchmarks/test_benchmark_comparator.py (100%) rename {tests => .codeflash}/benchmarks/test_benchmark_discover_unit_tests.py (100%) rename {tests => .codeflash}/benchmarks/test_benchmark_libcst_multi_file.py (100%) rename {tests => .codeflash}/benchmarks/test_benchmark_libcst_pipeline.py (100%) rename {tests => .codeflash}/benchmarks/test_benchmark_merge_test_results.py (100%) diff --git a/tests/benchmarks/__init__.py b/.codeflash/benchmarks/__init__.py similarity index 100% rename from tests/benchmarks/__init__.py rename to .codeflash/benchmarks/__init__.py diff --git a/tests/benchmarks/test_benchmark_code_extract_code_context.py b/.codeflash/benchmarks/test_benchmark_code_extract_code_context.py similarity index 100% rename from tests/benchmarks/test_benchmark_code_extract_code_context.py rename to .codeflash/benchmarks/test_benchmark_code_extract_code_context.py diff --git a/tests/benchmarks/test_benchmark_comparator.py b/.codeflash/benchmarks/test_benchmark_comparator.py similarity index 100% rename from tests/benchmarks/test_benchmark_comparator.py rename to .codeflash/benchmarks/test_benchmark_comparator.py diff --git a/tests/benchmarks/test_benchmark_discover_unit_tests.py b/.codeflash/benchmarks/test_benchmark_discover_unit_tests.py similarity index 100% rename from tests/benchmarks/test_benchmark_discover_unit_tests.py rename to .codeflash/benchmarks/test_benchmark_discover_unit_tests.py diff --git a/tests/benchmarks/test_benchmark_libcst_multi_file.py b/.codeflash/benchmarks/test_benchmark_libcst_multi_file.py similarity index 100% rename from tests/benchmarks/test_benchmark_libcst_multi_file.py rename to .codeflash/benchmarks/test_benchmark_libcst_multi_file.py diff --git a/tests/benchmarks/test_benchmark_libcst_pipeline.py b/.codeflash/benchmarks/test_benchmark_libcst_pipeline.py similarity index 100% rename from tests/benchmarks/test_benchmark_libcst_pipeline.py rename to .codeflash/benchmarks/test_benchmark_libcst_pipeline.py diff --git a/tests/benchmarks/test_benchmark_merge_test_results.py b/.codeflash/benchmarks/test_benchmark_merge_test_results.py similarity index 100% rename from tests/benchmarks/test_benchmark_merge_test_results.py rename to .codeflash/benchmarks/test_benchmark_merge_test_results.py diff --git a/codeflash.code-workspace b/codeflash.code-workspace index 67f000d35..2c9a31e22 100644 --- a/codeflash.code-workspace +++ b/codeflash.code-workspace @@ -16,7 +16,7 @@ "tests/", "-vv", "--ignore", - "tests/benchmarks/" + ".codeflash/benchmarks/" ], }, "launch": { diff --git a/codeflash/cli_cmds/cli.py b/codeflash/cli_cmds/cli.py index 400403843..2db13efe8 100644 --- a/codeflash/cli_cmds/cli.py +++ b/codeflash/cli_cmds/cli.py @@ -156,7 +156,14 @@ def process_pyproject_config(args: Namespace) -> Namespace: raise AssertionError("--tests-root must be specified") assert Path(args.tests_root).is_dir(), f"--tests-root {args.tests_root} must be a valid directory" if args.benchmark: - assert args.benchmarks_root is not None, "--benchmarks-root must be specified when running with --benchmark" + if args.benchmarks_root is None: + # Auto-discover .codeflash/benchmarks/ convention + candidate = Path.cwd() / ".codeflash" / "benchmarks" + if candidate.is_dir(): + args.benchmarks_root = str(candidate) + else: + msg = "--benchmarks-root must be specified when running with --benchmark, or .codeflash/benchmarks/ must exist" + raise AssertionError(msg) assert Path(args.benchmarks_root).is_dir(), ( f"--benchmarks-root {args.benchmarks_root} must be a valid directory" ) diff --git a/codeflash/cli_cmds/cmd_compare.py b/codeflash/cli_cmds/cmd_compare.py index 87d659fdb..fab917502 100644 --- a/codeflash/cli_cmds/cmd_compare.py +++ b/codeflash/cli_cmds/cmd_compare.py @@ -87,8 +87,14 @@ def run_compare(args: Namespace) -> None: benchmarks_root_str = pyproject_config.get("benchmarks_root") if not benchmarks_root_str: - logger.error("benchmarks-root must be configured in [tool.codeflash] to use compare") - sys.exit(1) + # Auto-discover .codeflash/benchmarks/ if it exists + candidate = project_root / ".codeflash" / "benchmarks" + if candidate.is_dir(): + benchmarks_root_str = str(candidate) + logger.info(f"Auto-discovered benchmarks at {candidate}") + else: + logger.error("benchmarks-root must be configured in [tool.codeflash] or .codeflash/benchmarks/ must exist") + sys.exit(1) benchmarks_root = Path(benchmarks_root_str).resolve() if not benchmarks_root.is_dir(): diff --git a/pyproject.toml b/pyproject.toml index 38256ebfb..7701725ea 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -354,7 +354,7 @@ __version__ = "{version}" # All paths are relative to this pyproject.toml's directory. module-root = "codeflash" tests-root = "tests" -benchmarks-root = "tests/benchmarks" +benchmarks-root = ".codeflash/benchmarks" ignore-paths = [] formatter-cmds = [ "uvx ruff check --exit-zero --fix $file",