codeflash-ai/codeflash
1
0
Fork 0
Code Issues 18 Pull requests 33 Projects Releases Packages Wiki Activity Actions 4.6k

wip instrumentation and execution

Browse source
This commit is contained in:
misrasaurabh1 2026-01-15 12:13:37 -08:00
parent 845dbaf620
commit 30eb5524ba
11 changed files with 915 additions and 97 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
MULTI_LANGUAGE_ARCHITECTURE.md
View file

@ -60,7 +60,7 @@ Verification → Benchmarking → Ranking → PR Creation
│ │ │ - replace_function() │ │
│ │ │ - run_tests() │ │
│ │ │ - discover_tests() │ │
│ │ │ - instrument_for_tracing() │ │
│ │ │ - instrument_for_behavior() │ │
│ │ │ - format_code() │ │
│ │ └──────────────────────────────────────────────────────────┘ │
│ └─────────────────────────────────────────────────────────────────┤
@ -250,7 +250,7 @@ class LanguageSupport(Protocol):
# === Instrumentation ===
def instrument_for_tracing(
def instrument_for_behavior(
self,
file_path: Path,
functions: list[FunctionInfo],
@ -492,7 +492,7 @@ class JavaScriptSupport(LanguageSupport):
results = self.parse_test_results(junit_path, result.stdout)
return results, junit_path
def instrument_for_tracing(
def instrument_for_behavior(
self,
file_path: Path,
functions: list[FunctionInfo],

186
code_to_optimize_js/codeflash-jest-helper.js
View file

@ -7,40 +7,86 @@
* Unlike Python which has separate instrumentation methods for generated
* vs existing tests, this helper works identically for ALL JavaScript tests.
*
* Uses SQLite for consistent data format with Python implementation.
*
* Usage:
* const codeflash = require('codeflash-jest-helper');
* const codeflash = require('./codeflash-jest-helper');
*
* // Wrap function calls to capture behavior
* const result = codeflash.capture('functionName', targetFunction, arg1, arg2);
*
* Environment Variables:
* CODEFLASH_OUTPUT_FILE - Path to write results (default: /tmp/codeflash_results.bin)
* CODEFLASH_LOOP_INDEX - Current benchmark loop iteration (default: 0)
* CODEFLASH_MODE - Testing mode: 'behavior' or 'performance' (default: 'behavior')
* CODEFLASH_OUTPUT_FILE - Path to write results SQLite file
* CODEFLASH_LOOP_INDEX - Current benchmark loop iteration (default: 1)
* CODEFLASH_TEST_ITERATION - Test iteration number (default: 0)
* CODEFLASH_TEST_MODULE - Test module path
*/
const fs = require('fs');
const path = require('path');
const { performance } = require('perf_hooks');
// Try to load better-sqlite3, fall back to JSON if not available
let Database;
let useSqlite = false;
try {
Database = require('better-sqlite3');
useSqlite = true;
} catch (e) {
// better-sqlite3 not available, will use JSON fallback
console.warn('[codeflash] better-sqlite3 not found, using JSON fallback');
}
// Configuration from environment
const OUTPUT_FILE = process.env.CODEFLASH_OUTPUT_FILE || '/tmp/codeflash_results.bin';
const LOOP_INDEX = parseInt(process.env.CODEFLASH_LOOP_INDEX || '0', 10);
const MODE = process.env.CODEFLASH_MODE || 'behavior';
const OUTPUT_FILE = process.env.CODEFLASH_OUTPUT_FILE || '/tmp/codeflash_results.sqlite';
const LOOP_INDEX = parseInt(process.env.CODEFLASH_LOOP_INDEX || '1', 10);
const TEST_ITERATION = process.env.CODEFLASH_TEST_ITERATION || '0';
const TEST_MODULE = process.env.CODEFLASH_TEST_MODULE || '';
// Current test context
let currentTestName = null;
let invocationCounter = 0;
let lineId = '0';
// Results buffer
// Results buffer (for JSON fallback)
const results = [];
// SQLite database (lazy initialized)
let db = null;
/**
* Safely serialize a value to JSON.
* Handles circular references and special types.
* Initialize the SQLite database.
*/
function initDatabase() {
if (!useSqlite || db) return;
try {
db = new Database(OUTPUT_FILE);
db.exec(`
CREATE TABLE IF NOT EXISTS test_results (
test_module_path TEXT,
test_class_name TEXT,
test_function_name TEXT,
function_getting_tested TEXT,
loop_index INTEGER,
iteration_id TEXT,
runtime INTEGER,
return_value BLOB,
verification_type TEXT
)
`);
} catch (e) {
console.error('[codeflash] Failed to initialize SQLite:', e.message);
useSqlite = false;
}
}
/**
* Safely serialize a value for storage.
* Uses JSON serialization with special handling for complex types.
*
* @param {any} value - Value to serialize
* @returns {any} - Serializable representation
* @returns {Buffer} - Serialized value as Buffer
*/
function safeSerialize(value) {
const seen = new WeakSet();
@ -94,14 +140,15 @@ function safeSerialize(value) {
}
try {
return serialize(value);
const serialized = serialize(value);
return Buffer.from(JSON.stringify(serialized));
} catch (e) {
return { __type: 'SerializationError', error: e.message };
return Buffer.from(JSON.stringify({ __type: 'SerializationError', error: e.message }));
}
}
/**
* Record a test result.
* Record a test result to SQLite or JSON buffer.
*
* @param {string} funcName - Name of the function being tested
* @param {Array} args - Arguments passed to the function
@ -110,23 +157,69 @@ function safeSerialize(value) {
* @param {number} durationNs - Execution time in nanoseconds
*/
function recordResult(funcName, args, returnValue, error, durationNs) {
const result = {
testName: currentTestName,
funcName,
args: safeSerialize(args),
returnValue: safeSerialize(returnValue),
error: error ? {
name: error.name,
message: error.message,
stack: error.stack
} : null,
durationNs: Math.round(durationNs),
invocationId: invocationCounter++,
loopIndex: LOOP_INDEX,
mode: MODE,
timestamp: Date.now()
};
results.push(result);
const invocationId = `${lineId}_${invocationCounter}`;
invocationCounter++;
// Get test module path from file being tested or env
const testModulePath = TEST_MODULE || currentTestName || 'unknown';
// Serialize the return value (args, kwargs (empty for JS), return_value) like Python does
const serializedValue = error
? safeSerialize(error)
: safeSerialize([args, {}, returnValue]);
if (useSqlite && db) {
try {
const stmt = db.prepare(`
INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
`);
stmt.run(
testModulePath, // test_module_path
null, // test_class_name (Jest doesn't use classes like Python)
currentTestName, // test_function_name
funcName, // function_getting_tested
LOOP_INDEX, // loop_index
invocationId, // iteration_id
Math.round(durationNs), // runtime (nanoseconds)
serializedValue, // return_value (serialized)
'function_call' // verification_type
);
} catch (e) {
console.error('[codeflash] Failed to write to SQLite:', e.message);
// Fall back to JSON
results.push({
testModulePath,
testClassName: null,
testFunctionName: currentTestName,
funcName,
loopIndex: LOOP_INDEX,
iterationId: invocationId,
durationNs: Math.round(durationNs),
returnValue: error ? null : returnValue,
error: error ? { name: error.name, message: error.message } : null,
verificationType: 'function_call'
});
}
} else {
// JSON fallback
results.push({
testModulePath,
testClassName: null,
testFunctionName: currentTestName,
funcName,
loopIndex: LOOP_INDEX,
iterationId: invocationId,
durationNs: Math.round(durationNs),
returnValue: error ? null : returnValue,
error: error ? { name: error.name, message: error.message } : null,
verificationType: 'function_call'
});
}
// Print stdout tag like Python does for test identification
const testClassName = '';
const testStdoutTag = `${testModulePath}:${testClassName}${currentTestName}:${funcName}:${LOOP_INDEX}:${invocationId}`;
console.log(`!$######${testStdoutTag}######$!`);
}
/**
@ -142,6 +235,9 @@ function recordResult(funcName, args, returnValue, error, durationNs) {
* @throws {Error} - Re-throws any error from the function
*/
function capture(funcName, fn, ...args) {
// Initialize database on first capture
initDatabase();
const startTime = performance.now();
let returnValue;
let error = null;
@ -191,24 +287,36 @@ function captureMultiple(funcName, fn, argsList) {
}
/**
* Write results to output file.
* Write remaining JSON results to file (fallback mode).
* Called automatically via Jest afterAll hook.
*/
function writeResults() {
// Close SQLite connection if open
if (db) {
try {
db.close();
} catch (e) {
// Ignore close errors
}
db = null;
return;
}
// Write JSON fallback if SQLite wasn't used
if (results.length === 0) return;
try {
// Write as JSON for fallback parsing
const jsonPath = OUTPUT_FILE.replace('.sqlite', '.json');
const output = {
version: '1.0.0',
mode: MODE,
loopIndex: LOOP_INDEX,
timestamp: Date.now(),
results
};
const buffer = Buffer.from(JSON.stringify(output, null, 2));
fs.writeFileSync(OUTPUT_FILE, buffer);
fs.writeFileSync(jsonPath, JSON.stringify(output, null, 2));
} catch (e) {
console.error('[codeflash] Error writing results:', e.message);
console.error('[codeflash] Error writing JSON results:', e.message);
}
}
@ -252,6 +360,7 @@ if (typeof beforeEach !== 'undefined') {
currentTestName = 'unknown';
}
invocationCounter = 0;
lineId = String(Date.now() % 1000000); // Unique line ID per test
});
}
@ -270,8 +379,9 @@ module.exports = {
getResults,
setTestName,
safeSerialize,
initDatabase,
// Constants
MODE,
LOOP_INDEX,
OUTPUT_FILE
OUTPUT_FILE,
TEST_ITERATION
};

436
code_to_optimize_js/package-lock.json generated
View file

@ -8,6 +8,9 @@
"name": "codeflash-js-test",
"version": "1.0.0",
"license": "MIT",
"dependencies": {
"better-sqlite3": "^12.6.0"
},
"devDependencies": {
"jest": "^29.7.0",
"jest-junit": "^16.0.0"
@ -1210,6 +1213,26 @@
"dev": true,
"license": "MIT"
},
"node_modules/base64-js": {
"version": "1.5.1",
"resolved": "https://registry.npmjs.org/base64-js/-/base64-js-1.5.1.tgz",
"integrity": "sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "MIT"
},
"node_modules/baseline-browser-mapping": {
"version": "2.9.14",
"resolved": "https://registry.npmjs.org/baseline-browser-mapping/-/baseline-browser-mapping-2.9.14.tgz",
@ -1220,6 +1243,40 @@
"baseline-browser-mapping": "dist/cli.js"
}
},
"node_modules/better-sqlite3": {
"version": "12.6.0",
"resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.6.0.tgz",
"integrity": "sha512-FXI191x+D6UPWSze5IzZjhz+i9MK9nsuHsmTX9bXVl52k06AfZ2xql0lrgIUuzsMsJ7Vgl5kIptvDgBLIV3ZSQ==",
"hasInstallScript": true,
"license": "MIT",
"dependencies": {
"bindings": "^1.5.0",
"prebuild-install": "^7.1.1"
},
"engines": {
"node": "20.x || 22.x || 23.x || 24.x || 25.x"
}
},
"node_modules/bindings": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.5.0.tgz",
"integrity": "sha512-p2q/t/mhvuOj/UeLlV6566GD/guowlr0hHxClI0W9m7MWYkL1F0hLo+0Aexs9HSPCtR1SXQ0TD3MMKrXZajbiQ==",
"license": "MIT",
"dependencies": {
"file-uri-to-path": "1.0.0"
}
},
"node_modules/bl": {
"version": "4.1.0",
"resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz",
"integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==",
"license": "MIT",
"dependencies": {
"buffer": "^5.5.0",
"inherits": "^2.0.4",
"readable-stream": "^3.4.0"
}
},
"node_modules/brace-expansion": {
"version": "1.1.12",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz",
@ -1288,6 +1345,30 @@
"node-int64": "^0.4.0"
}
},
"node_modules/buffer": {
"version": "5.7.1",
"resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz",
"integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "MIT",
"dependencies": {
"base64-js": "^1.3.1",
"ieee754": "^1.1.13"
}
},
"node_modules/buffer-from": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz",
@ -1363,6 +1444,12 @@
"node": ">=10"
}
},
"node_modules/chownr": {
"version": "1.1.4",
"resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz",
"integrity": "sha512-jJ0bqzaylmJtVnNgzTeSOs8DPavpbYgEr/b0YL8/2GO3xJEhInFmhKMUnEJQjZumK7KXGFhUy89PrsJWlakBVg==",
"license": "ISC"
},
"node_modules/ci-info": {
"version": "3.9.0",
"resolved": "https://registry.npmjs.org/ci-info/-/ci-info-3.9.0.tgz",
@ -1508,6 +1595,21 @@
}
}
},
"node_modules/decompress-response": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/decompress-response/-/decompress-response-6.0.0.tgz",
"integrity": "sha512-aW35yZM6Bb/4oJlZncMH2LCoZtJXTRxES17vE3hoRiowU2kWHaJKFkSBDnDR+cm9J+9QhXmREyIfv0pji9ejCQ==",
"license": "MIT",
"dependencies": {
"mimic-response": "^3.1.0"
},
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/dedent": {
"version": "1.7.1",
"resolved": "https://registry.npmjs.org/dedent/-/dedent-1.7.1.tgz",
@ -1523,6 +1625,15 @@
}
}
},
"node_modules/deep-extend": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/deep-extend/-/deep-extend-0.6.0.tgz",
"integrity": "sha512-LOHxIOaPYdHlJRtCQfDIVZtfw/ufM8+rVj649RIHzcm/vGwQRXFt6OPqIFWsm2XEMrNIEtWR64sY1LEKD2vAOA==",
"license": "MIT",
"engines": {
"node": ">=4.0.0"
}
},
"node_modules/deepmerge": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/deepmerge/-/deepmerge-4.3.1.tgz",
@ -1533,6 +1644,15 @@
"node": ">=0.10.0"
}
},
"node_modules/detect-libc": {
"version": "2.1.2",
"resolved": "https://registry.npmjs.org/detect-libc/-/detect-libc-2.1.2.tgz",
"integrity": "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ==",
"license": "Apache-2.0",
"engines": {
"node": ">=8"
}
},
"node_modules/detect-newline": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/detect-newline/-/detect-newline-3.1.0.tgz",
@ -1580,6 +1700,15 @@
"dev": true,
"license": "MIT"
},
"node_modules/end-of-stream": {
"version": "1.4.5",
"resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz",
"integrity": "sha512-ooEGc6HP26xXq/N+GCGOT0JKCLDGrq2bQUZrQ7gyrJiZANJ/8YDTxTpQBXGMn+WbIQXNVpyWymm7KYVICQnyOg==",
"license": "MIT",
"dependencies": {
"once": "^1.4.0"
}
},
"node_modules/error-ex": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/error-ex/-/error-ex-1.3.4.tgz",
@ -1657,6 +1786,15 @@
"node": ">= 0.8.0"
}
},
"node_modules/expand-template": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/expand-template/-/expand-template-2.0.3.tgz",
"integrity": "sha512-XYfuKMvj4O35f/pOXLObndIRvyQ+/+6AhODh+OKWj9S9498pHHn/IMszH+gt0fBCRWMNfk1ZSp5x3AifmnI2vg==",
"license": "(MIT OR WTFPL)",
"engines": {
"node": ">=6"
}
},
"node_modules/expect": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/expect/-/expect-29.7.0.tgz",
@ -1691,6 +1829,12 @@
"bser": "2.1.1"
}
},
"node_modules/file-uri-to-path": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz",
"integrity": "sha512-0Zt+s3L7Vf1biwWZ29aARiVYLx7iMGnEUl9x33fbB/j3jR81u/O2LbqK+Bm1CDSNDKVtJ/YjwY7TUd5SkeLQLw==",
"license": "MIT"
},
"node_modules/fill-range": {
"version": "7.1.1",
"resolved": "https://registry.npmjs.org/fill-range/-/fill-range-7.1.1.tgz",
@ -1718,6 +1862,12 @@
"node": ">=8"
}
},
"node_modules/fs-constants": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs-constants/-/fs-constants-1.0.0.tgz",
"integrity": "sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow==",
"license": "MIT"
},
"node_modules/fs.realpath": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz",
@ -1793,6 +1943,12 @@
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/github-from-package": {
"version": "0.0.0",
"resolved": "https://registry.npmjs.org/github-from-package/-/github-from-package-0.0.0.tgz",
"integrity": "sha512-SyHy3T1v2NUXn29OsWdxmK6RwHD+vkj3v8en8AOBZ1wBQ/hCAQ5bAQTD02kW4W9tUp/3Qh6J8r9EvntiyCmOOw==",
"license": "MIT"
},
"node_modules/glob": {
"version": "7.2.3",
"resolved": "https://registry.npmjs.org/glob/-/glob-7.2.3.tgz",
@ -1862,6 +2018,26 @@
"node": ">=10.17.0"
}
},
"node_modules/ieee754": {
"version": "1.2.1",
"resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz",
"integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "BSD-3-Clause"
},
"node_modules/import-local": {
"version": "3.2.0",
"resolved": "https://registry.npmjs.org/import-local/-/import-local-3.2.0.tgz",
@ -1908,7 +2084,12 @@
"version": "2.0.4",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz",
"integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==",
"dev": true,
"license": "ISC"
},
"node_modules/ini": {
"version": "1.3.8",
"resolved": "https://registry.npmjs.org/ini/-/ini-1.3.8.tgz",
"integrity": "sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==",
"license": "ISC"
},
"node_modules/is-arrayish": {
@ -2854,6 +3035,18 @@
"node": ">=6"
}
},
"node_modules/mimic-response": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/mimic-response/-/mimic-response-3.1.0.tgz",
"integrity": "sha512-z0yWI+4FDrrweS8Zmt4Ej5HdJmky15+L2e6Wgn3+iK5fWzb6T3fhNFq2+MeTRb064c6Wr4N/wv0DzQTjNzHNGQ==",
"license": "MIT",
"engines": {
"node": ">=10"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/minimatch": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.1.2.tgz",
@ -2867,6 +3060,15 @@
"node": "*"
}
},
"node_modules/minimist": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.8.tgz",
"integrity": "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA==",
"license": "MIT",
"funding": {
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/mkdirp": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-1.0.4.tgz",
@ -2880,6 +3082,12 @@
"node": ">=10"
}
},
"node_modules/mkdirp-classic": {
"version": "0.5.3",
"resolved": "https://registry.npmjs.org/mkdirp-classic/-/mkdirp-classic-0.5.3.tgz",
"integrity": "sha512-gKLcREMhtuZRwRAfqP3RFW+TK4JqApVBtOIftVgjuABpAtpxhPGaDcfvbhNvD0B8iD1oUr/txX35NjcaY6Ns/A==",
"license": "MIT"
},
"node_modules/ms": {
"version": "2.1.3",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@ -2887,6 +3095,12 @@
"dev": true,
"license": "MIT"
},
"node_modules/napi-build-utils": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/napi-build-utils/-/napi-build-utils-2.0.0.tgz",
"integrity": "sha512-GEbrYkbfF7MoNaoh2iGG84Mnf/WZfB0GdGEsM8wz7Expx/LlWf5U8t9nvJKXSp3qr5IsEbK04cBGhol/KwOsWA==",
"license": "MIT"
},
"node_modules/natural-compare": {
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/natural-compare/-/natural-compare-1.4.0.tgz",
@ -2894,6 +3108,30 @@
"dev": true,
"license": "MIT"
},
"node_modules/node-abi": {
"version": "3.85.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.85.0.tgz",
"integrity": "sha512-zsFhmbkAzwhTft6nd3VxcG0cvJsT70rL+BIGHWVq5fi6MwGrHwzqKaxXE+Hl2GmnGItnDKPPkO5/LQqjVkIdFg==",
"license": "MIT",
"dependencies": {
"semver": "^7.3.5"
},
"engines": {
"node": ">=10"
}
},
"node_modules/node-abi/node_modules/semver": {
"version": "7.7.3",
"resolved": "https://registry.npmjs.org/semver/-/semver-7.7.3.tgz",
"integrity": "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q==",
"license": "ISC",
"bin": {
"semver": "bin/semver.js"
},
"engines": {
"node": ">=10"
}
},
"node_modules/node-int64": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/node-int64/-/node-int64-0.4.0.tgz",
@ -2935,7 +3173,6 @@
"version": "1.4.0",
"resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz",
"integrity": "sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==",
"dev": true,
"license": "ISC",
"dependencies": {
"wrappy": "1"
@ -3111,6 +3348,32 @@
"node": ">=8"
}
},
"node_modules/prebuild-install": {
"version": "7.1.3",
"resolved": "https://registry.npmjs.org/prebuild-install/-/prebuild-install-7.1.3.tgz",
"integrity": "sha512-8Mf2cbV7x1cXPUILADGI3wuhfqWvtiLA1iclTDbFRZkgRQS0NqsPZphna9V+HyTEadheuPmjaJMsbzKQFOzLug==",
"license": "MIT",
"dependencies": {
"detect-libc": "^2.0.0",
"expand-template": "^2.0.3",
"github-from-package": "0.0.0",
"minimist": "^1.2.3",
"mkdirp-classic": "^0.5.3",
"napi-build-utils": "^2.0.0",
"node-abi": "^3.3.0",
"pump": "^3.0.0",
"rc": "^1.2.7",
"simple-get": "^4.0.0",
"tar-fs": "^2.0.0",
"tunnel-agent": "^0.6.0"
},
"bin": {
"prebuild-install": "bin.js"
},
"engines": {
"node": ">=10"
}
},
"node_modules/pretty-format": {
"version": "29.7.0",
"resolved": "https://registry.npmjs.org/pretty-format/-/pretty-format-29.7.0.tgz",
@ -3153,6 +3416,16 @@
"node": ">= 6"
}
},
"node_modules/pump": {
"version": "3.0.3",
"resolved": "https://registry.npmjs.org/pump/-/pump-3.0.3.tgz",
"integrity": "sha512-todwxLMY7/heScKmntwQG8CXVkWUOdYxIvY2s0VWAAMh/nd8SoYiRaKjlr7+iCs984f2P8zvrfWcDDYVb73NfA==",
"license": "MIT",
"dependencies": {
"end-of-stream": "^1.1.0",
"once": "^1.3.1"
}
},
"node_modules/pure-rand": {
"version": "6.1.0",
"resolved": "https://registry.npmjs.org/pure-rand/-/pure-rand-6.1.0.tgz",
@ -3170,6 +3443,30 @@
],
"license": "MIT"
},
"node_modules/rc": {
"version": "1.2.8",
"resolved": "https://registry.npmjs.org/rc/-/rc-1.2.8.tgz",
"integrity": "sha512-y3bGgqKj3QBdxLbLkomlohkvsA8gdAiUQlSBJnBhfn+BPxg4bc62d8TcBW15wavDfgexCgccckhcZvywyQYPOw==",
"license": "(BSD-2-Clause OR MIT OR Apache-2.0)",
"dependencies": {
"deep-extend": "^0.6.0",
"ini": "~1.3.0",
"minimist": "^1.2.0",
"strip-json-comments": "~2.0.1"
},
"bin": {
"rc": "cli.js"
}
},
"node_modules/rc/node_modules/strip-json-comments": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/strip-json-comments/-/strip-json-comments-2.0.1.tgz",
"integrity": "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ==",
"license": "MIT",
"engines": {
"node": ">=0.10.0"
}
},
"node_modules/react-is": {
"version": "18.3.1",
"resolved": "https://registry.npmjs.org/react-is/-/react-is-18.3.1.tgz",
@ -3177,6 +3474,20 @@
"dev": true,
"license": "MIT"
},
"node_modules/readable-stream": {
"version": "3.6.2",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.2.tgz",
"integrity": "sha512-9u/sniCrY3D5WdsERHzHE4G2YCXqoG5FTHUiCC4SIbr6XcLZBY05ya9EKjYek9O5xOAwjGq+1JdGBAS7Q9ScoA==",
"license": "MIT",
"dependencies": {
"inherits": "^2.0.3",
"string_decoder": "^1.1.1",
"util-deprecate": "^1.0.1"
},
"engines": {
"node": ">= 6"
}
},
"node_modules/require-directory": {
"version": "2.1.1",
"resolved": "https://registry.npmjs.org/require-directory/-/require-directory-2.1.1.tgz",
@ -3241,6 +3552,26 @@
"node": ">=10"
}
},
"node_modules/safe-buffer": {
"version": "5.2.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz",
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "MIT"
},
"node_modules/semver": {
"version": "6.3.1",
"resolved": "https://registry.npmjs.org/semver/-/semver-6.3.1.tgz",
@ -3281,6 +3612,51 @@
"dev": true,
"license": "ISC"
},
"node_modules/simple-concat": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/simple-concat/-/simple-concat-1.0.1.tgz",
"integrity": "sha512-cSFtAPtRhljv69IK0hTVZQ+OfE9nePi/rtJmw5UjHeVyVroEqJXP1sFztKUy1qU+xvz3u/sfYJLa947b7nAN2Q==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "MIT"
},
"node_modules/simple-get": {
"version": "4.0.1",
"resolved": "https://registry.npmjs.org/simple-get/-/simple-get-4.0.1.tgz",
"integrity": "sha512-brv7p5WgH0jmQJr1ZDDfKDOSeWWg+OVypG99A/5vYGPqJ6pxiaHLy8nxtFjBA7oMa01ebA9gfh1uMCFqOuXxvA==",
"funding": [
{
"type": "github",
"url": "https://github.com/sponsors/feross"
},
{
"type": "patreon",
"url": "https://www.patreon.com/feross"
},
{
"type": "consulting",
"url": "https://feross.org/support"
}
],
"license": "MIT",
"dependencies": {
"decompress-response": "^6.0.0",
"once": "^1.3.1",
"simple-concat": "^1.0.0"
}
},
"node_modules/sisteransi": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/sisteransi/-/sisteransi-1.0.5.tgz",
@ -3339,6 +3715,15 @@
"node": ">=10"
}
},
"node_modules/string_decoder": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz",
"integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==",
"license": "MIT",
"dependencies": {
"safe-buffer": "~5.2.0"
}
},
"node_modules/string-length": {
"version": "4.0.2",
"resolved": "https://registry.npmjs.org/string-length/-/string-length-4.0.2.tgz",
@ -3440,6 +3825,34 @@
"url": "https://github.com/sponsors/ljharb"
}
},
"node_modules/tar-fs": {
"version": "2.1.4",
"resolved": "https://registry.npmjs.org/tar-fs/-/tar-fs-2.1.4.tgz",
"integrity": "sha512-mDAjwmZdh7LTT6pNleZ05Yt65HC3E+NiQzl672vQG38jIrehtJk/J3mNwIg+vShQPcLF/LV7CMnDW6vjj6sfYQ==",
"license": "MIT",
"dependencies": {
"chownr": "^1.1.1",
"mkdirp-classic": "^0.5.2",
"pump": "^3.0.0",
"tar-stream": "^2.1.4"
}
},
"node_modules/tar-stream": {
"version": "2.2.0",
"resolved": "https://registry.npmjs.org/tar-stream/-/tar-stream-2.2.0.tgz",
"integrity": "sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ==",
"license": "MIT",
"dependencies": {
"bl": "^4.0.3",
"end-of-stream": "^1.4.1",
"fs-constants": "^1.0.0",
"inherits": "^2.0.3",
"readable-stream": "^3.1.1"
},
"engines": {
"node": ">=6"
}
},
"node_modules/test-exclude": {
"version": "6.0.0",
"resolved": "https://registry.npmjs.org/test-exclude/-/test-exclude-6.0.0.tgz",
@ -3475,6 +3888,18 @@
"node": ">=8.0"
}
},
"node_modules/tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha512-McnNiV1l8RYeY8tBgEpuodCC1mLUdbSN+CYBL7kJsJNInOP8UjDDEwdk6Mw60vdLLrr5NHKZhMAOSrR2NZuQ+w==",
"license": "Apache-2.0",
"dependencies": {
"safe-buffer": "^5.0.1"
},
"engines": {
"node": "*"
}
},
"node_modules/type-detect": {
"version": "4.0.8",
"resolved": "https://registry.npmjs.org/type-detect/-/type-detect-4.0.8.tgz",
@ -3536,6 +3961,12 @@
"browserslist": ">= 4.21.0"
}
},
"node_modules/util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw==",
"license": "MIT"
},
"node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
@ -3609,7 +4040,6 @@
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz",
"integrity": "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ==",
"dev": true,
"license": "ISC"
},
"node_modules/write-file-atomic": {

29
code_to_optimize_js/package.json
View file

@ -7,7 +7,11 @@
"test": "jest",
"test:coverage": "jest --coverage"
},
"keywords": ["codeflash", "optimization", "testing"],
"keywords": [
"codeflash",
"optimization",
"testing"
],
"author": "CodeFlash Inc.",
"license": "MIT",
"devDependencies": {
@ -16,14 +20,25 @@
},
"jest": {
"testEnvironment": "node",
"testMatch": ["**/tests/**/*.test.js"],
"collectCoverageFrom": ["*.js", "!jest.config.js"],
"testMatch": [
"**/tests/**/*.test.js"
],
"collectCoverageFrom": [
"*.js",
"!jest.config.js"
],
"reporters": [
"default",
["jest-junit", {
"outputDirectory": ".codeflash",
"outputName": "jest-results.xml"
}]
[
"jest-junit",
{
"outputDirectory": ".codeflash",
"outputName": "jest-results.xml"
}
]
]
},
"dependencies": {
"better-sqlite3": "^12.6.0"
}
}

6
codeflash/languages/base.py
View file

@ -442,17 +442,17 @@ class LanguageSupport(Protocol):
# === Instrumentation ===
def instrument_for_tracing(
def instrument_for_behavior(
self,
source: str,
functions: Sequence[FunctionInfo],
) -> str:
"""
Add tracing instrumentation to capture inputs/outputs.
Add behavior instrumentation to capture inputs/outputs.
Args:
source: Source code to instrument.
functions: Functions to add tracing to.
functions: Functions to add behavior capture.
Returns:
Instrumented source code.

4
codeflash/languages/javascript/support.py
View file

@ -541,10 +541,10 @@ class JavaScriptSupport:
# === Instrumentation ===
def instrument_for_tracing(
def instrument_for_behavior(
self, source: str, functions: Sequence[FunctionInfo], output_file: Path | None = None
) -> str:
"""Add tracing instrumentation to capture inputs/outputs.
"""Add behavior instrumentation to capture inputs/outputs.
For JavaScript, this wraps functions to capture their arguments
and return values.

6
codeflash/languages/python/support.py
View file

@ -459,19 +459,19 @@ class PythonSupport:
# === Instrumentation ===
def instrument_for_tracing(
def instrument_for_behavior(
self,
source: str,
functions: Sequence[FunctionInfo],
) -> str:
"""
Add tracing instrumentation to capture inputs/outputs.
Add behavior instrumentation to capture inputs/outputs.
For Python, this adds decorators to wrap function calls.
Args:
source: Source code to instrument.
functions: Functions to add tracing to.
functions: Functions to add behavior capture.
Returns:
Instrumented source code.

29
codeflash/optimization/function_optimizer.py
View file

@ -2116,18 +2116,25 @@ class FunctionOptimizer:
)
console.rule()
return Failure("Failed to establish a baseline for the original code - bevhavioral tests failed.")
if not coverage_critic(coverage_results):
did_pass_all_tests = all(result.did_pass for result in behavioral_results)
if not did_pass_all_tests:
return Failure("Tests failed to pass for the original code.")
return Failure(
f"Test coverage is {coverage_results.coverage}%, which is below the required threshold of {COVERAGE_THRESHOLD}%."
)
# Skip coverage check for JavaScript/TypeScript (coverage not yet supported)
if self.function_to_optimize.language not in ("javascript", "typescript"):
if not coverage_critic(coverage_results):
did_pass_all_tests = all(result.did_pass for result in behavioral_results)
if not did_pass_all_tests:
return Failure("Tests failed to pass for the original code.")
coverage_pct = coverage_results.coverage if coverage_results else 0
return Failure(
f"Test coverage is {coverage_pct}%, which is below the required threshold of {COVERAGE_THRESHOLD}%."
)
with progress_bar("Running line profiler to identify performance bottlenecks..."):
line_profile_results = self.line_profiler_step(
code_context=code_context, original_helper_code=original_helper_code, candidate_index=0
)
# Skip line profiler for JavaScript/TypeScript (not yet supported)
if self.function_to_optimize.language in ("javascript", "typescript"):
line_profile_results = None
else:
with progress_bar("Running line profiler to identify performance bottlenecks..."):
line_profile_results = self.line_profiler_step(
code_context=code_context, original_helper_code=original_helper_code, candidate_index=0
)
console.rule()
with progress_bar("Running performance benchmarks..."):
if self.function_to_optimize.is_async:

205
codeflash/verification/parse_test_output.py
View file

@ -72,6 +72,7 @@ def resolve_test_file_from_class_path(test_class_path: str, base_dir: Path) -> P
Args:
test_class_path: The full class path from pytest (e.g., "project.tests.test_file.TestClass")
or a file path from Jest (e.g., "tests/test_file.test.js")
base_dir: The base directory for tests (tests project root)
Returns:
@ -83,7 +84,25 @@ def resolve_test_file_from_class_path(test_class_path: str, base_dir: Path) -> P
>>> # Should find: /path/to/tests/unittest/test_file.py
"""
# First try the full path
# Handle JavaScript file paths (contain slashes and .js/.ts extension)
if "/" in test_class_path or "\\" in test_class_path:
# This is a file path, not a Python module path
# Try to resolve relative to base_dir's parent (project root)
project_root = base_dir.parent
potential_path = project_root / test_class_path
if potential_path.exists():
return potential_path
# Also try relative to base_dir itself
potential_path = base_dir / test_class_path
if potential_path.exists():
return potential_path
# Try the path as-is if it's absolute
potential_path = Path(test_class_path)
if potential_path.exists():
return potential_path
return None
# First try the full path (Python module path)
test_file_path = file_name_from_test_module_name(test_class_path, base_dir)
# If we couldn't find the file, try stripping the last component (likely a class name)
@ -114,6 +133,97 @@ def resolve_test_file_from_class_path(test_class_path: str, base_dir: Path) -> P
return test_file_path
def parse_jest_json_results(
file_location: Path,
test_files: TestFiles,
test_config: TestConfig,
function_name: str | None = None,
) -> TestResults:
"""Parse Jest test results from JSON format written by codeflash-jest-helper.
Args:
file_location: Path to the JSON results file.
test_files: TestFiles object containing test file information.
test_config: Test configuration.
function_name: Name of the function being tested.
Returns:
TestResults containing parsed test invocations.
"""
import json
test_results = TestResults()
if not file_location.exists():
logger.debug(f"No Jest JSON results at {file_location}")
return test_results
try:
with file_location.open("r") as f:
data = json.load(f)
results = data.get("results", [])
for result in results:
test_name = result.get("testName", "")
func_name = result.get("funcName", "")
duration_ns = result.get("durationNs", 0)
loop_index = result.get("loopIndex", 1)
invocation_id = result.get("invocationId", 0)
error = result.get("error")
# Try to find the test file from test_files
# Check both behavior and benchmarking paths since the same parser is used for both
test_file_path = None
test_type = TestType.GENERATED_REGRESSION # Default for Jest generated tests
for test_file in test_files.test_files:
# Check benchmarking path first (used for performance tests)
if test_file.benchmarking_file_path and test_file.benchmarking_file_path.exists():
test_file_path = test_file.benchmarking_file_path
test_type = test_file.test_type
break
# Fall back to behavior path (used for behavior tests)
if test_file.instrumented_behavior_file_path and test_file.instrumented_behavior_file_path.exists():
test_file_path = test_file.instrumented_behavior_file_path
test_type = test_file.test_type
break
if test_file_path is None:
logger.debug(f"Could not find test file for Jest result: {test_name}")
continue
# Create invocation ID - use funcName from result or passed function_name
function_getting_tested = func_name or function_name or "unknown"
test_module_path = module_name_from_file_path(test_file_path, test_config.tests_project_rootdir)
invocation_id_obj = InvocationId(
test_module_path=test_module_path,
test_class_name=None,
test_function_name=test_name or func_name,
function_getting_tested=function_getting_tested,
iteration_id=str(invocation_id),
)
test_results.add(
function_test_invocation=FunctionTestInvocation(
loop_index=loop_index,
id=invocation_id_obj,
file_name=test_file_path,
did_pass=error is None,
runtime=duration_ns,
test_framework=test_config.test_framework,
test_type=test_type,
return_value=result.get("returnValue"),
timed_out=False,
verification_type=VerificationType.FUNCTION_CALL,
)
)
except Exception as e:
logger.warning(f"Failed to parse Jest JSON results from {file_location}: {e}")
return test_results
def parse_test_return_values_bin(file_location: Path, test_files: TestFiles, test_config: TestConfig) -> TestResults:
test_results = TestResults()
if not file_location.exists():
@ -196,13 +306,26 @@ def parse_sqlite_test_results(sqlite_file_path: Path, test_files: TestFiles, tes
return test_results
finally:
db.close()
# Check if this is a JavaScript test (use JSON) or Python test (use pickle)
is_javascript = test_config.test_framework == "jest"
for val in data:
try:
test_module_path = val[0]
test_class_name = val[1] if val[1] else None
test_function_name = val[2] if val[2] else None
function_getting_tested = val[3]
test_file_path = file_path_from_module_name(test_module_path, test_config.tests_project_rootdir)
# For JavaScript, test_module_path is already a file path (e.g., "tests/foo.test.js")
# For Python, it's a module path (e.g., "tests.test_foo") that needs conversion
if is_javascript:
# JavaScript: test_module_path is a relative file path
test_file_path = test_config.tests_project_rootdir / test_module_path
else:
# Python: convert module path to file path
test_file_path = file_path_from_module_name(test_module_path, test_config.tests_project_rootdir)
loop_index = val[4]
iteration_id = val[5]
runtime = val[6]
@ -212,10 +335,34 @@ def parse_sqlite_test_results(sqlite_file_path: Path, test_files: TestFiles, tes
else:
# TODO : this is because sqlite writes original file module path. Should make it consistent
test_type = test_files.get_test_type_by_original_file_path(test_file_path)
try:
ret_val = (pickle.loads(val[7]) if loop_index == 1 else None,)
except Exception: # noqa: S112
continue
# Default to GENERATED_REGRESSION for JavaScript tests when test type can't be determined
if test_type is None and is_javascript:
test_type = TestType.GENERATED_REGRESSION
elif test_type is None:
# Skip results where test type cannot be determined
logger.debug(f"Skipping result for {test_function_name}: could not determine test type")
continue
# Deserialize return value - try JSON first (for JavaScript), then pickle (for Python)
ret_val = None
if loop_index == 1 and val[7]:
try:
if is_javascript:
# JavaScript uses JSON serialization
import json
# val[7] might be bytes or string
raw_value = val[7]
if isinstance(raw_value, bytes):
raw_value = raw_value.decode('utf-8')
ret_val = (json.loads(raw_value),)
else:
# Python uses pickle serialization
ret_val = (pickle.loads(val[7]),)
except Exception as e: # noqa: S112
# If deserialization fails, skip this result
logger.debug(f"Failed to deserialize return value for {test_function_name}: {e}")
continue
test_results.add(
function_test_invocation=FunctionTestInvocation(
loop_index=loop_index,
@ -434,12 +581,14 @@ def merge_test_results(
test_function_name = result.id.test_function_name[: result.id.test_function_name.index("[")]
else:
test_function_name = result.id.test_function_name
if test_framework == "unittest":
elif test_framework == "unittest":
test_function_name = result.id.test_function_name
is_parameterized, new_test_function_name, _ = discover_parameters_unittest(test_function_name)
if is_parameterized: # handle parameterized test
test_function_name = new_test_function_name
else:
# Jest and other frameworks - use test function name as-is
test_function_name = result.id.test_function_name
grouped_xml_results[
(result.id.test_module_path or "")
@ -621,34 +770,42 @@ def parse_test_results(
test_results_xml = parse_test_xml(
test_xml_path, test_files=test_files, test_config=test_config, run_result=run_result
)
try:
bin_results_file = get_run_tmp_file(Path(f"test_return_values_{optimization_iteration}.bin"))
test_results_bin_file = (
parse_test_return_values_bin(bin_results_file, test_files=test_files, test_config=test_config)
if bin_results_file.exists()
else TestResults()
)
except AttributeError as e:
logger.exception(e)
test_results_bin_file = TestResults()
get_run_tmp_file(Path(f"test_return_values_{optimization_iteration}.bin")).unlink(missing_ok=True)
# Parse timing/behavior data from SQLite (used by both Python and JavaScript)
# JavaScript (Jest) uses SQLite exclusively via codeflash-jest-helper
# Python can use SQLite (preferred) or legacy binary format
test_results_data = TestResults()
try:
sql_results_file = get_run_tmp_file(Path(f"test_return_values_{optimization_iteration}.sqlite"))
if sql_results_file.exists():
test_results_sqlite_file = parse_sqlite_test_results(
test_results_data = parse_sqlite_test_results(
sqlite_file_path=sql_results_file, test_files=test_files, test_config=test_config
)
test_results_bin_file.merge(test_results_sqlite_file)
except AttributeError as e:
logger.exception(e)
logger.debug(f"Parsed {len(test_results_data.test_results)} results from SQLite")
except Exception as e:
logger.exception(f"Failed to parse SQLite test results: {e}")
# Fall back to legacy binary format for Python tests if SQLite doesn't exist
if not test_results_data.test_results and test_config.test_framework != "jest":
try:
bin_results_file = get_run_tmp_file(Path(f"test_return_values_{optimization_iteration}.bin"))
if bin_results_file.exists():
test_results_data = parse_test_return_values_bin(
bin_results_file, test_files=test_files, test_config=test_config
)
except AttributeError as e:
logger.exception(e)
# Cleanup temp files
get_run_tmp_file(Path(f"test_return_values_{optimization_iteration}.bin")).unlink(missing_ok=True)
get_run_tmp_file(Path("pytest_results.xml")).unlink(missing_ok=True)
get_run_tmp_file(Path("unittest_results.xml")).unlink(missing_ok=True)
get_run_tmp_file(Path("jest_results.xml")).unlink(missing_ok=True)
get_run_tmp_file(Path("jest_perf_results.xml")).unlink(missing_ok=True)
get_run_tmp_file(Path(f"test_return_values_{optimization_iteration}.sqlite")).unlink(missing_ok=True)
results = merge_test_results(test_results_xml, test_results_bin_file, test_config.test_framework)
results = merge_test_results(test_results_xml, test_results_data, test_config.test_framework)
all_args = False
if coverage_database_file and source_file and code_context and function_name:

99
codeflash/verification/test_runner.py
View file

@ -98,6 +98,16 @@ def run_jest_behavioral_tests(
jest_env["JEST_JUNIT_OUTPUT_FILE"] = str(result_file_path)
jest_env["JEST_JUNIT_OUTPUT_DIR"] = str(result_file_path.parent)
jest_env["JEST_JUNIT_OUTPUT_NAME"] = result_file_path.name
# Configure jest-junit to use filepath-based classnames for proper parsing
jest_env["JEST_JUNIT_CLASSNAME"] = "{filepath}"
jest_env["JEST_JUNIT_SUITE_NAME"] = "{filepath}"
jest_env["JEST_JUNIT_ADD_FILE_ATTRIBUTE"] = "true"
# Set codeflash output file for the jest helper to write timing/behavior data (SQLite format)
codeflash_sqlite_file = get_run_tmp_file(Path("test_return_values_0.sqlite"))
jest_env["CODEFLASH_OUTPUT_FILE"] = str(codeflash_sqlite_file)
jest_env["CODEFLASH_TEST_ITERATION"] = "0"
jest_env["CODEFLASH_LOOP_INDEX"] = "1"
jest_env["CODEFLASH_MODE"] = "behavior"
logger.debug(f"Running Jest tests with command: {' '.join(jest_cmd)}")
@ -304,6 +314,93 @@ def run_line_profile_tests(
return result_file_path, results
def run_jest_benchmarking_tests(
test_paths: TestFiles,
test_env: dict[str, str],
cwd: Path,
*,
timeout: int | None = None,
) -> tuple[Path, subprocess.CompletedProcess]:
"""Run Jest benchmarking tests.
Args:
test_paths: TestFiles object containing test file information.
test_env: Environment variables for the test run.
cwd: Working directory for running tests.
timeout: Optional timeout in seconds.
Returns:
Tuple of (result_file_path, subprocess_result).
"""
result_file_path = get_run_tmp_file(Path("jest_perf_results.xml"))
# Get performance test files
test_files = [str(file.benchmarking_file_path) for file in test_paths.test_files if file.benchmarking_file_path]
# Find the JavaScript project root
js_project_root = None
if test_files:
first_test_file = Path(test_files[0])
js_project_root = _find_js_project_root(first_test_file)
effective_cwd = js_project_root if js_project_root else cwd
logger.debug(f"Jest benchmarking working directory: {effective_cwd}")
# Build Jest command for performance tests
jest_cmd = [
"npx",
"jest",
"--reporters=default",
"--reporters=jest-junit",
"--runInBand",
"--forceExit",
]
if test_files:
test_pattern = "|".join(str(Path(f).resolve()) for f in test_files)
jest_cmd.append(f"--testPathPattern={test_pattern}")
if timeout:
jest_cmd.append(f"--testTimeout={timeout * 1000}")
# Set up environment
jest_env = test_env.copy()
jest_env["JEST_JUNIT_OUTPUT_FILE"] = str(result_file_path)
jest_env["JEST_JUNIT_OUTPUT_DIR"] = str(result_file_path.parent)
jest_env["JEST_JUNIT_OUTPUT_NAME"] = result_file_path.name
jest_env["JEST_JUNIT_CLASSNAME"] = "{filepath}"
jest_env["JEST_JUNIT_SUITE_NAME"] = "{filepath}"
jest_env["JEST_JUNIT_ADD_FILE_ATTRIBUTE"] = "true"
# Set codeflash output file for the jest helper to write timing data (SQLite format)
codeflash_sqlite_file = get_run_tmp_file(Path("test_return_values_0.sqlite"))
jest_env["CODEFLASH_OUTPUT_FILE"] = str(codeflash_sqlite_file)
jest_env["CODEFLASH_TEST_ITERATION"] = "0"
jest_env["CODEFLASH_LOOP_INDEX"] = "1"
jest_env["CODEFLASH_MODE"] = "performance"
logger.debug(f"Running Jest benchmarking tests: {' '.join(jest_cmd)}")
try:
run_args = get_cross_platform_subprocess_run_args(
cwd=effective_cwd, env=jest_env, timeout=timeout or 600, check=False, text=True, capture_output=True
)
result = subprocess.run(jest_cmd, **run_args) # noqa: PLW1510
logger.debug(f"Jest benchmarking result: returncode={result.returncode}")
except subprocess.TimeoutExpired:
logger.warning(f"Jest benchmarking tests timed out after {timeout}s")
result = subprocess.CompletedProcess(
args=jest_cmd, returncode=-1, stdout="", stderr="Benchmarking tests timed out"
)
except FileNotFoundError:
logger.error("Jest not found for benchmarking")
result = subprocess.CompletedProcess(
args=jest_cmd, returncode=-1, stdout="", stderr="Jest not found"
)
return result_file_path, result
def run_benchmarking_tests(
test_paths: TestFiles,
pytest_cmd: str,
@ -316,6 +413,8 @@ def run_benchmarking_tests(
pytest_min_loops: int = 5,
pytest_max_loops: int = 100_000,
) -> tuple[Path, subprocess.CompletedProcess]:
if test_framework == "jest":
return run_jest_benchmarking_tests(test_paths, test_env, cwd, timeout=pytest_timeout)
if test_framework in {"pytest", "unittest"}: # pytest runs both pytest and unittest tests
pytest_cmd_list = (
shlex.split(f"{SAFE_SYS_EXECUTABLE} -m pytest", posix=IS_POSIX)

6
tests/test_languages/test_javascript_instrumentation.py
View file

@ -147,8 +147,8 @@ function multiply(x, y) {
class TestJavaScriptSupportInstrumentation:
"""Integration tests for JavaScript support instrumentation methods."""
def test_javascript_support_instrument_for_tracing(self):
"""Test JavaScriptSupport.instrument_for_tracing method."""
def test_javascript_support_instrument_for_behavior(self):
"""Test JavaScriptSupport.instrument_for_behavior method."""
from codeflash.languages import get_language_support
js_support = get_language_support(Language.JAVASCRIPT)
@ -173,7 +173,7 @@ function greet(name) {
)
output_file = file_path.parent / ".codeflash" / "traces.db"
instrumented = js_support.instrument_for_tracing(
instrumented = js_support.instrument_for_behavior(
source, [func_info], output_file=output_file
)