fixes and refactor
This commit is contained in:
misrasaurabh1
parent
39d31400be
commit
7bcc02aafd
11 changed files with 1409 additions and 2341 deletions
|
|
@ -1,406 +0,0 @@
|
|||
/**
|
||||
* Codeflash Comparator - Deep equality comparison for JavaScript values
|
||||
*
|
||||
* This module provides a robust comparator function for comparing JavaScript
|
||||
* values to determine behavioral equivalence between original and optimized code.
|
||||
*
|
||||
* Features:
|
||||
* - Handles all JavaScript primitive types
|
||||
* - Floating point comparison with relative tolerance (like Python's math.isclose)
|
||||
* - Deep comparison of objects, arrays, Maps, Sets
|
||||
* - Handles special values: NaN, Infinity, -Infinity, undefined, null
|
||||
* - Handles TypedArrays, Date, RegExp, Error objects
|
||||
* - Circular reference detection
|
||||
* - Superset mode: allows new object to have additional keys
|
||||
*
|
||||
* Usage:
|
||||
* const { comparator } = require('./codeflash-comparator');
|
||||
* comparator(original, optimized); // Exact comparison
|
||||
* comparator(original, optimized, { supersetObj: true }); // Allow extra keys
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
/**
|
||||
* Default options for the comparator.
|
||||
*/
|
||||
const DEFAULT_OPTIONS = {
|
||||
// Relative tolerance for floating point comparison (like Python's rtol)
|
||||
rtol: 1e-9,
|
||||
// Absolute tolerance for floating point comparison (like Python's atol)
|
||||
atol: 0,
|
||||
// If true, the new object is allowed to have more keys than the original
|
||||
supersetObj: false,
|
||||
// Maximum recursion depth to prevent stack overflow
|
||||
maxDepth: 1000,
|
||||
};
|
||||
|
||||
/**
|
||||
* Check if two floating point numbers are close within tolerance.
|
||||
* Equivalent to Python's math.isclose(a, b, rel_tol, abs_tol).
|
||||
*
|
||||
* @param {number} a - First number
|
||||
* @param {number} b - Second number
|
||||
* @param {number} rtol - Relative tolerance (default: 1e-9)
|
||||
* @param {number} atol - Absolute tolerance (default: 0)
|
||||
* @returns {boolean} - True if numbers are close
|
||||
*/
|
||||
function isClose(a, b, rtol = 1e-9, atol = 0) {
|
||||
// Handle identical values (including both being 0)
|
||||
if (a === b) return true;
|
||||
|
||||
// Handle NaN
|
||||
if (Number.isNaN(a) && Number.isNaN(b)) return true;
|
||||
if (Number.isNaN(a) || Number.isNaN(b)) return false;
|
||||
|
||||
// Handle Infinity
|
||||
if (!Number.isFinite(a) || !Number.isFinite(b)) {
|
||||
return a === b; // Both must be same infinity
|
||||
}
|
||||
|
||||
// Use the same formula as Python's math.isclose
|
||||
// abs(a-b) <= max(rel_tol * max(abs(a), abs(b)), abs_tol)
|
||||
const diff = Math.abs(a - b);
|
||||
const maxAbs = Math.max(Math.abs(a), Math.abs(b));
|
||||
return diff <= Math.max(rtol * maxAbs, atol);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the precise type of a value for comparison.
|
||||
*
|
||||
* @param {any} value - The value to get the type of
|
||||
* @returns {string} - The type name
|
||||
*/
|
||||
function getType(value) {
|
||||
if (value === null) return 'null';
|
||||
if (value === undefined) return 'undefined';
|
||||
|
||||
const type = typeof value;
|
||||
if (type !== 'object') return type;
|
||||
|
||||
// Get the constructor name for objects
|
||||
const constructorName = value.constructor?.name;
|
||||
if (constructorName) return constructorName;
|
||||
|
||||
// Fallback to Object.prototype.toString
|
||||
return Object.prototype.toString.call(value).slice(8, -1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a value is a TypedArray.
|
||||
*
|
||||
* @param {any} value - The value to check
|
||||
* @returns {boolean} - True if TypedArray
|
||||
*/
|
||||
function isTypedArray(value) {
|
||||
return ArrayBuffer.isView(value) && !(value instanceof DataView);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two values for deep equality.
|
||||
*
|
||||
* @param {any} orig - Original value
|
||||
* @param {any} newVal - New value to compare
|
||||
* @param {Object} options - Comparison options
|
||||
* @param {number} options.rtol - Relative tolerance for floats
|
||||
* @param {number} options.atol - Absolute tolerance for floats
|
||||
* @param {boolean} options.supersetObj - Allow new object to have extra keys
|
||||
* @param {number} options.maxDepth - Maximum recursion depth
|
||||
* @returns {boolean} - True if values are equivalent
|
||||
*/
|
||||
function comparator(orig, newVal, options = {}) {
|
||||
const opts = { ...DEFAULT_OPTIONS, ...options };
|
||||
|
||||
// Track visited objects to handle circular references
|
||||
const visited = new WeakMap();
|
||||
|
||||
function compare(a, b, depth) {
|
||||
// Check recursion depth
|
||||
if (depth > opts.maxDepth) {
|
||||
console.warn('[comparator] Maximum recursion depth exceeded');
|
||||
return false;
|
||||
}
|
||||
|
||||
// === Identical references ===
|
||||
if (a === b) return true;
|
||||
|
||||
// === Handle null and undefined ===
|
||||
if (a === null || a === undefined || b === null || b === undefined) {
|
||||
return a === b;
|
||||
}
|
||||
|
||||
// === Type checking ===
|
||||
const typeA = typeof a;
|
||||
const typeB = typeof b;
|
||||
|
||||
if (typeA !== typeB) {
|
||||
// Special case: comparing number with BigInt
|
||||
// In JavaScript, 1n !== 1, but we might want to consider them equal
|
||||
// For strict behavioral comparison, we'll say they're different
|
||||
return false;
|
||||
}
|
||||
|
||||
// === Primitives ===
|
||||
|
||||
// Numbers (including NaN and Infinity)
|
||||
if (typeA === 'number') {
|
||||
return isClose(a, b, opts.rtol, opts.atol);
|
||||
}
|
||||
|
||||
// Strings, booleans
|
||||
if (typeA === 'string' || typeA === 'boolean') {
|
||||
return a === b;
|
||||
}
|
||||
|
||||
// BigInt
|
||||
if (typeA === 'bigint') {
|
||||
return a === b;
|
||||
}
|
||||
|
||||
// Symbols - compare by description since Symbol() always creates unique
|
||||
if (typeA === 'symbol') {
|
||||
return a.description === b.description;
|
||||
}
|
||||
|
||||
// Functions - compare by reference (same function)
|
||||
if (typeA === 'function') {
|
||||
// Functions are equal if they're the same reference
|
||||
// or if they have the same name and source code
|
||||
if (a === b) return true;
|
||||
// For bound functions or native functions, we can only compare by reference
|
||||
try {
|
||||
return a.name === b.name && a.toString() === b.toString();
|
||||
} catch (e) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// === Objects (typeA === 'object') ===
|
||||
|
||||
// Check for circular references
|
||||
if (visited.has(a)) {
|
||||
// If we've seen 'a' before, check if 'b' was the corresponding value
|
||||
return visited.get(a) === b;
|
||||
}
|
||||
|
||||
// Get constructor names for type comparison
|
||||
const constructorA = a.constructor?.name || 'Object';
|
||||
const constructorB = b.constructor?.name || 'Object';
|
||||
|
||||
// Different constructors means different types
|
||||
// Exception: plain objects might have different constructors due to different realms
|
||||
if (constructorA !== constructorB) {
|
||||
// Allow comparison between plain objects from different realms
|
||||
if (!(constructorA === 'Object' && constructorB === 'Object')) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// Mark as visited before recursing
|
||||
visited.set(a, b);
|
||||
|
||||
try {
|
||||
// === Arrays ===
|
||||
if (Array.isArray(a)) {
|
||||
if (!Array.isArray(b)) return false;
|
||||
if (a.length !== b.length) return false;
|
||||
return a.every((elem, i) => compare(elem, b[i], depth + 1));
|
||||
}
|
||||
|
||||
// === TypedArrays (Int8Array, Uint8Array, Float32Array, etc.) ===
|
||||
if (isTypedArray(a)) {
|
||||
if (!isTypedArray(b)) return false;
|
||||
if (a.constructor !== b.constructor) return false;
|
||||
if (a.length !== b.length) return false;
|
||||
|
||||
// For float arrays, use tolerance comparison
|
||||
if (a instanceof Float32Array || a instanceof Float64Array) {
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
if (!isClose(a[i], b[i], opts.rtol, opts.atol)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// For integer arrays, use exact comparison
|
||||
for (let i = 0; i < a.length; i++) {
|
||||
if (a[i] !== b[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// === ArrayBuffer ===
|
||||
if (a instanceof ArrayBuffer) {
|
||||
if (!(b instanceof ArrayBuffer)) return false;
|
||||
if (a.byteLength !== b.byteLength) return false;
|
||||
const viewA = new Uint8Array(a);
|
||||
const viewB = new Uint8Array(b);
|
||||
for (let i = 0; i < viewA.length; i++) {
|
||||
if (viewA[i] !== viewB[i]) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// === DataView ===
|
||||
if (a instanceof DataView) {
|
||||
if (!(b instanceof DataView)) return false;
|
||||
if (a.byteLength !== b.byteLength) return false;
|
||||
for (let i = 0; i < a.byteLength; i++) {
|
||||
if (a.getUint8(i) !== b.getUint8(i)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// === Date ===
|
||||
if (a instanceof Date) {
|
||||
if (!(b instanceof Date)) return false;
|
||||
// Handle Invalid Date (NaN time)
|
||||
const timeA = a.getTime();
|
||||
const timeB = b.getTime();
|
||||
if (Number.isNaN(timeA) && Number.isNaN(timeB)) return true;
|
||||
return timeA === timeB;
|
||||
}
|
||||
|
||||
// === RegExp ===
|
||||
if (a instanceof RegExp) {
|
||||
if (!(b instanceof RegExp)) return false;
|
||||
return a.source === b.source && a.flags === b.flags;
|
||||
}
|
||||
|
||||
// === Error ===
|
||||
if (a instanceof Error) {
|
||||
if (!(b instanceof Error)) return false;
|
||||
// Compare error name and message
|
||||
if (a.name !== b.name) return false;
|
||||
if (a.message !== b.message) return false;
|
||||
// Optionally compare stack traces (usually not, as they differ)
|
||||
return true;
|
||||
}
|
||||
|
||||
// === Map ===
|
||||
if (a instanceof Map) {
|
||||
if (!(b instanceof Map)) return false;
|
||||
if (a.size !== b.size) return false;
|
||||
for (const [key, val] of a) {
|
||||
if (!b.has(key)) return false;
|
||||
if (!compare(val, b.get(key), depth + 1)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// === Set ===
|
||||
if (a instanceof Set) {
|
||||
if (!(b instanceof Set)) return false;
|
||||
if (a.size !== b.size) return false;
|
||||
// For Sets, we need to find matching elements
|
||||
// This is O(n^2) but necessary for deep comparison
|
||||
const bArray = Array.from(b);
|
||||
for (const valA of a) {
|
||||
let found = false;
|
||||
for (let i = 0; i < bArray.length; i++) {
|
||||
if (compare(valA, bArray[i], depth + 1)) {
|
||||
found = true;
|
||||
bArray.splice(i, 1); // Remove matched element
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (!found) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
// === WeakMap / WeakSet ===
|
||||
// Cannot iterate over these, so we can only compare by reference
|
||||
if (a instanceof WeakMap || a instanceof WeakSet) {
|
||||
return a === b;
|
||||
}
|
||||
|
||||
// === Promise ===
|
||||
// Promises can only be compared by reference
|
||||
if (a instanceof Promise) {
|
||||
return a === b;
|
||||
}
|
||||
|
||||
// === URL ===
|
||||
if (typeof URL !== 'undefined' && a instanceof URL) {
|
||||
if (!(b instanceof URL)) return false;
|
||||
return a.href === b.href;
|
||||
}
|
||||
|
||||
// === URLSearchParams ===
|
||||
if (typeof URLSearchParams !== 'undefined' && a instanceof URLSearchParams) {
|
||||
if (!(b instanceof URLSearchParams)) return false;
|
||||
return a.toString() === b.toString();
|
||||
}
|
||||
|
||||
// === Plain Objects ===
|
||||
// This includes class instances
|
||||
|
||||
const keysA = Object.keys(a);
|
||||
const keysB = Object.keys(b);
|
||||
|
||||
if (opts.supersetObj) {
|
||||
// In superset mode, all keys from original must exist in new
|
||||
// but new can have additional keys
|
||||
for (const key of keysA) {
|
||||
if (!(key in b)) return false;
|
||||
if (!compare(a[key], b[key], depth + 1)) return false;
|
||||
}
|
||||
return true;
|
||||
} else {
|
||||
// Exact key matching
|
||||
if (keysA.length !== keysB.length) return false;
|
||||
|
||||
for (const key of keysA) {
|
||||
if (!(key in b)) return false;
|
||||
if (!compare(a[key], b[key], depth + 1)) return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
} finally {
|
||||
// Clean up visited tracking
|
||||
// Note: We don't delete from visited because the same object
|
||||
// might appear multiple times in the structure
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
return compare(orig, newVal, 0);
|
||||
} catch (e) {
|
||||
console.error('[comparator] Error during comparison:', e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a comparator with custom default options.
|
||||
*
|
||||
* @param {Object} defaultOptions - Default options for all comparisons
|
||||
* @returns {Function} - Comparator function with bound defaults
|
||||
*/
|
||||
function createComparator(defaultOptions = {}) {
|
||||
const opts = { ...DEFAULT_OPTIONS, ...defaultOptions };
|
||||
return (orig, newVal, overrideOptions = {}) => {
|
||||
return comparator(orig, newVal, { ...opts, ...overrideOptions });
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Strict comparator that requires exact equality (no tolerance).
|
||||
*/
|
||||
const strictComparator = createComparator({ rtol: 0, atol: 0 });
|
||||
|
||||
/**
|
||||
* Loose comparator with larger tolerance for floating point.
|
||||
*/
|
||||
const looseComparator = createComparator({ rtol: 1e-6, atol: 1e-9 });
|
||||
|
||||
// Export public API
|
||||
module.exports = {
|
||||
comparator,
|
||||
createComparator,
|
||||
strictComparator,
|
||||
looseComparator,
|
||||
isClose,
|
||||
getType,
|
||||
DEFAULT_OPTIONS,
|
||||
};
|
||||
|
|
@ -1,313 +0,0 @@
|
|||
#!/usr/bin/env node
|
||||
/**
|
||||
* Codeflash Result Comparator
|
||||
*
|
||||
* This script compares test results between original and optimized code runs.
|
||||
* It reads serialized behavior data from SQLite databases and compares them
|
||||
* using the codeflash-comparator in JavaScript land.
|
||||
*
|
||||
* Usage:
|
||||
* node codeflash-compare-results.js <original_db> <candidate_db>
|
||||
* node codeflash-compare-results.js --json <json_input>
|
||||
*
|
||||
* Output (JSON):
|
||||
* {
|
||||
* "equivalent": true/false,
|
||||
* "diffs": [
|
||||
* {
|
||||
* "invocation_id": "...",
|
||||
* "scope": "return_value|stdout|did_pass",
|
||||
* "original": "...",
|
||||
* "candidate": "..."
|
||||
* }
|
||||
* ],
|
||||
* "error": null | "error message"
|
||||
* }
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Import our modules
|
||||
const { deserialize } = require('./codeflash-serializer');
|
||||
const { comparator } = require('./codeflash-comparator');
|
||||
|
||||
// Try to load better-sqlite3
|
||||
let Database;
|
||||
try {
|
||||
Database = require('better-sqlite3');
|
||||
} catch (e) {
|
||||
console.error(JSON.stringify({
|
||||
equivalent: false,
|
||||
diffs: [],
|
||||
error: 'better-sqlite3 not installed'
|
||||
}));
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
/**
|
||||
* Read test results from a SQLite database.
|
||||
*
|
||||
* @param {string} dbPath - Path to SQLite database
|
||||
* @returns {Map<string, object>} Map of invocation_id -> result object
|
||||
*/
|
||||
function readTestResults(dbPath) {
|
||||
const results = new Map();
|
||||
|
||||
if (!fs.existsSync(dbPath)) {
|
||||
throw new Error(`Database not found: ${dbPath}`);
|
||||
}
|
||||
|
||||
const db = new Database(dbPath, { readonly: true });
|
||||
|
||||
try {
|
||||
const stmt = db.prepare(`
|
||||
SELECT
|
||||
test_module_path,
|
||||
test_class_name,
|
||||
test_function_name,
|
||||
function_getting_tested,
|
||||
loop_index,
|
||||
iteration_id,
|
||||
runtime,
|
||||
return_value,
|
||||
verification_type
|
||||
FROM test_results
|
||||
WHERE loop_index = 1
|
||||
`);
|
||||
|
||||
for (const row of stmt.iterate()) {
|
||||
// Build unique invocation ID (matches Python's format)
|
||||
const invocationId = `${row.loop_index}:${row.test_module_path}:${row.test_class_name || ''}:${row.test_function_name}:${row.function_getting_tested}:${row.iteration_id}`;
|
||||
|
||||
// Deserialize the return value
|
||||
let returnValue = null;
|
||||
if (row.return_value) {
|
||||
try {
|
||||
returnValue = deserialize(row.return_value);
|
||||
} catch (e) {
|
||||
console.error(`Failed to deserialize result for ${invocationId}: ${e.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
results.set(invocationId, {
|
||||
testModulePath: row.test_module_path,
|
||||
testClassName: row.test_class_name,
|
||||
testFunctionName: row.test_function_name,
|
||||
functionGettingTested: row.function_getting_tested,
|
||||
loopIndex: row.loop_index,
|
||||
iterationId: row.iteration_id,
|
||||
runtime: row.runtime,
|
||||
returnValue,
|
||||
verificationType: row.verification_type,
|
||||
});
|
||||
}
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two sets of test results.
|
||||
*
|
||||
* @param {Map<string, object>} originalResults - Results from original code
|
||||
* @param {Map<string, object>} candidateResults - Results from optimized code
|
||||
* @returns {object} Comparison result
|
||||
*/
|
||||
function compareResults(originalResults, candidateResults) {
|
||||
const diffs = [];
|
||||
let allEquivalent = true;
|
||||
|
||||
// Get all unique invocation IDs
|
||||
const allIds = new Set([...originalResults.keys(), ...candidateResults.keys()]);
|
||||
|
||||
for (const invocationId of allIds) {
|
||||
const original = originalResults.get(invocationId);
|
||||
const candidate = candidateResults.get(invocationId);
|
||||
|
||||
// If candidate has extra results not in original, that's OK
|
||||
if (candidate && !original) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// If original has results not in candidate, that's a diff
|
||||
if (original && !candidate) {
|
||||
allEquivalent = false;
|
||||
diffs.push({
|
||||
invocation_id: invocationId,
|
||||
scope: 'missing',
|
||||
original: summarizeValue(original.returnValue),
|
||||
candidate: null,
|
||||
test_info: {
|
||||
test_module_path: original.testModulePath,
|
||||
test_function_name: original.testFunctionName,
|
||||
function_getting_tested: original.functionGettingTested,
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Compare return values using the JavaScript comparator
|
||||
// The return value format is [args, kwargs, returnValue] (behavior tuple)
|
||||
const originalValue = original.returnValue;
|
||||
const candidateValue = candidate.returnValue;
|
||||
|
||||
const isEqual = comparator(originalValue, candidateValue);
|
||||
|
||||
if (!isEqual) {
|
||||
allEquivalent = false;
|
||||
diffs.push({
|
||||
invocation_id: invocationId,
|
||||
scope: 'return_value',
|
||||
original: summarizeValue(originalValue),
|
||||
candidate: summarizeValue(candidateValue),
|
||||
test_info: {
|
||||
test_module_path: original.testModulePath,
|
||||
test_function_name: original.testFunctionName,
|
||||
function_getting_tested: original.functionGettingTested,
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
equivalent: allEquivalent,
|
||||
diffs,
|
||||
total_invocations: allIds.size,
|
||||
original_count: originalResults.size,
|
||||
candidate_count: candidateResults.size,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a summary of a value for diff reporting.
|
||||
* Truncates long values to avoid huge output.
|
||||
*
|
||||
* @param {any} value - Value to summarize
|
||||
* @returns {string} String representation
|
||||
*/
|
||||
function summarizeValue(value, maxLength = 500) {
|
||||
try {
|
||||
let str;
|
||||
if (value === undefined) {
|
||||
str = 'undefined';
|
||||
} else if (value === null) {
|
||||
str = 'null';
|
||||
} else if (typeof value === 'function') {
|
||||
str = `[Function: ${value.name || 'anonymous'}]`;
|
||||
} else if (value instanceof Map) {
|
||||
str = `Map(${value.size}) { ${[...value.entries()].slice(0, 3).map(([k, v]) => `${summarizeValue(k, 50)} => ${summarizeValue(v, 50)}`).join(', ')}${value.size > 3 ? ', ...' : ''} }`;
|
||||
} else if (value instanceof Set) {
|
||||
str = `Set(${value.size}) { ${[...value].slice(0, 3).map(v => summarizeValue(v, 50)).join(', ')}${value.size > 3 ? ', ...' : ''} }`;
|
||||
} else if (value instanceof Date) {
|
||||
str = value.toISOString();
|
||||
} else if (Array.isArray(value)) {
|
||||
if (value.length <= 5) {
|
||||
str = JSON.stringify(value);
|
||||
} else {
|
||||
str = `[${value.slice(0, 3).map(v => summarizeValue(v, 50)).join(', ')}, ... (${value.length} items)]`;
|
||||
}
|
||||
} else if (typeof value === 'object') {
|
||||
str = JSON.stringify(value);
|
||||
} else {
|
||||
str = String(value);
|
||||
}
|
||||
|
||||
if (str.length > maxLength) {
|
||||
return str.slice(0, maxLength - 3) + '...';
|
||||
}
|
||||
return str;
|
||||
} catch (e) {
|
||||
return `[Unable to stringify: ${e.message}]`;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare results from serialized buffers directly (for stdin input).
|
||||
*
|
||||
* @param {Buffer} originalBuffer - Serialized original result
|
||||
* @param {Buffer} candidateBuffer - Serialized candidate result
|
||||
* @returns {boolean} True if equivalent
|
||||
*/
|
||||
function compareBuffers(originalBuffer, candidateBuffer) {
|
||||
try {
|
||||
const original = deserialize(originalBuffer);
|
||||
const candidate = deserialize(candidateBuffer);
|
||||
return comparator(original, candidate);
|
||||
} catch (e) {
|
||||
console.error(`Comparison error: ${e.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Main entry point.
|
||||
*/
|
||||
function main() {
|
||||
const args = process.argv.slice(2);
|
||||
|
||||
if (args.length === 0) {
|
||||
console.error('Usage: node codeflash-compare-results.js <original_db> <candidate_db>');
|
||||
console.error(' node codeflash-compare-results.js --stdin (reads JSON from stdin)');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Handle stdin mode for programmatic use
|
||||
if (args[0] === '--stdin') {
|
||||
let input = '';
|
||||
process.stdin.setEncoding('utf8');
|
||||
process.stdin.on('data', chunk => input += chunk);
|
||||
process.stdin.on('end', () => {
|
||||
try {
|
||||
const data = JSON.parse(input);
|
||||
const originalBuffer = Buffer.from(data.original, 'base64');
|
||||
const candidateBuffer = Buffer.from(data.candidate, 'base64');
|
||||
const isEqual = compareBuffers(originalBuffer, candidateBuffer);
|
||||
console.log(JSON.stringify({ equivalent: isEqual, error: null }));
|
||||
} catch (e) {
|
||||
console.log(JSON.stringify({ equivalent: false, error: e.message }));
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Standard mode: compare two SQLite databases
|
||||
if (args.length < 2) {
|
||||
console.error('Usage: node codeflash-compare-results.js <original_db> <candidate_db>');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const [originalDb, candidateDb] = args;
|
||||
|
||||
try {
|
||||
const originalResults = readTestResults(originalDb);
|
||||
const candidateResults = readTestResults(candidateDb);
|
||||
|
||||
const comparison = compareResults(originalResults, candidateResults);
|
||||
|
||||
console.log(JSON.stringify(comparison, null, 2));
|
||||
process.exit(comparison.equivalent ? 0 : 1);
|
||||
} catch (e) {
|
||||
console.log(JSON.stringify({
|
||||
equivalent: false,
|
||||
diffs: [],
|
||||
error: e.message
|
||||
}));
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
// Export for programmatic use
|
||||
module.exports = {
|
||||
readTestResults,
|
||||
compareResults,
|
||||
compareBuffers,
|
||||
summarizeValue,
|
||||
};
|
||||
|
||||
// Run if called directly
|
||||
if (require.main === module) {
|
||||
main();
|
||||
}
|
||||
|
|
@ -1,701 +0,0 @@
|
|||
/**
|
||||
* Codeflash Jest Helper - Unified Test Instrumentation
|
||||
*
|
||||
* This module provides a unified approach to instrumenting JavaScript tests
|
||||
* for both behavior verification and performance measurement.
|
||||
*
|
||||
* The instrumentation mirrors Python's codeflash implementation:
|
||||
* - Static identifiers (testModule, testFunction, lineId) are passed at instrumentation time
|
||||
* - Dynamic invocation counter increments only when same call site is seen again (e.g., in loops)
|
||||
* - Uses hrtime for nanosecond precision timing
|
||||
* - SQLite for consistent data format with Python implementation
|
||||
*
|
||||
* Usage:
|
||||
* const codeflash = require('./codeflash-jest-helper');
|
||||
*
|
||||
* // For behavior verification (writes to SQLite):
|
||||
* const result = codeflash.capture('functionName', lineId, targetFunction, arg1, arg2);
|
||||
*
|
||||
* // For performance benchmarking (stdout only):
|
||||
* const result = codeflash.capturePerf('functionName', lineId, targetFunction, arg1, arg2);
|
||||
*
|
||||
* Environment Variables:
|
||||
* CODEFLASH_OUTPUT_FILE - Path to write results SQLite file
|
||||
* CODEFLASH_LOOP_INDEX - Current benchmark loop iteration (default: 1)
|
||||
* CODEFLASH_TEST_ITERATION - Test iteration number (default: 0)
|
||||
* CODEFLASH_TEST_MODULE - Test module path
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
// Load the codeflash serializer for robust value serialization
|
||||
const serializer = require('./codeflash-serializer');
|
||||
|
||||
// Try to load better-sqlite3, fall back to JSON if not available
|
||||
let Database;
|
||||
let useSqlite = false;
|
||||
try {
|
||||
Database = require('better-sqlite3');
|
||||
useSqlite = true;
|
||||
} catch (e) {
|
||||
// better-sqlite3 not available, will use JSON fallback
|
||||
console.warn('[codeflash] better-sqlite3 not found, using JSON fallback');
|
||||
}
|
||||
|
||||
// Configuration from environment
|
||||
const OUTPUT_FILE = process.env.CODEFLASH_OUTPUT_FILE || '/tmp/codeflash_results.sqlite';
|
||||
const LOOP_INDEX = parseInt(process.env.CODEFLASH_LOOP_INDEX || '1', 10);
|
||||
const TEST_ITERATION = process.env.CODEFLASH_TEST_ITERATION || '0';
|
||||
const TEST_MODULE = process.env.CODEFLASH_TEST_MODULE || '';
|
||||
|
||||
// Looping configuration for performance benchmarking
|
||||
const MIN_LOOPS = parseInt(process.env.CODEFLASH_MIN_LOOPS || '5', 10);
|
||||
const MAX_LOOPS = parseInt(process.env.CODEFLASH_MAX_LOOPS || '100000', 10);
|
||||
const TARGET_DURATION_MS = parseInt(process.env.CODEFLASH_TARGET_DURATION_MS || '10000', 10);
|
||||
const STABILITY_CHECK = process.env.CODEFLASH_STABILITY_CHECK !== 'false';
|
||||
|
||||
// Stability checking constants (matching Python's pytest_plugin.py)
|
||||
const STABILITY_WINDOW_SIZE = 0.35; // 35% of estimated total loops
|
||||
const STABILITY_CENTER_TOLERANCE = 0.0025; // ±0.25% around median
|
||||
const STABILITY_SPREAD_TOLERANCE = 0.0025; // 0.25% window spread
|
||||
|
||||
// Current test context (set by Jest hooks)
|
||||
let currentTestName = null;
|
||||
|
||||
// Invocation counter map: tracks how many times each testId has been seen
|
||||
// Key: testId (testModule:testClass:testFunction:lineId:loopIndex)
|
||||
// Value: count (starts at 0, increments each time same key is seen)
|
||||
const invocationCounterMap = new Map();
|
||||
|
||||
// Results buffer (for JSON fallback)
|
||||
const results = [];
|
||||
|
||||
// SQLite database (lazy initialized)
|
||||
let db = null;
|
||||
|
||||
/**
|
||||
* Get high-resolution time in nanoseconds.
|
||||
* Prefers process.hrtime.bigint() for nanosecond precision,
|
||||
* falls back to performance.now() * 1e6 for non-Node environments.
|
||||
*
|
||||
* @returns {bigint|number} - Time in nanoseconds
|
||||
*/
|
||||
function getTimeNs() {
|
||||
if (typeof process !== 'undefined' && process.hrtime && process.hrtime.bigint) {
|
||||
return process.hrtime.bigint();
|
||||
}
|
||||
// Fallback to performance.now() in milliseconds, converted to nanoseconds
|
||||
const { performance } = require('perf_hooks');
|
||||
return BigInt(Math.floor(performance.now() * 1_000_000));
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate duration in nanoseconds.
|
||||
*
|
||||
* @param {bigint} start - Start time in nanoseconds
|
||||
* @param {bigint} end - End time in nanoseconds
|
||||
* @returns {number} - Duration in nanoseconds (as Number for SQLite compatibility)
|
||||
*/
|
||||
function getDurationNs(start, end) {
|
||||
const duration = end - start;
|
||||
// Convert to Number for SQLite storage (SQLite INTEGER is 64-bit)
|
||||
return Number(duration);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create invocation index for a testId.
|
||||
* This mirrors Python's index tracking per wrapper function.
|
||||
*
|
||||
* @param {string} testId - Unique test identifier
|
||||
* @returns {number} - Current invocation index (0-based)
|
||||
*/
|
||||
function getInvocationIndex(testId) {
|
||||
const currentIndex = invocationCounterMap.get(testId);
|
||||
if (currentIndex === undefined) {
|
||||
invocationCounterMap.set(testId, 0);
|
||||
return 0;
|
||||
}
|
||||
invocationCounterMap.set(testId, currentIndex + 1);
|
||||
return currentIndex + 1;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reset invocation counter for a test.
|
||||
* Called at the start of each test to ensure consistent indexing.
|
||||
*/
|
||||
function resetInvocationCounters() {
|
||||
invocationCounterMap.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Initialize the SQLite database.
|
||||
*/
|
||||
function initDatabase() {
|
||||
if (!useSqlite || db) return;
|
||||
|
||||
try {
|
||||
db = new Database(OUTPUT_FILE);
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS test_results (
|
||||
test_module_path TEXT,
|
||||
test_class_name TEXT,
|
||||
test_function_name TEXT,
|
||||
function_getting_tested TEXT,
|
||||
loop_index INTEGER,
|
||||
iteration_id TEXT,
|
||||
runtime INTEGER,
|
||||
return_value BLOB,
|
||||
verification_type TEXT
|
||||
)
|
||||
`);
|
||||
} catch (e) {
|
||||
console.error('[codeflash] Failed to initialize SQLite:', e.message);
|
||||
useSqlite = false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely serialize a value for storage.
|
||||
*
|
||||
* @param {any} value - Value to serialize
|
||||
* @returns {Buffer} - Serialized value as Buffer
|
||||
*/
|
||||
function safeSerialize(value) {
|
||||
try {
|
||||
return serializer.serialize(value);
|
||||
} catch (e) {
|
||||
console.warn('[codeflash] Serialization failed:', e.message);
|
||||
return Buffer.from(JSON.stringify({ __type: 'SerializationError', error: e.message }));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Safely deserialize a buffer back to a value.
|
||||
*
|
||||
* @param {Buffer|Uint8Array} buffer - Serialized buffer
|
||||
* @returns {any} - Deserialized value
|
||||
*/
|
||||
function safeDeserialize(buffer) {
|
||||
try {
|
||||
return serializer.deserialize(buffer);
|
||||
} catch (e) {
|
||||
console.warn('[codeflash] Deserialization failed:', e.message);
|
||||
return { __type: 'DeserializationError', error: e.message };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Record a test result to SQLite or JSON buffer.
|
||||
*
|
||||
* @param {string} testModulePath - Test module path
|
||||
* @param {string|null} testClassName - Test class name (null for Jest)
|
||||
* @param {string} testFunctionName - Test function name
|
||||
* @param {string} funcName - Name of the function being tested
|
||||
* @param {string} invocationId - Unique invocation identifier (lineId_index)
|
||||
* @param {Array} args - Arguments passed to the function
|
||||
* @param {any} returnValue - Return value from the function
|
||||
* @param {Error|null} error - Error thrown by the function (if any)
|
||||
* @param {number} durationNs - Execution time in nanoseconds
|
||||
*/
|
||||
function recordResult(testModulePath, testClassName, testFunctionName, funcName, invocationId, args, returnValue, error, durationNs) {
|
||||
// Serialize the return value (args, kwargs (empty for JS), return_value) like Python does
|
||||
const serializedValue = error
|
||||
? safeSerialize(error)
|
||||
: safeSerialize([args, {}, returnValue]);
|
||||
|
||||
if (useSqlite && db) {
|
||||
try {
|
||||
const stmt = db.prepare(`
|
||||
INSERT INTO test_results VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
|
||||
`);
|
||||
stmt.run(
|
||||
testModulePath, // test_module_path
|
||||
testClassName, // test_class_name
|
||||
testFunctionName, // test_function_name
|
||||
funcName, // function_getting_tested
|
||||
LOOP_INDEX, // loop_index
|
||||
invocationId, // iteration_id
|
||||
durationNs, // runtime (nanoseconds) - no rounding
|
||||
serializedValue, // return_value (serialized)
|
||||
'function_call' // verification_type
|
||||
);
|
||||
} catch (e) {
|
||||
console.error('[codeflash] Failed to write to SQLite:', e.message);
|
||||
// Fall back to JSON
|
||||
results.push({
|
||||
testModulePath,
|
||||
testClassName,
|
||||
testFunctionName,
|
||||
funcName,
|
||||
loopIndex: LOOP_INDEX,
|
||||
iterationId: invocationId,
|
||||
durationNs,
|
||||
returnValue: error ? null : returnValue,
|
||||
error: error ? { name: error.name, message: error.message } : null,
|
||||
verificationType: 'function_call'
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// JSON fallback
|
||||
results.push({
|
||||
testModulePath,
|
||||
testClassName,
|
||||
testFunctionName,
|
||||
funcName,
|
||||
loopIndex: LOOP_INDEX,
|
||||
iterationId: invocationId,
|
||||
durationNs,
|
||||
returnValue: error ? null : returnValue,
|
||||
error: error ? { name: error.name, message: error.message } : null,
|
||||
verificationType: 'function_call'
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture a function call with full behavior tracking.
|
||||
*
|
||||
* This is the main API for instrumenting function calls for BEHAVIOR verification.
|
||||
* It captures inputs, outputs, errors, and timing.
|
||||
* Results are written to SQLite for comparison between original and optimized code.
|
||||
*
|
||||
* Static parameters (funcName, lineId) are determined at instrumentation time.
|
||||
* The lineId enables tracking when the same call site is invoked multiple times (e.g., in loops).
|
||||
*
|
||||
* @param {string} funcName - Name of the function being tested (static)
|
||||
* @param {string} lineId - Line number identifier in test file (static)
|
||||
* @param {Function} fn - The function to call
|
||||
* @param {...any} args - Arguments to pass to the function
|
||||
* @returns {any} - The function's return value
|
||||
* @throws {Error} - Re-throws any error from the function
|
||||
*/
|
||||
function capture(funcName, lineId, fn, ...args) {
|
||||
// Initialize database on first capture
|
||||
initDatabase();
|
||||
|
||||
// Get test context
|
||||
const testModulePath = TEST_MODULE || currentTestName || 'unknown';
|
||||
const testClassName = null; // Jest doesn't use classes like Python
|
||||
const testFunctionName = currentTestName || 'unknown';
|
||||
|
||||
// Create testId for invocation tracking (matches Python format)
|
||||
const testId = `${testModulePath}:${testClassName}:${testFunctionName}:${lineId}:${LOOP_INDEX}`;
|
||||
|
||||
// Get invocation index (increments if same testId seen again)
|
||||
const invocationIndex = getInvocationIndex(testId);
|
||||
const invocationId = `${lineId}_${invocationIndex}`;
|
||||
|
||||
// Format stdout tag (matches Python format)
|
||||
const testStdoutTag = `${testModulePath}:${testClassName ? testClassName + '.' : ''}${testFunctionName}:${funcName}:${LOOP_INDEX}:${invocationId}`;
|
||||
|
||||
// Print start tag
|
||||
console.log(`!$######${testStdoutTag}######$!`);
|
||||
|
||||
// Timing with nanosecond precision
|
||||
const startTime = getTimeNs();
|
||||
let returnValue;
|
||||
let error = null;
|
||||
|
||||
try {
|
||||
returnValue = fn(...args);
|
||||
|
||||
// Handle promises (async functions)
|
||||
if (returnValue instanceof Promise) {
|
||||
return returnValue.then(
|
||||
(resolved) => {
|
||||
const endTime = getTimeNs();
|
||||
const durationNs = getDurationNs(startTime, endTime);
|
||||
recordResult(testModulePath, testClassName, testFunctionName, funcName, invocationId, args, resolved, null, durationNs);
|
||||
// Print end tag (no duration for behavior mode)
|
||||
console.log(`!######${testStdoutTag}######!`);
|
||||
return resolved;
|
||||
},
|
||||
(err) => {
|
||||
const endTime = getTimeNs();
|
||||
const durationNs = getDurationNs(startTime, endTime);
|
||||
recordResult(testModulePath, testClassName, testFunctionName, funcName, invocationId, args, null, err, durationNs);
|
||||
console.log(`!######${testStdoutTag}######!`);
|
||||
throw err;
|
||||
}
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
error = e;
|
||||
}
|
||||
|
||||
const endTime = getTimeNs();
|
||||
const durationNs = getDurationNs(startTime, endTime);
|
||||
recordResult(testModulePath, testClassName, testFunctionName, funcName, invocationId, args, returnValue, error, durationNs);
|
||||
|
||||
// Print end tag (no duration for behavior mode, matching Python)
|
||||
console.log(`!######${testStdoutTag}######!`);
|
||||
|
||||
if (error) throw error;
|
||||
return returnValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture a function call for PERFORMANCE benchmarking only.
|
||||
*
|
||||
* This is a lightweight instrumentation that only measures timing.
|
||||
* It prints start/end tags to stdout (no SQLite writes, no serialization overhead).
|
||||
* Used when we've already verified behavior and just need accurate timing.
|
||||
*
|
||||
* The timing measurement is done exactly around the function call for accuracy.
|
||||
*
|
||||
* Output format matches Python's codeflash_performance wrapper:
|
||||
* Start: !$######test_module:test_class.test_name:func_name:loop_index:invocation_id######$!
|
||||
* End: !######test_module:test_class.test_name:func_name:loop_index:invocation_id:duration_ns######!
|
||||
*
|
||||
* @param {string} funcName - Name of the function being tested (static)
|
||||
* @param {string} lineId - Line number identifier in test file (static)
|
||||
* @param {Function} fn - The function to call
|
||||
* @param {...any} args - Arguments to pass to the function
|
||||
* @returns {any} - The function's return value
|
||||
* @throws {Error} - Re-throws any error from the function
|
||||
*/
|
||||
function capturePerf(funcName, lineId, fn, ...args) {
|
||||
// Get test context
|
||||
const testModulePath = TEST_MODULE || currentTestName || 'unknown';
|
||||
const testClassName = null; // Jest doesn't use classes like Python
|
||||
const testFunctionName = currentTestName || 'unknown';
|
||||
|
||||
// Create testId for invocation tracking (matches Python format)
|
||||
const testId = `${testModulePath}:${testClassName}:${testFunctionName}:${lineId}:${LOOP_INDEX}`;
|
||||
|
||||
// Get invocation index (increments if same testId seen again)
|
||||
const invocationIndex = getInvocationIndex(testId);
|
||||
const invocationId = `${lineId}_${invocationIndex}`;
|
||||
|
||||
// Format stdout tag (matches Python format)
|
||||
const testStdoutTag = `${testModulePath}:${testClassName ? testClassName + '.' : ''}${testFunctionName}:${funcName}:${LOOP_INDEX}:${invocationId}`;
|
||||
|
||||
// Print start tag
|
||||
console.log(`!$######${testStdoutTag}######$!`);
|
||||
|
||||
// Timing with nanosecond precision - exactly around the function call
|
||||
let returnValue;
|
||||
let error = null;
|
||||
let durationNs;
|
||||
|
||||
try {
|
||||
const startTime = getTimeNs();
|
||||
returnValue = fn(...args);
|
||||
const endTime = getTimeNs();
|
||||
durationNs = getDurationNs(startTime, endTime);
|
||||
|
||||
// Handle promises (async functions)
|
||||
if (returnValue instanceof Promise) {
|
||||
return returnValue.then(
|
||||
(resolved) => {
|
||||
// For async, we measure until resolution
|
||||
const asyncEndTime = getTimeNs();
|
||||
const asyncDurationNs = getDurationNs(startTime, asyncEndTime);
|
||||
// Print end tag with timing
|
||||
console.log(`!######${testStdoutTag}:${asyncDurationNs}######!`);
|
||||
return resolved;
|
||||
},
|
||||
(err) => {
|
||||
const asyncEndTime = getTimeNs();
|
||||
const asyncDurationNs = getDurationNs(startTime, asyncEndTime);
|
||||
// Print end tag with timing even on error
|
||||
console.log(`!######${testStdoutTag}:${asyncDurationNs}######!`);
|
||||
throw err;
|
||||
}
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
const endTime = getTimeNs();
|
||||
// For sync errors, we still need to calculate duration
|
||||
// Use a fallback if we didn't capture startTime yet
|
||||
durationNs = 0;
|
||||
error = e;
|
||||
}
|
||||
|
||||
// Print end tag with timing (no rounding)
|
||||
console.log(`!######${testStdoutTag}:${durationNs}######!`);
|
||||
|
||||
if (error) throw error;
|
||||
return returnValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if performance measurements have stabilized.
|
||||
* Implements the same stability criteria as Python's pytest_plugin.py.
|
||||
*
|
||||
* @param {number[]} runtimes - Array of runtime measurements
|
||||
* @param {number} windowSize - Size of the window to check
|
||||
* @returns {boolean} - True if performance has stabilized
|
||||
*/
|
||||
function checkStability(runtimes, windowSize) {
|
||||
if (runtimes.length < windowSize || windowSize < 3) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get recent window
|
||||
const window = runtimes.slice(-windowSize);
|
||||
|
||||
// Check center tolerance (all values within ±0.25% of median)
|
||||
const sorted = [...window].sort((a, b) => a - b);
|
||||
const medianIndex = Math.floor(sorted.length / 2);
|
||||
const median = sorted[medianIndex];
|
||||
const centerTolerance = median * STABILITY_CENTER_TOLERANCE;
|
||||
|
||||
const withinCenter = window.every(v => Math.abs(v - median) <= centerTolerance);
|
||||
if (!withinCenter) return false;
|
||||
|
||||
// Check spread tolerance (max-min ≤ 0.25% of min)
|
||||
const minVal = Math.min(...window);
|
||||
const maxVal = Math.max(...window);
|
||||
const spreadTolerance = minVal * STABILITY_SPREAD_TOLERANCE;
|
||||
|
||||
return (maxVal - minVal) <= spreadTolerance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture a function call with internal looping for stable performance measurement.
|
||||
*
|
||||
* This function runs the target function multiple times within a single test execution,
|
||||
* similar to Python's pytest_plugin behavior. It provides stable timing by:
|
||||
* - Running multiple iterations to warm up JIT
|
||||
* - Continuing until timing stabilizes or time limit is reached
|
||||
* - Outputting timing data for each iteration
|
||||
*
|
||||
* Environment Variables:
|
||||
* CODEFLASH_MIN_LOOPS - Minimum number of loops (default: 5)
|
||||
* CODEFLASH_MAX_LOOPS - Maximum number of loops (default: 100000)
|
||||
* CODEFLASH_TARGET_DURATION_MS - Target duration in ms (default: 10000)
|
||||
* CODEFLASH_STABILITY_CHECK - Enable stability checking (default: true)
|
||||
*
|
||||
* @param {string} funcName - Name of the function being tested (static)
|
||||
* @param {string} lineId - Line number identifier in test file (static)
|
||||
* @param {Function} fn - The function to call
|
||||
* @param {...any} args - Arguments to pass to the function
|
||||
* @returns {any} - The function's return value from the last iteration
|
||||
* @throws {Error} - Re-throws any error from the function
|
||||
*/
|
||||
function capturePerfLooped(funcName, lineId, fn, ...args) {
|
||||
// Get test context
|
||||
const testModulePath = TEST_MODULE || currentTestName || 'unknown';
|
||||
const testClassName = null; // Jest doesn't use classes like Python
|
||||
const testFunctionName = currentTestName || 'unknown';
|
||||
|
||||
// Create base testId for invocation tracking
|
||||
const baseTestId = `${testModulePath}:${testClassName}:${testFunctionName}:${lineId}`;
|
||||
|
||||
// Get invocation index (same call site in loops within test)
|
||||
const invocationIndex = getInvocationIndex(baseTestId + ':base');
|
||||
const invocationId = `${lineId}_${invocationIndex}`;
|
||||
|
||||
// Track runtimes for stability checking
|
||||
const runtimes = [];
|
||||
let returnValue;
|
||||
let error = null;
|
||||
|
||||
const loopStartTime = Date.now();
|
||||
let loopCount = 0;
|
||||
|
||||
while (true) {
|
||||
loopCount++;
|
||||
|
||||
// Create per-loop stdout tag
|
||||
const testStdoutTag = `${testModulePath}:${testClassName ? testClassName + '.' : ''}${testFunctionName}:${funcName}:${loopCount}:${invocationId}`;
|
||||
|
||||
// Print start tag
|
||||
console.log(`!$######${testStdoutTag}######$!`);
|
||||
|
||||
// Timing with nanosecond precision
|
||||
let durationNs;
|
||||
try {
|
||||
const startTime = getTimeNs();
|
||||
returnValue = fn(...args);
|
||||
const endTime = getTimeNs();
|
||||
durationNs = getDurationNs(startTime, endTime);
|
||||
|
||||
// Handle promises - for async, we can't easily loop internally
|
||||
// Fall back to single execution for async functions
|
||||
if (returnValue instanceof Promise) {
|
||||
return returnValue.then(
|
||||
(resolved) => {
|
||||
const asyncEndTime = getTimeNs();
|
||||
const asyncDurationNs = getDurationNs(startTime, asyncEndTime);
|
||||
console.log(`!######${testStdoutTag}:${asyncDurationNs}######!`);
|
||||
return resolved;
|
||||
},
|
||||
(err) => {
|
||||
const asyncEndTime = getTimeNs();
|
||||
const asyncDurationNs = getDurationNs(startTime, asyncEndTime);
|
||||
console.log(`!######${testStdoutTag}:${asyncDurationNs}######!`);
|
||||
throw err;
|
||||
}
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
durationNs = 0;
|
||||
error = e;
|
||||
// Print end tag even on error
|
||||
console.log(`!######${testStdoutTag}:${durationNs}######!`);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Print end tag with timing
|
||||
console.log(`!######${testStdoutTag}:${durationNs}######!`);
|
||||
|
||||
// Track runtime for stability
|
||||
runtimes.push(durationNs);
|
||||
|
||||
// Check stopping conditions
|
||||
const elapsedMs = Date.now() - loopStartTime;
|
||||
|
||||
// Stop if we've reached max loops
|
||||
if (loopCount >= MAX_LOOPS) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Stop if we've reached min loops AND exceeded time limit
|
||||
if (loopCount >= MIN_LOOPS && elapsedMs >= TARGET_DURATION_MS) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Stability check
|
||||
if (STABILITY_CHECK && loopCount >= MIN_LOOPS) {
|
||||
// Estimate total loops based on current rate
|
||||
const rate = loopCount / elapsedMs;
|
||||
const estimatedTotalLoops = Math.floor(rate * TARGET_DURATION_MS);
|
||||
const windowSize = Math.max(3, Math.floor(STABILITY_WINDOW_SIZE * estimatedTotalLoops));
|
||||
|
||||
if (checkStability(runtimes, windowSize)) {
|
||||
// Performance has stabilized
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return returnValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture multiple invocations for benchmarking.
|
||||
*
|
||||
* @param {string} funcName - Name of the function being tested
|
||||
* @param {string} lineId - Line number identifier
|
||||
* @param {Function} fn - The function to call
|
||||
* @param {Array<Array>} argsList - List of argument arrays to test
|
||||
* @returns {Array} - Array of return values
|
||||
*/
|
||||
function captureMultiple(funcName, lineId, fn, argsList) {
|
||||
return argsList.map(args => capture(funcName, lineId, fn, ...args));
|
||||
}
|
||||
|
||||
/**
|
||||
* Write remaining JSON results to file (fallback mode).
|
||||
* Called automatically via Jest afterAll hook.
|
||||
*/
|
||||
function writeResults() {
|
||||
// Close SQLite connection if open
|
||||
if (db) {
|
||||
try {
|
||||
db.close();
|
||||
} catch (e) {
|
||||
// Ignore close errors
|
||||
}
|
||||
db = null;
|
||||
return;
|
||||
}
|
||||
|
||||
// Write JSON fallback if SQLite wasn't used
|
||||
if (results.length === 0) return;
|
||||
|
||||
try {
|
||||
// Write as JSON for fallback parsing
|
||||
const jsonPath = OUTPUT_FILE.replace('.sqlite', '.json');
|
||||
const output = {
|
||||
version: '1.0.0',
|
||||
loopIndex: LOOP_INDEX,
|
||||
timestamp: Date.now(),
|
||||
results
|
||||
};
|
||||
fs.writeFileSync(jsonPath, JSON.stringify(output, null, 2));
|
||||
} catch (e) {
|
||||
console.error('[codeflash] Error writing JSON results:', e.message);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Clear all recorded results.
|
||||
* Useful for resetting between test files.
|
||||
*/
|
||||
function clearResults() {
|
||||
results.length = 0;
|
||||
resetInvocationCounters();
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the current results buffer.
|
||||
* Useful for debugging or custom result handling.
|
||||
*
|
||||
* @returns {Array} - Current results buffer
|
||||
*/
|
||||
function getResults() {
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Set the current test name.
|
||||
* Called automatically via Jest beforeEach hook.
|
||||
*
|
||||
* @param {string} name - Test name
|
||||
*/
|
||||
function setTestName(name) {
|
||||
currentTestName = name;
|
||||
resetInvocationCounters();
|
||||
}
|
||||
|
||||
// Jest lifecycle hooks - these run automatically when this module is imported
|
||||
if (typeof beforeEach !== 'undefined') {
|
||||
beforeEach(() => {
|
||||
// Get current test name from Jest's expect state
|
||||
try {
|
||||
currentTestName = expect.getState().currentTestName || 'unknown';
|
||||
} catch (e) {
|
||||
currentTestName = 'unknown';
|
||||
}
|
||||
// Reset invocation counters for each test
|
||||
resetInvocationCounters();
|
||||
});
|
||||
}
|
||||
|
||||
if (typeof afterAll !== 'undefined') {
|
||||
afterAll(() => {
|
||||
writeResults();
|
||||
});
|
||||
}
|
||||
|
||||
// Export public API
|
||||
module.exports = {
|
||||
capture, // Behavior verification (writes to SQLite)
|
||||
capturePerf, // Performance benchmarking (prints to stdout only, single run)
|
||||
capturePerfLooped, // Performance benchmarking with internal looping
|
||||
captureMultiple,
|
||||
writeResults,
|
||||
clearResults,
|
||||
getResults,
|
||||
setTestName,
|
||||
safeSerialize,
|
||||
safeDeserialize,
|
||||
initDatabase,
|
||||
resetInvocationCounters,
|
||||
getInvocationIndex,
|
||||
checkStability,
|
||||
// Serializer info
|
||||
getSerializerType: serializer.getSerializerType,
|
||||
// Constants
|
||||
LOOP_INDEX,
|
||||
OUTPUT_FILE,
|
||||
TEST_ITERATION,
|
||||
MIN_LOOPS,
|
||||
MAX_LOOPS,
|
||||
TARGET_DURATION_MS,
|
||||
STABILITY_CHECK
|
||||
};
|
||||
|
|
@ -1,851 +0,0 @@
|
|||
/**
|
||||
* Codeflash Universal Serializer
|
||||
*
|
||||
* A robust serialization system for JavaScript values that:
|
||||
* 1. Prefers V8 serialization (Node.js native) - fastest, handles all JS types
|
||||
* 2. Falls back to msgpack with custom extensions (for Bun/browser environments)
|
||||
*
|
||||
* Supports:
|
||||
* - All primitive types (null, undefined, boolean, number, string, bigint, symbol)
|
||||
* - Special numbers (NaN, Infinity, -Infinity)
|
||||
* - Objects, Arrays (including sparse arrays)
|
||||
* - Map, Set, WeakMap references, WeakSet references
|
||||
* - Date, RegExp, Error (and subclasses)
|
||||
* - TypedArrays (Int8Array, Uint8Array, Float32Array, etc.)
|
||||
* - ArrayBuffer, SharedArrayBuffer, DataView
|
||||
* - Circular references
|
||||
* - Functions (by reference/name only)
|
||||
*
|
||||
* Usage:
|
||||
* const { serialize, deserialize, getSerializerType } = require('./codeflash-serializer');
|
||||
*
|
||||
* const buffer = serialize(value);
|
||||
* const restored = deserialize(buffer);
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
// ============================================================================
|
||||
// SERIALIZER DETECTION
|
||||
// ============================================================================
|
||||
|
||||
let useV8 = false;
|
||||
let v8Module = null;
|
||||
|
||||
// Try to load V8 module (available in Node.js)
|
||||
try {
|
||||
v8Module = require('v8');
|
||||
// Verify serialize/deserialize are available
|
||||
if (typeof v8Module.serialize === 'function' && typeof v8Module.deserialize === 'function') {
|
||||
// Perform a self-test to verify V8 serialization works correctly
|
||||
// This catches cases like Jest's VM context where V8 serialization
|
||||
// produces data that deserializes incorrectly (Maps become plain objects)
|
||||
const testMap = new Map([['__test__', 1]]);
|
||||
const testBuffer = v8Module.serialize(testMap);
|
||||
const testRestored = v8Module.deserialize(testBuffer);
|
||||
|
||||
if (testRestored instanceof Map && testRestored.get('__test__') === 1) {
|
||||
useV8 = true;
|
||||
} else {
|
||||
// V8 serialization is broken in this environment (e.g., Jest)
|
||||
useV8 = false;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
// V8 not available (Bun, browser, etc.)
|
||||
}
|
||||
|
||||
// Load msgpack as fallback
|
||||
let msgpack = null;
|
||||
try {
|
||||
msgpack = require('@msgpack/msgpack');
|
||||
} catch (e) {
|
||||
// msgpack not installed
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the serializer type being used.
|
||||
* @returns {string} - 'v8' or 'msgpack'
|
||||
*/
|
||||
function getSerializerType() {
|
||||
return useV8 ? 'v8' : 'msgpack';
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// V8 SERIALIZATION (PRIMARY)
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Serialize a value using V8's native serialization.
|
||||
* This handles all JavaScript types including:
|
||||
* - Primitives, Objects, Arrays
|
||||
* - Map, Set, Date, RegExp, Error
|
||||
* - TypedArrays, ArrayBuffer
|
||||
* - Circular references
|
||||
*
|
||||
* @param {any} value - Value to serialize
|
||||
* @returns {Buffer} - Serialized buffer
|
||||
*/
|
||||
function serializeV8(value) {
|
||||
try {
|
||||
return v8Module.serialize(value);
|
||||
} catch (e) {
|
||||
// V8 can't serialize some things (functions, symbols in some contexts)
|
||||
// Fall back to wrapped serialization
|
||||
return v8Module.serialize(wrapForV8(value));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deserialize a V8-serialized buffer.
|
||||
*
|
||||
* @param {Buffer} buffer - Serialized buffer
|
||||
* @returns {any} - Deserialized value
|
||||
*/
|
||||
function deserializeV8(buffer) {
|
||||
const value = v8Module.deserialize(buffer);
|
||||
return unwrapFromV8(value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wrap values that V8 can't serialize natively.
|
||||
* V8 can't serialize: functions, symbols (in some cases)
|
||||
*/
|
||||
function wrapForV8(value, seen = new WeakMap()) {
|
||||
if (value === null || value === undefined) return value;
|
||||
|
||||
const type = typeof value;
|
||||
|
||||
// Primitives that V8 handles
|
||||
if (type === 'number' || type === 'string' || type === 'boolean' || type === 'bigint') {
|
||||
return value;
|
||||
}
|
||||
|
||||
// Symbols - wrap with marker
|
||||
if (type === 'symbol') {
|
||||
return { __codeflash_type__: 'Symbol', description: value.description };
|
||||
}
|
||||
|
||||
// Functions - wrap with marker
|
||||
if (type === 'function') {
|
||||
return {
|
||||
__codeflash_type__: 'Function',
|
||||
name: value.name || 'anonymous',
|
||||
// Can't serialize function body reliably
|
||||
};
|
||||
}
|
||||
|
||||
// Objects
|
||||
if (type === 'object') {
|
||||
// Check for circular reference
|
||||
if (seen.has(value)) {
|
||||
return seen.get(value);
|
||||
}
|
||||
|
||||
// V8 handles most objects natively
|
||||
// Just need to recurse into arrays and plain objects to wrap nested functions/symbols
|
||||
|
||||
if (Array.isArray(value)) {
|
||||
const wrapped = [];
|
||||
seen.set(value, wrapped);
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
if (i in value) {
|
||||
wrapped[i] = wrapForV8(value[i], seen);
|
||||
}
|
||||
}
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
// V8 handles these natively
|
||||
if (value instanceof Date || value instanceof RegExp || value instanceof Error ||
|
||||
value instanceof Map || value instanceof Set ||
|
||||
ArrayBuffer.isView(value) || value instanceof ArrayBuffer) {
|
||||
return value;
|
||||
}
|
||||
|
||||
// Plain objects - recurse
|
||||
const wrapped = {};
|
||||
seen.set(value, wrapped);
|
||||
for (const key of Object.keys(value)) {
|
||||
wrapped[key] = wrapForV8(value[key], seen);
|
||||
}
|
||||
return wrapped;
|
||||
}
|
||||
|
||||
return value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Unwrap values that were wrapped for V8 serialization.
|
||||
*/
|
||||
function unwrapFromV8(value, seen = new WeakMap()) {
|
||||
if (value === null || value === undefined) return value;
|
||||
|
||||
const type = typeof value;
|
||||
|
||||
if (type !== 'object') return value;
|
||||
|
||||
// Check for circular reference
|
||||
if (seen.has(value)) {
|
||||
return seen.get(value);
|
||||
}
|
||||
|
||||
// Check for wrapped types
|
||||
if (value.__codeflash_type__) {
|
||||
switch (value.__codeflash_type__) {
|
||||
case 'Symbol':
|
||||
return Symbol(value.description);
|
||||
case 'Function':
|
||||
// Can't restore function body, return a placeholder
|
||||
const fn = function() { throw new Error(`Deserialized function placeholder: ${value.name}`); };
|
||||
Object.defineProperty(fn, 'name', { value: value.name });
|
||||
return fn;
|
||||
default:
|
||||
// Unknown wrapped type, return as-is
|
||||
return value;
|
||||
}
|
||||
}
|
||||
|
||||
// Arrays
|
||||
if (Array.isArray(value)) {
|
||||
const unwrapped = [];
|
||||
seen.set(value, unwrapped);
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
if (i in value) {
|
||||
unwrapped[i] = unwrapFromV8(value[i], seen);
|
||||
}
|
||||
}
|
||||
return unwrapped;
|
||||
}
|
||||
|
||||
// V8 restores these natively
|
||||
if (value instanceof Date || value instanceof RegExp || value instanceof Error ||
|
||||
value instanceof Map || value instanceof Set ||
|
||||
ArrayBuffer.isView(value) || value instanceof ArrayBuffer) {
|
||||
return value;
|
||||
}
|
||||
|
||||
// Plain objects - recurse
|
||||
const unwrapped = {};
|
||||
seen.set(value, unwrapped);
|
||||
for (const key of Object.keys(value)) {
|
||||
unwrapped[key] = unwrapFromV8(value[key], seen);
|
||||
}
|
||||
return unwrapped;
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// MSGPACK SERIALIZATION (FALLBACK)
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Extension type IDs for msgpack.
|
||||
* Using negative IDs to avoid conflicts with user-defined extensions.
|
||||
*/
|
||||
const EXT_TYPES = {
|
||||
UNDEFINED: 0x01,
|
||||
NAN: 0x02,
|
||||
INFINITY_POS: 0x03,
|
||||
INFINITY_NEG: 0x04,
|
||||
BIGINT: 0x05,
|
||||
SYMBOL: 0x06,
|
||||
DATE: 0x07,
|
||||
REGEXP: 0x08,
|
||||
ERROR: 0x09,
|
||||
MAP: 0x0A,
|
||||
SET: 0x0B,
|
||||
INT8ARRAY: 0x10,
|
||||
UINT8ARRAY: 0x11,
|
||||
UINT8CLAMPEDARRAY: 0x12,
|
||||
INT16ARRAY: 0x13,
|
||||
UINT16ARRAY: 0x14,
|
||||
INT32ARRAY: 0x15,
|
||||
UINT32ARRAY: 0x16,
|
||||
FLOAT32ARRAY: 0x17,
|
||||
FLOAT64ARRAY: 0x18,
|
||||
BIGINT64ARRAY: 0x19,
|
||||
BIGUINT64ARRAY: 0x1A,
|
||||
ARRAYBUFFER: 0x1B,
|
||||
DATAVIEW: 0x1C,
|
||||
FUNCTION: 0x1D,
|
||||
CIRCULAR_REF: 0x1E,
|
||||
SPARSE_ARRAY: 0x1F,
|
||||
};
|
||||
|
||||
/**
|
||||
* Create msgpack extension codec for JavaScript types.
|
||||
*/
|
||||
function createMsgpackCodec() {
|
||||
const extensionCodec = new msgpack.ExtensionCodec();
|
||||
|
||||
// Undefined
|
||||
extensionCodec.register({
|
||||
type: EXT_TYPES.UNDEFINED,
|
||||
encode: (value) => {
|
||||
if (value === undefined) return new Uint8Array(0);
|
||||
return null;
|
||||
},
|
||||
decode: () => undefined,
|
||||
});
|
||||
|
||||
// NaN
|
||||
extensionCodec.register({
|
||||
type: EXT_TYPES.NAN,
|
||||
encode: (value) => {
|
||||
if (typeof value === 'number' && Number.isNaN(value)) return new Uint8Array(0);
|
||||
return null;
|
||||
},
|
||||
decode: () => NaN,
|
||||
});
|
||||
|
||||
// Positive Infinity
|
||||
extensionCodec.register({
|
||||
type: EXT_TYPES.INFINITY_POS,
|
||||
encode: (value) => {
|
||||
if (value === Infinity) return new Uint8Array(0);
|
||||
return null;
|
||||
},
|
||||
decode: () => Infinity,
|
||||
});
|
||||
|
||||
// Negative Infinity
|
||||
extensionCodec.register({
|
||||
type: EXT_TYPES.INFINITY_NEG,
|
||||
encode: (value) => {
|
||||
if (value === -Infinity) return new Uint8Array(0);
|
||||
return null;
|
||||
},
|
||||
decode: () => -Infinity,
|
||||
});
|
||||
|
||||
// BigInt
|
||||
extensionCodec.register({
|
||||
type: EXT_TYPES.BIGINT,
|
||||
encode: (value) => {
|
||||
if (typeof value === 'bigint') {
|
||||
const str = value.toString();
|
||||
return new TextEncoder().encode(str);
|
||||
}
|
||||
return null;
|
||||
},
|
||||
decode: (data) => {
|
||||
const str = new TextDecoder().decode(data);
|
||||
return BigInt(str);
|
||||
},
|
||||
});
|
||||
|
||||
// Symbol
|
||||
extensionCodec.register({
|
||||
type: EXT_TYPES.SYMBOL,
|
||||
encode: (value) => {
|
||||
if (typeof value === 'symbol') {
|
||||
// Distinguish between undefined description and empty string
|
||||
// Use a special marker for undefined description
|
||||
const desc = value.description;
|
||||
if (desc === undefined) {
|
||||
return new TextEncoder().encode('\x00__UNDEF__');
|
||||
}
|
||||
return new TextEncoder().encode(desc);
|
||||
}
|
||||
return null;
|
||||
},
|
||||
decode: (data) => {
|
||||
const description = new TextDecoder().decode(data);
|
||||
// Check for undefined marker
|
||||
if (description === '\x00__UNDEF__') {
|
||||
return Symbol();
|
||||
}
|
||||
return Symbol(description);
|
||||
},
|
||||
});
|
||||
|
||||
// Note: Date is handled via marker objects in prepareForMsgpack/restoreFromMsgpack
|
||||
// because msgpack's built-in timestamp extension doesn't properly handle NaN (Invalid Date)
|
||||
|
||||
// RegExp - use Object.prototype.toString for cross-context detection
|
||||
extensionCodec.register({
|
||||
type: EXT_TYPES.REGEXP,
|
||||
encode: (value) => {
|
||||
if (Object.prototype.toString.call(value) === '[object RegExp]') {
|
||||
const obj = { source: value.source, flags: value.flags };
|
||||
return msgpack.encode(obj);
|
||||
}
|
||||
return null;
|
||||
},
|
||||
decode: (data) => {
|
||||
const obj = msgpack.decode(data);
|
||||
return new RegExp(obj.source, obj.flags);
|
||||
},
|
||||
});
|
||||
|
||||
// Error - use Object.prototype.toString for cross-context detection
|
||||
extensionCodec.register({
|
||||
type: EXT_TYPES.ERROR,
|
||||
encode: (value) => {
|
||||
// Check for Error-like objects (cross-VM-context compatible)
|
||||
if (Object.prototype.toString.call(value) === '[object Error]' ||
|
||||
(value && value.name && value.message !== undefined && value.stack !== undefined)) {
|
||||
const obj = {
|
||||
name: value.name,
|
||||
message: value.message,
|
||||
stack: value.stack,
|
||||
// Include custom properties
|
||||
...Object.fromEntries(
|
||||
Object.entries(value).filter(([k]) => !['name', 'message', 'stack'].includes(k))
|
||||
),
|
||||
};
|
||||
return msgpack.encode(obj);
|
||||
}
|
||||
return null;
|
||||
},
|
||||
decode: (data) => {
|
||||
const obj = msgpack.decode(data);
|
||||
let ErrorClass = Error;
|
||||
// Try to use the appropriate error class
|
||||
const errorClasses = {
|
||||
TypeError, RangeError, SyntaxError, ReferenceError,
|
||||
URIError, EvalError, Error
|
||||
};
|
||||
if (obj.name in errorClasses) {
|
||||
ErrorClass = errorClasses[obj.name];
|
||||
}
|
||||
const error = new ErrorClass(obj.message);
|
||||
error.stack = obj.stack;
|
||||
// Restore custom properties
|
||||
for (const [key, val] of Object.entries(obj)) {
|
||||
if (!['name', 'message', 'stack'].includes(key)) {
|
||||
error[key] = val;
|
||||
}
|
||||
}
|
||||
return error;
|
||||
},
|
||||
});
|
||||
|
||||
// Function (limited - can't serialize body)
|
||||
extensionCodec.register({
|
||||
type: EXT_TYPES.FUNCTION,
|
||||
encode: (value) => {
|
||||
if (typeof value === 'function') {
|
||||
return new TextEncoder().encode(value.name || 'anonymous');
|
||||
}
|
||||
return null;
|
||||
},
|
||||
decode: (data) => {
|
||||
const name = new TextDecoder().decode(data);
|
||||
const fn = function() { throw new Error(`Deserialized function placeholder: ${name}`); };
|
||||
Object.defineProperty(fn, 'name', { value: name });
|
||||
return fn;
|
||||
},
|
||||
});
|
||||
|
||||
return extensionCodec;
|
||||
}
|
||||
|
||||
// Singleton codec instance
|
||||
let msgpackCodec = null;
|
||||
|
||||
function getMsgpackCodec() {
|
||||
if (!msgpackCodec && msgpack) {
|
||||
msgpackCodec = createMsgpackCodec();
|
||||
}
|
||||
return msgpackCodec;
|
||||
}
|
||||
|
||||
/**
|
||||
* Prepare a value for msgpack serialization.
|
||||
* Handles types that need special treatment beyond extensions.
|
||||
*/
|
||||
function prepareForMsgpack(value, seen = new Map(), refId = { current: 0 }) {
|
||||
if (value === null) return null;
|
||||
// undefined needs special handling because msgpack converts it to null
|
||||
if (value === undefined) return { __codeflash_undefined__: true };
|
||||
|
||||
const type = typeof value;
|
||||
|
||||
// Special number values that msgpack doesn't handle correctly
|
||||
if (type === 'number') {
|
||||
if (Number.isNaN(value)) return { __codeflash_nan__: true };
|
||||
if (value === Infinity) return { __codeflash_infinity__: true };
|
||||
if (value === -Infinity) return { __codeflash_neg_infinity__: true };
|
||||
return value;
|
||||
}
|
||||
|
||||
// Primitives that msgpack handles or our extensions handle
|
||||
if (type === 'string' || type === 'boolean' ||
|
||||
type === 'bigint' || type === 'symbol' || type === 'function') {
|
||||
return value;
|
||||
}
|
||||
|
||||
if (type !== 'object') return value;
|
||||
|
||||
// Check for circular reference
|
||||
if (seen.has(value)) {
|
||||
return { __codeflash_circular__: seen.get(value) };
|
||||
}
|
||||
|
||||
// Assign reference ID for potential circular refs
|
||||
const id = refId.current++;
|
||||
seen.set(value, id);
|
||||
|
||||
// Use toString for cross-VM-context type detection
|
||||
const tag = Object.prototype.toString.call(value);
|
||||
|
||||
// Date - handle specially because msgpack's built-in timestamp doesn't handle NaN
|
||||
if (tag === '[object Date]') {
|
||||
const time = value.getTime();
|
||||
// Store as marker object with the timestamp
|
||||
// We use a string representation to preserve NaN
|
||||
return {
|
||||
__codeflash_date__: Number.isNaN(time) ? '__NAN__' : time,
|
||||
__id__: id,
|
||||
};
|
||||
}
|
||||
|
||||
// RegExp, Error - handled by extensions
|
||||
if (tag === '[object RegExp]' || tag === '[object Error]') {
|
||||
return value;
|
||||
}
|
||||
|
||||
// Map (use toString for cross-VM-context)
|
||||
if (tag === '[object Map]') {
|
||||
const entries = [];
|
||||
for (const [k, v] of value) {
|
||||
entries.push([prepareForMsgpack(k, seen, refId), prepareForMsgpack(v, seen, refId)]);
|
||||
}
|
||||
return { __codeflash_map__: entries, __id__: id };
|
||||
}
|
||||
|
||||
// Set (use toString for cross-VM-context)
|
||||
if (tag === '[object Set]') {
|
||||
const values = [];
|
||||
for (const v of value) {
|
||||
values.push(prepareForMsgpack(v, seen, refId));
|
||||
}
|
||||
return { __codeflash_set__: values, __id__: id };
|
||||
}
|
||||
|
||||
// TypedArrays (use ArrayBuffer.isView which works cross-context)
|
||||
if (ArrayBuffer.isView(value) && tag !== '[object DataView]') {
|
||||
return {
|
||||
__codeflash_typedarray__: value.constructor.name,
|
||||
data: Array.from(value),
|
||||
__id__: id,
|
||||
};
|
||||
}
|
||||
|
||||
// DataView (use toString for cross-VM-context)
|
||||
if (tag === '[object DataView]') {
|
||||
return {
|
||||
__codeflash_dataview__: true,
|
||||
data: Array.from(new Uint8Array(value.buffer, value.byteOffset, value.byteLength)),
|
||||
__id__: id,
|
||||
};
|
||||
}
|
||||
|
||||
// ArrayBuffer (use toString for cross-VM-context)
|
||||
if (tag === '[object ArrayBuffer]') {
|
||||
return {
|
||||
__codeflash_arraybuffer__: true,
|
||||
data: Array.from(new Uint8Array(value)),
|
||||
__id__: id,
|
||||
};
|
||||
}
|
||||
|
||||
// Arrays - always wrap in marker to preserve __id__ for circular references
|
||||
// (msgpack doesn't preserve non-numeric properties on arrays)
|
||||
if (Array.isArray(value)) {
|
||||
const isSparse = value.length > 0 && Object.keys(value).length !== value.length;
|
||||
if (isSparse) {
|
||||
// Sparse array - store as object with indices
|
||||
const sparse = { __codeflash_sparse_array__: true, length: value.length, elements: {}, __id__: id };
|
||||
for (const key of Object.keys(value)) {
|
||||
sparse.elements[key] = prepareForMsgpack(value[key], seen, refId);
|
||||
}
|
||||
return sparse;
|
||||
}
|
||||
// Dense array - wrap in marker object to preserve __id__
|
||||
const elements = [];
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
elements[i] = prepareForMsgpack(value[i], seen, refId);
|
||||
}
|
||||
return { __codeflash_array__: elements, __id__: id };
|
||||
}
|
||||
|
||||
// Plain objects
|
||||
const obj = { __id__: id };
|
||||
for (const key of Object.keys(value)) {
|
||||
obj[key] = prepareForMsgpack(value[key], seen, refId);
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
/**
|
||||
* Restore a value after msgpack deserialization.
|
||||
*/
|
||||
function restoreFromMsgpack(value, refs = new Map()) {
|
||||
if (value === null || value === undefined) return value;
|
||||
|
||||
const type = typeof value;
|
||||
if (type !== 'object') return value;
|
||||
|
||||
// Built-in types that msgpack handles via extensions - return as-is
|
||||
// These should NOT be treated as plain objects (use toString for cross-VM-context)
|
||||
// Note: Date is handled via marker objects, so not included here
|
||||
const tag = Object.prototype.toString.call(value);
|
||||
if (tag === '[object RegExp]' || tag === '[object Error]') {
|
||||
return value;
|
||||
}
|
||||
|
||||
// Special value markers
|
||||
if (value.__codeflash_undefined__) return undefined;
|
||||
if (value.__codeflash_nan__) return NaN;
|
||||
if (value.__codeflash_infinity__) return Infinity;
|
||||
if (value.__codeflash_neg_infinity__) return -Infinity;
|
||||
|
||||
// Date marker
|
||||
if (value.__codeflash_date__ !== undefined) {
|
||||
const time = value.__codeflash_date__ === '__NAN__' ? NaN : value.__codeflash_date__;
|
||||
const date = new Date(time);
|
||||
const id = value.__id__;
|
||||
if (id !== undefined) refs.set(id, date);
|
||||
return date;
|
||||
}
|
||||
|
||||
// Check for circular reference marker
|
||||
if (value.__codeflash_circular__ !== undefined) {
|
||||
return refs.get(value.__codeflash_circular__);
|
||||
}
|
||||
|
||||
// Store reference if this object has an ID
|
||||
const id = value.__id__;
|
||||
|
||||
// Map
|
||||
if (value.__codeflash_map__) {
|
||||
const map = new Map();
|
||||
if (id !== undefined) refs.set(id, map);
|
||||
for (const [k, v] of value.__codeflash_map__) {
|
||||
map.set(restoreFromMsgpack(k, refs), restoreFromMsgpack(v, refs));
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
// Set
|
||||
if (value.__codeflash_set__) {
|
||||
const set = new Set();
|
||||
if (id !== undefined) refs.set(id, set);
|
||||
for (const v of value.__codeflash_set__) {
|
||||
set.add(restoreFromMsgpack(v, refs));
|
||||
}
|
||||
return set;
|
||||
}
|
||||
|
||||
// TypedArrays
|
||||
if (value.__codeflash_typedarray__) {
|
||||
const TypedArrayClass = globalThis[value.__codeflash_typedarray__];
|
||||
if (TypedArrayClass) {
|
||||
const arr = new TypedArrayClass(value.data);
|
||||
if (id !== undefined) refs.set(id, arr);
|
||||
return arr;
|
||||
}
|
||||
}
|
||||
|
||||
// DataView
|
||||
if (value.__codeflash_dataview__) {
|
||||
const buffer = new ArrayBuffer(value.data.length);
|
||||
new Uint8Array(buffer).set(value.data);
|
||||
const view = new DataView(buffer);
|
||||
if (id !== undefined) refs.set(id, view);
|
||||
return view;
|
||||
}
|
||||
|
||||
// ArrayBuffer
|
||||
if (value.__codeflash_arraybuffer__) {
|
||||
const buffer = new ArrayBuffer(value.data.length);
|
||||
new Uint8Array(buffer).set(value.data);
|
||||
if (id !== undefined) refs.set(id, buffer);
|
||||
return buffer;
|
||||
}
|
||||
|
||||
// Dense array marker
|
||||
if (value.__codeflash_array__) {
|
||||
const arr = [];
|
||||
if (id !== undefined) refs.set(id, arr);
|
||||
const elements = value.__codeflash_array__;
|
||||
for (let i = 0; i < elements.length; i++) {
|
||||
arr[i] = restoreFromMsgpack(elements[i], refs);
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
// Sparse array
|
||||
if (value.__codeflash_sparse_array__) {
|
||||
const arr = new Array(value.length);
|
||||
if (id !== undefined) refs.set(id, arr);
|
||||
for (const [key, val] of Object.entries(value.elements)) {
|
||||
arr[parseInt(key, 10)] = restoreFromMsgpack(val, refs);
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
// Arrays (legacy - shouldn't happen with new format, but keep for safety)
|
||||
if (Array.isArray(value)) {
|
||||
const arr = [];
|
||||
if (id !== undefined) refs.set(id, arr);
|
||||
for (let i = 0; i < value.length; i++) {
|
||||
if (i in value) {
|
||||
arr[i] = restoreFromMsgpack(value[i], refs);
|
||||
}
|
||||
}
|
||||
return arr;
|
||||
}
|
||||
|
||||
// Plain objects - remove __id__ from result
|
||||
const obj = {};
|
||||
if (id !== undefined) refs.set(id, obj);
|
||||
for (const [key, val] of Object.entries(value)) {
|
||||
if (key !== '__id__') {
|
||||
obj[key] = restoreFromMsgpack(val, refs);
|
||||
}
|
||||
}
|
||||
return obj;
|
||||
}
|
||||
|
||||
/**
|
||||
* Serialize a value using msgpack with extensions.
|
||||
*
|
||||
* @param {any} value - Value to serialize
|
||||
* @returns {Buffer} - Serialized buffer
|
||||
*/
|
||||
function serializeMsgpack(value) {
|
||||
if (!msgpack) {
|
||||
throw new Error('msgpack not available and V8 serialization not available');
|
||||
}
|
||||
|
||||
const codec = getMsgpackCodec();
|
||||
const prepared = prepareForMsgpack(value);
|
||||
const encoded = msgpack.encode(prepared, { extensionCodec: codec });
|
||||
return Buffer.from(encoded);
|
||||
}
|
||||
|
||||
/**
|
||||
* Deserialize a msgpack-serialized buffer.
|
||||
*
|
||||
* @param {Buffer|Uint8Array} buffer - Serialized buffer
|
||||
* @returns {any} - Deserialized value
|
||||
*/
|
||||
function deserializeMsgpack(buffer) {
|
||||
if (!msgpack) {
|
||||
throw new Error('msgpack not available');
|
||||
}
|
||||
|
||||
const codec = getMsgpackCodec();
|
||||
const decoded = msgpack.decode(buffer, { extensionCodec: codec });
|
||||
return restoreFromMsgpack(decoded);
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// PUBLIC API
|
||||
// ============================================================================
|
||||
|
||||
/**
|
||||
* Serialize a value using the best available method.
|
||||
* Prefers V8 serialization, falls back to msgpack.
|
||||
*
|
||||
* @param {any} value - Value to serialize
|
||||
* @returns {Buffer} - Serialized buffer with format marker
|
||||
*/
|
||||
function serialize(value) {
|
||||
// Add a format marker byte at the start
|
||||
// 0x01 = V8, 0x02 = msgpack
|
||||
if (useV8) {
|
||||
const serialized = serializeV8(value);
|
||||
const result = Buffer.allocUnsafe(serialized.length + 1);
|
||||
result[0] = 0x01;
|
||||
serialized.copy(result, 1);
|
||||
return result;
|
||||
} else {
|
||||
const serialized = serializeMsgpack(value);
|
||||
const result = Buffer.allocUnsafe(serialized.length + 1);
|
||||
result[0] = 0x02;
|
||||
serialized.copy(result, 1);
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Deserialize a buffer that was serialized with serialize().
|
||||
* Automatically detects the format from the marker byte.
|
||||
*
|
||||
* @param {Buffer|Uint8Array} buffer - Serialized buffer
|
||||
* @returns {any} - Deserialized value
|
||||
*/
|
||||
function deserialize(buffer) {
|
||||
if (!buffer || buffer.length === 0) {
|
||||
throw new Error('Empty buffer cannot be deserialized');
|
||||
}
|
||||
|
||||
const format = buffer[0];
|
||||
const data = buffer.slice(1);
|
||||
|
||||
if (format === 0x01) {
|
||||
// V8 format
|
||||
if (!useV8) {
|
||||
throw new Error('Buffer was serialized with V8 but V8 is not available');
|
||||
}
|
||||
return deserializeV8(data);
|
||||
} else if (format === 0x02) {
|
||||
// msgpack format
|
||||
return deserializeMsgpack(data);
|
||||
} else {
|
||||
throw new Error(`Unknown serialization format: ${format}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Force serialization using a specific method.
|
||||
* Useful for testing or cross-environment compatibility.
|
||||
*/
|
||||
const serializeWith = {
|
||||
v8: useV8 ? (value) => {
|
||||
const serialized = serializeV8(value);
|
||||
const result = Buffer.allocUnsafe(serialized.length + 1);
|
||||
result[0] = 0x01;
|
||||
serialized.copy(result, 1);
|
||||
return result;
|
||||
} : null,
|
||||
|
||||
msgpack: msgpack ? (value) => {
|
||||
const serialized = serializeMsgpack(value);
|
||||
const result = Buffer.allocUnsafe(serialized.length + 1);
|
||||
result[0] = 0x02;
|
||||
serialized.copy(result, 1);
|
||||
return result;
|
||||
} : null,
|
||||
};
|
||||
|
||||
// ============================================================================
|
||||
// EXPORTS
|
||||
// ============================================================================
|
||||
|
||||
module.exports = {
|
||||
// Main API
|
||||
serialize,
|
||||
deserialize,
|
||||
getSerializerType,
|
||||
|
||||
// Force specific serializer
|
||||
serializeWith,
|
||||
|
||||
// Low-level (for testing)
|
||||
serializeV8: useV8 ? serializeV8 : null,
|
||||
deserializeV8: useV8 ? deserializeV8 : null,
|
||||
serializeMsgpack: msgpack ? serializeMsgpack : null,
|
||||
deserializeMsgpack: msgpack ? deserializeMsgpack : null,
|
||||
|
||||
// Feature detection
|
||||
hasV8: useV8,
|
||||
hasMsgpack: !!msgpack,
|
||||
|
||||
// Extension types (for reference)
|
||||
EXT_TYPES,
|
||||
};
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
try {
|
||||
const Database = require('better-sqlite3');
|
||||
console.log('better-sqlite3 loaded successfully');
|
||||
const db = new Database('/tmp/test_better_sqlite.db');
|
||||
db.exec('CREATE TABLE test (id INTEGER)');
|
||||
db.exec('INSERT INTO test VALUES (1)');
|
||||
const row = db.prepare('SELECT * FROM test').get();
|
||||
console.log('Row:', row);
|
||||
db.close();
|
||||
console.log('Database test passed');
|
||||
} catch (e) {
|
||||
console.error('Error:', e.message);
|
||||
}
|
||||
|
|
@ -1,22 +0,0 @@
|
|||
const codeflash = require('./codeflash-jest-helper');
|
||||
const { reverseString } = require('./string_utils');
|
||||
|
||||
// Manually set test context
|
||||
process.env.CODEFLASH_OUTPUT_FILE = '/tmp/test_codeflash.sqlite';
|
||||
process.env.CODEFLASH_LOOP_INDEX = '1';
|
||||
process.env.CODEFLASH_TEST_MODULE = 'test_module';
|
||||
|
||||
// Mock beforeEach/afterAll for non-Jest environment
|
||||
global.expect = { getState: () => ({ currentTestName: 'manual_test' }) };
|
||||
|
||||
// Initialize database
|
||||
codeflash.initDatabase();
|
||||
codeflash.setTestName('manual_test');
|
||||
|
||||
// Capture a function call
|
||||
const result = codeflash.capture('reverseString', reverseString, 'hello');
|
||||
console.log('Result:', result);
|
||||
|
||||
// Write results
|
||||
codeflash.writeResults();
|
||||
console.log('Done');
|
||||
|
|
@ -49,8 +49,20 @@ const LOOP_INDEX = parseInt(process.env.CODEFLASH_LOOP_INDEX || '1', 10);
|
|||
const TEST_ITERATION = process.env.CODEFLASH_TEST_ITERATION || '0';
|
||||
const TEST_MODULE = process.env.CODEFLASH_TEST_MODULE || '';
|
||||
|
||||
// Looping configuration for performance benchmarking
|
||||
const MIN_LOOPS = parseInt(process.env.CODEFLASH_MIN_LOOPS || '5', 10);
|
||||
const MAX_LOOPS = parseInt(process.env.CODEFLASH_MAX_LOOPS || '100000', 10);
|
||||
const TARGET_DURATION_MS = parseInt(process.env.CODEFLASH_TARGET_DURATION_MS || '10000', 10);
|
||||
const STABILITY_CHECK = process.env.CODEFLASH_STABILITY_CHECK !== 'false';
|
||||
|
||||
// Stability checking constants (matching Python's pytest_plugin.py)
|
||||
const STABILITY_WINDOW_SIZE = 0.35; // 35% of estimated total loops
|
||||
const STABILITY_CENTER_TOLERANCE = 0.0025; // ±0.25% around median
|
||||
const STABILITY_SPREAD_TOLERANCE = 0.0025; // 0.25% window spread
|
||||
|
||||
// Current test context (set by Jest hooks)
|
||||
let currentTestName = null;
|
||||
let currentTestPath = null; // Test file path from Jest
|
||||
|
||||
// Invocation counter map: tracks how many times each testId has been seen
|
||||
// Key: testId (testModule:testClass:testFunction:lineId:loopIndex)
|
||||
|
|
@ -65,20 +77,19 @@ let db = null;
|
|||
|
||||
/**
|
||||
* Get high-resolution time in nanoseconds.
|
||||
* Cached at module load time for minimal overhead during timing.
|
||||
* Prefers process.hrtime.bigint() for nanosecond precision,
|
||||
* falls back to performance.now() * 1e6 for non-Node environments.
|
||||
*
|
||||
* @returns {bigint} - Time in nanoseconds
|
||||
* @returns {bigint|number} - Time in nanoseconds
|
||||
*/
|
||||
const getTimeNs = (() => {
|
||||
// Determine timing method once at module load, not on every call
|
||||
function getTimeNs() {
|
||||
if (typeof process !== 'undefined' && process.hrtime && process.hrtime.bigint) {
|
||||
// Node.js with BigInt hrtime support - fastest, most precise
|
||||
return () => process.hrtime.bigint();
|
||||
return process.hrtime.bigint();
|
||||
}
|
||||
// Fallback: pre-import performance module once
|
||||
// Fallback to performance.now() in milliseconds, converted to nanoseconds
|
||||
const { performance } = require('perf_hooks');
|
||||
return () => BigInt(Math.floor(performance.now() * 1_000_000));
|
||||
})();
|
||||
return BigInt(Math.floor(performance.now() * 1_000_000));
|
||||
}
|
||||
|
||||
/**
|
||||
* Calculate duration in nanoseconds.
|
||||
|
|
@ -93,6 +104,23 @@ function getDurationNs(start, end) {
|
|||
return Number(duration);
|
||||
}
|
||||
|
||||
/**
|
||||
* Sanitize a string for use in test IDs.
|
||||
* Replaces special characters that could conflict with regex extraction
|
||||
* during stdout parsing.
|
||||
*
|
||||
* Characters replaced with '_': ! # : (space) ( ) [ ] { } | \ / * ? ^ $ . + -
|
||||
*
|
||||
* @param {string} str - String to sanitize
|
||||
* @returns {string} - Sanitized string safe for test IDs
|
||||
*/
|
||||
function sanitizeTestId(str) {
|
||||
if (!str) return str;
|
||||
// Replace characters that could conflict with our delimiter pattern (######)
|
||||
// or the colon-separated format, or general regex metacharacters
|
||||
return str.replace(/[!#: ()\[\]{}|\\/*?^$.+\-]/g, '_');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get or create invocation index for a testId.
|
||||
* This mirrors Python's index tracking per wrapper function.
|
||||
|
|
@ -264,20 +292,41 @@ function capture(funcName, lineId, fn, ...args) {
|
|||
// Initialize database on first capture
|
||||
initDatabase();
|
||||
|
||||
// Get test context
|
||||
const testModulePath = TEST_MODULE || currentTestName || 'unknown';
|
||||
// Get test context (raw values for SQLite storage)
|
||||
// Use TEST_MODULE env var if set, otherwise derive from test file path
|
||||
let testModulePath;
|
||||
if (TEST_MODULE) {
|
||||
testModulePath = TEST_MODULE;
|
||||
} else if (currentTestPath) {
|
||||
// Get relative path from cwd and convert to module-style path
|
||||
const path = require('path');
|
||||
const relativePath = path.relative(process.cwd(), currentTestPath);
|
||||
// Convert to Python module-style path (e.g., "tests/test_foo.test.js" -> "tests.test_foo.test")
|
||||
// This matches what Jest's junit XML produces
|
||||
testModulePath = relativePath
|
||||
.replace(/\\/g, '/') // Handle Windows paths
|
||||
.replace(/\.js$/, '') // Remove .js extension
|
||||
.replace(/\.test$/, '.test') // Keep .test suffix
|
||||
.replace(/\//g, '.'); // Convert path separators to dots
|
||||
} else {
|
||||
testModulePath = currentTestName || 'unknown';
|
||||
}
|
||||
const testClassName = null; // Jest doesn't use classes like Python
|
||||
const testFunctionName = currentTestName || 'unknown';
|
||||
|
||||
// Sanitized versions for stdout tags (avoid regex conflicts)
|
||||
const safeModulePath = sanitizeTestId(testModulePath);
|
||||
const safeTestFunctionName = sanitizeTestId(testFunctionName);
|
||||
|
||||
// Create testId for invocation tracking (matches Python format)
|
||||
const testId = `${testModulePath}:${testClassName}:${testFunctionName}:${lineId}:${LOOP_INDEX}`;
|
||||
const testId = `${safeModulePath}:${testClassName}:${safeTestFunctionName}:${lineId}:${LOOP_INDEX}`;
|
||||
|
||||
// Get invocation index (increments if same testId seen again)
|
||||
const invocationIndex = getInvocationIndex(testId);
|
||||
const invocationId = `${lineId}_${invocationIndex}`;
|
||||
|
||||
// Format stdout tag (matches Python format)
|
||||
const testStdoutTag = `${testModulePath}:${testClassName ? testClassName + '.' : ''}${testFunctionName}:${funcName}:${LOOP_INDEX}:${invocationId}`;
|
||||
// Format stdout tag (matches Python format, uses sanitized names)
|
||||
const testStdoutTag = `${safeModulePath}:${testClassName ? testClassName + '.' : ''}${safeTestFunctionName}:${funcName}:${LOOP_INDEX}:${invocationId}`;
|
||||
|
||||
// Print start tag
|
||||
console.log(`!$######${testStdoutTag}######$!`);
|
||||
|
|
@ -347,19 +396,39 @@ function capture(funcName, lineId, fn, ...args) {
|
|||
*/
|
||||
function capturePerf(funcName, lineId, fn, ...args) {
|
||||
// Get test context
|
||||
const testModulePath = TEST_MODULE || currentTestName || 'unknown';
|
||||
// Use TEST_MODULE env var if set, otherwise derive from test file path
|
||||
let testModulePath;
|
||||
if (TEST_MODULE) {
|
||||
testModulePath = TEST_MODULE;
|
||||
} else if (currentTestPath) {
|
||||
// Get relative path from cwd and convert to module-style path
|
||||
const path = require('path');
|
||||
const relativePath = path.relative(process.cwd(), currentTestPath);
|
||||
// Convert to Python module-style path (e.g., "tests/test_foo.test.js" -> "tests.test_foo.test")
|
||||
testModulePath = relativePath
|
||||
.replace(/\\/g, '/')
|
||||
.replace(/\.js$/, '')
|
||||
.replace(/\.test$/, '.test')
|
||||
.replace(/\//g, '.');
|
||||
} else {
|
||||
testModulePath = currentTestName || 'unknown';
|
||||
}
|
||||
const testClassName = null; // Jest doesn't use classes like Python
|
||||
const testFunctionName = currentTestName || 'unknown';
|
||||
|
||||
// Sanitized versions for stdout tags (avoid regex conflicts)
|
||||
const safeModulePath = sanitizeTestId(testModulePath);
|
||||
const safeTestFunctionName = sanitizeTestId(testFunctionName);
|
||||
|
||||
// Create testId for invocation tracking (matches Python format)
|
||||
const testId = `${testModulePath}:${testClassName}:${testFunctionName}:${lineId}:${LOOP_INDEX}`;
|
||||
const testId = `${safeModulePath}:${testClassName}:${safeTestFunctionName}:${lineId}:${LOOP_INDEX}`;
|
||||
|
||||
// Get invocation index (increments if same testId seen again)
|
||||
const invocationIndex = getInvocationIndex(testId);
|
||||
const invocationId = `${lineId}_${invocationIndex}`;
|
||||
|
||||
// Format stdout tag (matches Python format)
|
||||
const testStdoutTag = `${testModulePath}:${testClassName ? testClassName + '.' : ''}${testFunctionName}:${funcName}:${LOOP_INDEX}:${invocationId}`;
|
||||
// Format stdout tag (matches Python format, uses sanitized names)
|
||||
const testStdoutTag = `${safeModulePath}:${testClassName ? testClassName + '.' : ''}${safeTestFunctionName}:${funcName}:${LOOP_INDEX}:${invocationId}`;
|
||||
|
||||
// Print start tag
|
||||
console.log(`!$######${testStdoutTag}######$!`);
|
||||
|
|
@ -410,6 +479,181 @@ function capturePerf(funcName, lineId, fn, ...args) {
|
|||
return returnValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if performance measurements have stabilized.
|
||||
* Implements the same stability criteria as Python's pytest_plugin.py.
|
||||
*
|
||||
* @param {number[]} runtimes - Array of runtime measurements
|
||||
* @param {number} windowSize - Size of the window to check
|
||||
* @returns {boolean} - True if performance has stabilized
|
||||
*/
|
||||
function checkStability(runtimes, windowSize) {
|
||||
if (runtimes.length < windowSize || windowSize < 3) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Get recent window
|
||||
const window = runtimes.slice(-windowSize);
|
||||
|
||||
// Check center tolerance (all values within ±0.25% of median)
|
||||
const sorted = [...window].sort((a, b) => a - b);
|
||||
const medianIndex = Math.floor(sorted.length / 2);
|
||||
const median = sorted[medianIndex];
|
||||
const centerTolerance = median * STABILITY_CENTER_TOLERANCE;
|
||||
|
||||
const withinCenter = window.every(v => Math.abs(v - median) <= centerTolerance);
|
||||
if (!withinCenter) return false;
|
||||
|
||||
// Check spread tolerance (max-min ≤ 0.25% of min)
|
||||
const minVal = Math.min(...window);
|
||||
const maxVal = Math.max(...window);
|
||||
const spreadTolerance = minVal * STABILITY_SPREAD_TOLERANCE;
|
||||
|
||||
return (maxVal - minVal) <= spreadTolerance;
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture a function call with internal looping for stable performance measurement.
|
||||
*
|
||||
* This function runs the target function multiple times within a single test execution,
|
||||
* similar to Python's pytest_plugin behavior. It provides stable timing by:
|
||||
* - Running multiple iterations to warm up JIT
|
||||
* - Continuing until timing stabilizes or time limit is reached
|
||||
* - Outputting timing data for each iteration
|
||||
*
|
||||
* Environment Variables:
|
||||
* CODEFLASH_MIN_LOOPS - Minimum number of loops (default: 5)
|
||||
* CODEFLASH_MAX_LOOPS - Maximum number of loops (default: 100000)
|
||||
* CODEFLASH_TARGET_DURATION_MS - Target duration in ms (default: 10000)
|
||||
* CODEFLASH_STABILITY_CHECK - Enable stability checking (default: true)
|
||||
*
|
||||
* @param {string} funcName - Name of the function being tested (static)
|
||||
* @param {string} lineId - Line number identifier in test file (static)
|
||||
* @param {Function} fn - The function to call
|
||||
* @param {...any} args - Arguments to pass to the function
|
||||
* @returns {any} - The function's return value from the last iteration
|
||||
* @throws {Error} - Re-throws any error from the function
|
||||
*/
|
||||
function capturePerfLooped(funcName, lineId, fn, ...args) {
|
||||
// Get test context
|
||||
// Use TEST_MODULE env var if set, otherwise derive from test file path
|
||||
let testModulePath;
|
||||
if (TEST_MODULE) {
|
||||
testModulePath = TEST_MODULE;
|
||||
} else if (currentTestPath) {
|
||||
// Get relative path from cwd and convert to module-style path
|
||||
const path = require('path');
|
||||
const relativePath = path.relative(process.cwd(), currentTestPath);
|
||||
// Convert to Python module-style path (e.g., "tests/test_foo.test.js" -> "tests.test_foo.test")
|
||||
testModulePath = relativePath
|
||||
.replace(/\\/g, '/')
|
||||
.replace(/\.js$/, '')
|
||||
.replace(/\.test$/, '.test')
|
||||
.replace(/\//g, '.');
|
||||
} else {
|
||||
testModulePath = currentTestName || 'unknown';
|
||||
}
|
||||
const testClassName = null; // Jest doesn't use classes like Python
|
||||
const testFunctionName = currentTestName || 'unknown';
|
||||
|
||||
// Sanitized versions for stdout tags (avoid regex conflicts)
|
||||
const safeModulePath = sanitizeTestId(testModulePath);
|
||||
const safeTestFunctionName = sanitizeTestId(testFunctionName);
|
||||
|
||||
// Create base testId for invocation tracking
|
||||
const baseTestId = `${safeModulePath}:${testClassName}:${safeTestFunctionName}:${lineId}`;
|
||||
|
||||
// Get invocation index (same call site in loops within test)
|
||||
const invocationIndex = getInvocationIndex(baseTestId + ':base');
|
||||
const invocationId = `${lineId}_${invocationIndex}`;
|
||||
|
||||
// Track runtimes for stability checking
|
||||
const runtimes = [];
|
||||
let returnValue;
|
||||
let error = null;
|
||||
|
||||
const loopStartTime = Date.now();
|
||||
let loopCount = 0;
|
||||
|
||||
while (true) {
|
||||
loopCount++;
|
||||
|
||||
// Create per-loop stdout tag (uses sanitized names)
|
||||
const testStdoutTag = `${safeModulePath}:${testClassName ? testClassName + '.' : ''}${safeTestFunctionName}:${funcName}:${loopCount}:${invocationId}`;
|
||||
|
||||
// Print start tag
|
||||
console.log(`!$######${testStdoutTag}######$!`);
|
||||
|
||||
// Timing with nanosecond precision
|
||||
let durationNs;
|
||||
try {
|
||||
const startTime = getTimeNs();
|
||||
returnValue = fn(...args);
|
||||
const endTime = getTimeNs();
|
||||
durationNs = getDurationNs(startTime, endTime);
|
||||
|
||||
// Handle promises - for async, we can't easily loop internally
|
||||
// Fall back to single execution for async functions
|
||||
if (returnValue instanceof Promise) {
|
||||
return returnValue.then(
|
||||
(resolved) => {
|
||||
const asyncEndTime = getTimeNs();
|
||||
const asyncDurationNs = getDurationNs(startTime, asyncEndTime);
|
||||
console.log(`!######${testStdoutTag}:${asyncDurationNs}######!`);
|
||||
return resolved;
|
||||
},
|
||||
(err) => {
|
||||
const asyncEndTime = getTimeNs();
|
||||
const asyncDurationNs = getDurationNs(startTime, asyncEndTime);
|
||||
console.log(`!######${testStdoutTag}:${asyncDurationNs}######!`);
|
||||
throw err;
|
||||
}
|
||||
);
|
||||
}
|
||||
} catch (e) {
|
||||
durationNs = 0;
|
||||
error = e;
|
||||
// Print end tag even on error
|
||||
console.log(`!######${testStdoutTag}:${durationNs}######!`);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Print end tag with timing
|
||||
console.log(`!######${testStdoutTag}:${durationNs}######!`);
|
||||
|
||||
// Track runtime for stability
|
||||
runtimes.push(durationNs);
|
||||
|
||||
// Check stopping conditions
|
||||
const elapsedMs = Date.now() - loopStartTime;
|
||||
|
||||
// Stop if we've reached max loops
|
||||
if (loopCount >= MAX_LOOPS) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Stop if we've reached min loops AND exceeded time limit
|
||||
if (loopCount >= MIN_LOOPS && elapsedMs >= TARGET_DURATION_MS) {
|
||||
break;
|
||||
}
|
||||
|
||||
// Stability check
|
||||
if (STABILITY_CHECK && loopCount >= MIN_LOOPS) {
|
||||
// Estimate total loops based on current rate
|
||||
const rate = loopCount / elapsedMs;
|
||||
const estimatedTotalLoops = Math.floor(rate * TARGET_DURATION_MS);
|
||||
const windowSize = Math.max(3, Math.floor(STABILITY_WINDOW_SIZE * estimatedTotalLoops));
|
||||
|
||||
if (checkStability(runtimes, windowSize)) {
|
||||
// Performance has stabilized
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return returnValue;
|
||||
}
|
||||
|
||||
/**
|
||||
* Capture multiple invocations for benchmarking.
|
||||
*
|
||||
|
|
@ -490,11 +734,15 @@ function setTestName(name) {
|
|||
// Jest lifecycle hooks - these run automatically when this module is imported
|
||||
if (typeof beforeEach !== 'undefined') {
|
||||
beforeEach(() => {
|
||||
// Get current test name from Jest's expect state
|
||||
// Get current test name and path from Jest's expect state
|
||||
try {
|
||||
currentTestName = expect.getState().currentTestName || 'unknown';
|
||||
const state = expect.getState();
|
||||
currentTestName = state.currentTestName || 'unknown';
|
||||
// testPath is the absolute path to the test file
|
||||
currentTestPath = state.testPath || null;
|
||||
} catch (e) {
|
||||
currentTestName = 'unknown';
|
||||
currentTestPath = null;
|
||||
}
|
||||
// Reset invocation counters for each test
|
||||
resetInvocationCounters();
|
||||
|
|
@ -510,7 +758,8 @@ if (typeof afterAll !== 'undefined') {
|
|||
// Export public API
|
||||
module.exports = {
|
||||
capture, // Behavior verification (writes to SQLite)
|
||||
capturePerf, // Performance benchmarking (prints to stdout only)
|
||||
capturePerf, // Performance benchmarking (prints to stdout only, single run)
|
||||
capturePerfLooped, // Performance benchmarking with internal looping
|
||||
captureMultiple,
|
||||
writeResults,
|
||||
clearResults,
|
||||
|
|
@ -521,10 +770,16 @@ module.exports = {
|
|||
initDatabase,
|
||||
resetInvocationCounters,
|
||||
getInvocationIndex,
|
||||
checkStability,
|
||||
sanitizeTestId, // Sanitize test names for stdout tags
|
||||
// Serializer info
|
||||
getSerializerType: serializer.getSerializerType,
|
||||
// Constants
|
||||
LOOP_INDEX,
|
||||
OUTPUT_FILE,
|
||||
TEST_ITERATION
|
||||
TEST_ITERATION,
|
||||
MIN_LOOPS,
|
||||
MAX_LOOPS,
|
||||
TARGET_DURATION_MS,
|
||||
STABILITY_CHECK
|
||||
};
|
||||
|
|
|
|||
|
|
@ -996,14 +996,20 @@ def parse_test_results(
|
|||
except Exception as e:
|
||||
logger.exception(f"Failed to parse SQLite test results: {e}")
|
||||
|
||||
# Fall back to legacy binary format for Python tests if SQLite doesn't exist
|
||||
if not test_results_data.test_results and test_config.test_framework != "jest":
|
||||
# Also try to read legacy binary format for Python tests
|
||||
# Binary file may contain additional results (e.g., from codeflash_wrap) even if SQLite has data
|
||||
# from @codeflash_capture. We need to merge both sources.
|
||||
if test_config.test_framework != "jest":
|
||||
try:
|
||||
bin_results_file = get_run_tmp_file(Path(f"test_return_values_{optimization_iteration}.bin"))
|
||||
if bin_results_file.exists():
|
||||
test_results_data = parse_test_return_values_bin(
|
||||
bin_test_results = parse_test_return_values_bin(
|
||||
bin_results_file, test_files=test_files, test_config=test_config
|
||||
)
|
||||
# Merge binary results with SQLite results
|
||||
for result in bin_test_results:
|
||||
test_results_data.add(result)
|
||||
logger.debug(f"Merged {len(bin_test_results)} results from binary file")
|
||||
except AttributeError as e:
|
||||
logger.exception(e)
|
||||
|
||||
|
|
|
|||
|
|
@ -18,6 +18,7 @@ from tempfile import TemporaryDirectory
|
|||
import pytest
|
||||
|
||||
from codeflash.discovery.functions_to_optimize import FunctionToOptimize
|
||||
from codeflash.languages.javascript.runtime import get_all_runtime_files
|
||||
from codeflash.models.models import TestFile, TestFiles
|
||||
from codeflash.models.test_type import TestType
|
||||
from codeflash.verification.verification_utils import TestConfig
|
||||
|
|
@ -26,10 +27,16 @@ from codeflash.verification.test_runner import run_jest_behavioral_tests, run_je
|
|||
from codeflash.code_utils.code_utils import get_run_tmp_file
|
||||
|
||||
|
||||
# Path to the JavaScript test project
|
||||
# Path to the JavaScript test project (sample code only)
|
||||
JS_PROJECT_ROOT = Path(__file__).parent.parent / "code_to_optimize_js"
|
||||
|
||||
|
||||
def setup_js_test_environment(project_dir: Path) -> None:
|
||||
"""Copy JavaScript runtime files from codeflash package to project directory."""
|
||||
for runtime_file in get_all_runtime_files():
|
||||
shutil.copy(runtime_file, project_dir / runtime_file.name)
|
||||
|
||||
|
||||
class TestJavaScriptInstrumentation:
|
||||
"""Test JavaScript test instrumentation."""
|
||||
|
||||
|
|
@ -47,7 +54,7 @@ const { reverseString } = require('../string_utils');
|
|||
describe('reverseString', () => {
|
||||
test('should reverse a string', () => {
|
||||
// Behavior mode: capture inputs, outputs, timing to SQLite
|
||||
const result = codeflash.capture('reverseString', reverseString, 'hello');
|
||||
const result = codeflash.capture('reverseString', '8', reverseString, 'hello');
|
||||
// [codeflash-disabled] expect(result).toBe('olleh');
|
||||
});
|
||||
});
|
||||
|
|
@ -61,7 +68,7 @@ const { reverseString } = require('../string_utils');
|
|||
describe('reverseString', () => {
|
||||
test('benchmark reverseString', () => {
|
||||
// Performance mode: only timing to stdout, no SQLite overhead
|
||||
const result = codeflash.capturePerf('reverseString', reverseString, 'hello');
|
||||
const result = codeflash.capturePerf('reverseString', '8', reverseString, 'hello');
|
||||
// [codeflash-disabled] expect(result).toBe('olleh');
|
||||
});
|
||||
});
|
||||
|
|
@ -86,6 +93,9 @@ class TestJavaScriptTestExecution:
|
|||
project_dir = tmp_path / "js_project"
|
||||
shutil.copytree(JS_PROJECT_ROOT, project_dir)
|
||||
|
||||
# Copy runtime JS files from codeflash package
|
||||
setup_js_test_environment(project_dir)
|
||||
|
||||
# Create a simple instrumented test file
|
||||
test_file = project_dir / "tests" / "test_instrumented.test.js"
|
||||
test_file.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
|
@ -96,12 +106,12 @@ const { reverseString } = require('../string_utils');
|
|||
|
||||
describe('reverseString instrumented', () => {
|
||||
test('should reverse hello', () => {
|
||||
const result = codeflash.capture('reverseString', reverseString, 'hello');
|
||||
const result = codeflash.capture('reverseString', '7', reverseString, 'hello');
|
||||
// [codeflash-disabled] expect(result).toBe('olleh');
|
||||
});
|
||||
|
||||
test('should reverse world', () => {
|
||||
const result = codeflash.capture('reverseString', reverseString, 'world');
|
||||
const result = codeflash.capture('reverseString', '12', reverseString, 'world');
|
||||
// [codeflash-disabled] expect(result).toBe('dlrow');
|
||||
});
|
||||
});
|
||||
|
|
@ -345,6 +355,9 @@ class TestEndToEndJavaScript:
|
|||
project_dir = tmp_path / "js_project"
|
||||
shutil.copytree(JS_PROJECT_ROOT, project_dir)
|
||||
|
||||
# Copy runtime JS files from codeflash package
|
||||
setup_js_test_environment(project_dir)
|
||||
|
||||
# Ensure dependencies are installed
|
||||
subprocess.run(
|
||||
["npm", "install"],
|
||||
|
|
@ -367,7 +380,7 @@ const { reverseString } = require('../string_utils');
|
|||
|
||||
describe('reverseString behavior', () => {
|
||||
test('reverses hello', () => {
|
||||
const result = codeflash.capture('reverseString', reverseString, 'hello');
|
||||
const result = codeflash.capture('reverseString', '8', reverseString, 'hello');
|
||||
// [codeflash-disabled] expect(result).toBe('olleh');
|
||||
});
|
||||
});
|
||||
|
|
@ -440,7 +453,7 @@ const { reverseString } = require('../string_utils');
|
|||
|
||||
describe('reverseString benchmark', () => {
|
||||
test('benchmark reverseString', () => {
|
||||
const result = codeflash.capture('reverseString', reverseString, 'hello world');
|
||||
const result = codeflash.capture('reverseString', '8', reverseString, 'hello world');
|
||||
// [codeflash-disabled] expect(result).toBe('dlrow olleh');
|
||||
});
|
||||
});
|
||||
|
|
@ -502,7 +515,7 @@ const { reverseString } = require('../string_utils');
|
|||
describe('reverseString perf only', () => {
|
||||
test('perf test reverseString', () => {
|
||||
// Use capturePerf instead of capture for performance-only
|
||||
const result = codeflash.capturePerf('reverseString', reverseString, 'hello world');
|
||||
const result = codeflash.capturePerf('reverseString', '9', reverseString, 'hello world');
|
||||
// [codeflash-disabled] expect(result).toBe('dlrow olleh');
|
||||
});
|
||||
});
|
||||
|
|
|
|||
1099
tests/test_javascript_instrumentation_comprehensive.py
Normal file
1099
tests/test_javascript_instrumentation_comprehensive.py
Normal file
File diff suppressed because it is too large
Load diff
|
|
@ -59,9 +59,10 @@ def test_mirror_paths_for_worktree_mode(monkeypatch: pytest.MonkeyPatch):
|
|||
assert optimizer.args.project_root == worktree_dir
|
||||
assert optimizer.args.test_project_root == worktree_dir
|
||||
assert optimizer.args.module_root == worktree_dir / "codeflash"
|
||||
assert optimizer.args.tests_root == worktree_dir / "tests"
|
||||
# tests_root is configured as "codeflash" in pyproject.toml
|
||||
assert optimizer.args.tests_root == worktree_dir / "codeflash"
|
||||
assert optimizer.args.file == worktree_dir / "codeflash/optimization/optimizer.py"
|
||||
|
||||
assert optimizer.test_cfg.tests_root == worktree_dir / "tests"
|
||||
assert optimizer.test_cfg.tests_root == worktree_dir / "codeflash"
|
||||
assert optimizer.test_cfg.project_root_path == worktree_dir # same as project_root
|
||||
assert optimizer.test_cfg.tests_project_rootdir == worktree_dir # same as test_project_root
|
||||
|
|
|
|||
Loading…
Reference in a new issue