// diff.ce — differential testing: run tests optimized vs unoptimized, compare results // // Usage: // cell diff - diff all test files in current package // cell diff suite - diff a specific test file (tests/suite.cm) // cell diff tests/foo - diff a specific test file by path var shop = use('internal/shop') var pkg = use('package') var fd = use('fd') var time = use('time') var _args = args == null ? [] : args var analyze = use('os').analyze var run_ast_fn = use('os').run_ast_fn var run_ast_noopt_fn = use('os').run_ast_noopt_fn if (!run_ast_noopt_fn) { log.console("error: run_ast_noopt_fn not available (rebuild bootstrap)") $stop() return } // Parse arguments: diff [test_path] var target_test = null if (length(_args) > 0) { target_test = _args[0] } function is_valid_package(dir) { var _dir = dir == null ? '.' : dir return fd.is_file(_dir + '/cell.toml') } if (!is_valid_package('.')) { log.console('No cell.toml found in current directory') $stop() return } // Collect test files function collect_tests(specific_test) { var files = pkg.list_files(null) var test_files = [] var i = 0 var f = null var test_name = null var match_name = null var match_base = null for (i = 0; i < length(files); i++) { f = files[i] if (starts_with(f, "tests/") && ends_with(f, ".cm")) { if (specific_test) { test_name = text(f, 0, -3) match_name = specific_test if (!starts_with(match_name, 'tests/')) match_name = 'tests/' + match_name match_base = ends_with(match_name, '.cm') ? text(match_name, 0, -3) : match_name if (test_name != match_base) continue } push(test_files, f) } } return test_files } // Deep comparison of two values function values_equal(a, b) { var i = 0 var ka = null var kb = null if (a == b) return true if (is_null(a) && is_null(b)) return true if (is_null(a) || is_null(b)) return false if (is_array(a) && is_array(b)) { if (length(a) != length(b)) return false i = 0 while (i < length(a)) { if (!values_equal(a[i], b[i])) return false i = i + 1 } return true } if (is_object(a) && is_object(b)) { ka = array(a) kb = array(b) if (length(ka) != length(kb)) return false i = 0 while (i < length(ka)) { if (!values_equal(a[ka[i]], b[ka[i]])) return false i = i + 1 } return true } return false } function describe(val) { if (is_null(val)) return "null" if (is_text(val)) return `"${val}"` if (is_number(val)) return text(val) if (is_logical(val)) return text(val) if (is_function(val)) return "" if (is_array(val)) return `[array length=${text(length(val))}]` if (is_object(val)) return `{record keys=${text(length(array(val)))}}` return "" } // Run a single test file through both paths function diff_test_file(file_path) { var mod_path = text(file_path, 0, -3) var src_path = fd.realpath('.') + '/' + file_path var src = null var ast = null var mod_opt = null var mod_noopt = null var results = {file: file_path, tests: [], passed: 0, failed: 0, errors: []} var use_pkg = fd.realpath('.') var opt_error = null var noopt_error = null var keys = null var i = 0 var k = null var opt_result = null var noopt_result = null var opt_err = null var noopt_err = null var _run_one_opt = null var _run_one_noopt = null // Build env for module loading var make_env = function() { return { use: function(path) { return shop.use(path, use_pkg) } } } // Read and parse var _read = function() { src = text(fd.slurp(src_path)) ast = analyze(src, src_path) } disruption { push(results.errors, `failed to parse ${file_path}`) return results } _read() if (length(results.errors) > 0) return results // Run optimized var _run_opt = function() { mod_opt = run_ast_fn(mod_path, ast, make_env()) } disruption { opt_error = "disrupted" } _run_opt() // Run unoptimized var _run_noopt = function() { mod_noopt = run_ast_noopt_fn(mod_path, ast, make_env()) } disruption { noopt_error = "disrupted" } _run_noopt() // Compare module-level behavior if (opt_error != noopt_error) { push(results.errors, `module load mismatch: opt=${opt_error != null ? opt_error : "ok"} noopt=${noopt_error != null ? noopt_error : "ok"}`) results.failed = results.failed + 1 return results } if (opt_error != null) { // Both disrupted during load — that's consistent results.passed = results.passed + 1 push(results.tests, {name: "", status: "passed"}) return results } // If module returns a record of functions, test each one if (is_object(mod_opt) && is_object(mod_noopt)) { keys = array(mod_opt) while (i < length(keys)) { k = keys[i] if (is_function(mod_opt[k]) && is_function(mod_noopt[k])) { opt_result = null noopt_result = null opt_err = null noopt_err = null _run_one_opt = function() { opt_result = mod_opt[k]() } disruption { opt_err = "disrupted" } _run_one_opt() _run_one_noopt = function() { noopt_result = mod_noopt[k]() } disruption { noopt_err = "disrupted" } _run_one_noopt() if (opt_err != noopt_err) { push(results.tests, {name: k, status: "failed"}) push(results.errors, `${k}: disruption mismatch opt=${opt_err != null ? opt_err : "ok"} noopt=${noopt_err != null ? noopt_err : "ok"}`) results.failed = results.failed + 1 } else if (!values_equal(opt_result, noopt_result)) { push(results.tests, {name: k, status: "failed"}) push(results.errors, `${k}: result mismatch opt=${describe(opt_result)} noopt=${describe(noopt_result)}`) results.failed = results.failed + 1 } else { push(results.tests, {name: k, status: "passed"}) results.passed = results.passed + 1 } } i = i + 1 } } else { // Compare direct return values if (!values_equal(mod_opt, mod_noopt)) { push(results.tests, {name: "", status: "failed"}) push(results.errors, `return value mismatch: opt=${describe(mod_opt)} noopt=${describe(mod_noopt)}`) results.failed = results.failed + 1 } else { push(results.tests, {name: "", status: "passed"}) results.passed = results.passed + 1 } } return results } // Main var test_files = collect_tests(target_test) log.console(`Differential testing: ${text(length(test_files))} file(s)`) var total_passed = 0 var total_failed = 0 var i = 0 var result = null var j = 0 while (i < length(test_files)) { result = diff_test_file(test_files[i]) log.console(` ${result.file}: ${text(result.passed)} passed, ${text(result.failed)} failed`) j = 0 while (j < length(result.errors)) { log.console(` MISMATCH: ${result.errors[j]}`) j = j + 1 } total_passed = total_passed + result.passed total_failed = total_failed + result.failed i = i + 1 } log.console(`----------------------------------------`) log.console(`Diff: ${text(total_passed)} passed, ${text(total_failed)} failed, ${text(total_passed + total_failed)} total`) if (total_failed > 0) { log.console(`DIFFERENTIAL FAILURES DETECTED`) } $stop()