265 lines
7.3 KiB
Plaintext
265 lines
7.3 KiB
Plaintext
// diff.ce — differential testing: run tests optimized vs unoptimized, compare results
|
|
//
|
|
// Usage:
|
|
// cell diff - diff all test files in current package
|
|
// cell diff suite - diff a specific test file (tests/suite.cm)
|
|
// cell diff tests/foo - diff a specific test file by path
|
|
var shop = use('internal/shop')
|
|
var pkg = use('package')
|
|
var fd = use('fd')
|
|
var time = use('time')
|
|
|
|
var _args = args == null ? [] : args
|
|
|
|
var analyze = use('internal/os').analyze
|
|
var run_ast_fn = use('internal/os').run_ast_fn
|
|
var run_ast_noopt_fn = use('internal/os').run_ast_noopt_fn
|
|
|
|
if (!run_ast_noopt_fn) {
|
|
log.console("error: run_ast_noopt_fn not available (rebuild bootstrap)")
|
|
$stop()
|
|
return
|
|
}
|
|
|
|
// Parse arguments: diff [test_path]
|
|
var target_test = null
|
|
if (length(_args) > 0) {
|
|
target_test = _args[0]
|
|
}
|
|
|
|
function is_valid_package(dir) {
|
|
var _dir = dir == null ? '.' : dir
|
|
return fd.is_file(_dir + '/cell.toml')
|
|
}
|
|
|
|
if (!is_valid_package('.')) {
|
|
log.console('No cell.toml found in current directory')
|
|
$stop()
|
|
return
|
|
}
|
|
|
|
// Collect test files
|
|
function collect_tests(specific_test) {
|
|
var files = pkg.list_files(null)
|
|
var test_files = []
|
|
var i = 0
|
|
var f = null
|
|
var test_name = null
|
|
var match_name = null
|
|
var match_base = null
|
|
for (i = 0; i < length(files); i++) {
|
|
f = files[i]
|
|
if (starts_with(f, "tests/") && ends_with(f, ".cm")) {
|
|
if (specific_test) {
|
|
test_name = text(f, 0, -3)
|
|
match_name = specific_test
|
|
if (!starts_with(match_name, 'tests/')) match_name = 'tests/' + match_name
|
|
match_base = ends_with(match_name, '.cm') ? text(match_name, 0, -3) : match_name
|
|
if (test_name != match_base) continue
|
|
}
|
|
push(test_files, f)
|
|
}
|
|
}
|
|
return test_files
|
|
}
|
|
|
|
// Deep comparison of two values
|
|
function values_equal(a, b) {
|
|
var i = 0
|
|
var ka = null
|
|
var kb = null
|
|
if (a == b) return true
|
|
if (is_null(a) && is_null(b)) return true
|
|
if (is_null(a) || is_null(b)) return false
|
|
if (is_array(a) && is_array(b)) {
|
|
if (length(a) != length(b)) return false
|
|
i = 0
|
|
while (i < length(a)) {
|
|
if (!values_equal(a[i], b[i])) return false
|
|
i = i + 1
|
|
}
|
|
return true
|
|
}
|
|
if (is_object(a) && is_object(b)) {
|
|
ka = array(a)
|
|
kb = array(b)
|
|
if (length(ka) != length(kb)) return false
|
|
i = 0
|
|
while (i < length(ka)) {
|
|
if (!values_equal(a[ka[i]], b[ka[i]])) return false
|
|
i = i + 1
|
|
}
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
function describe(val) {
|
|
if (is_null(val)) return "null"
|
|
if (is_text(val)) return `"${val}"`
|
|
if (is_number(val)) return text(val)
|
|
if (is_logical(val)) return text(val)
|
|
if (is_function(val)) return "<function>"
|
|
if (is_array(val)) return `[array length=${text(length(val))}]`
|
|
if (is_object(val)) return `{record keys=${text(length(array(val)))}}`
|
|
return "<unknown>"
|
|
}
|
|
|
|
// Run a single test file through both paths
|
|
function diff_test_file(file_path) {
|
|
var mod_path = text(file_path, 0, -3)
|
|
var src_path = fd.realpath('.') + '/' + file_path
|
|
var src = null
|
|
var ast = null
|
|
var mod_opt = null
|
|
var mod_noopt = null
|
|
var results = {file: file_path, tests: [], passed: 0, failed: 0, errors: []}
|
|
var use_pkg = fd.realpath('.')
|
|
var opt_error = null
|
|
var noopt_error = null
|
|
var keys = null
|
|
var i = 0
|
|
var k = null
|
|
var opt_result = null
|
|
var noopt_result = null
|
|
var opt_err = null
|
|
var noopt_err = null
|
|
var _run_one_opt = null
|
|
var _run_one_noopt = null
|
|
|
|
// Build env for module loading
|
|
var make_env = function() {
|
|
return stone({
|
|
use: function(path) {
|
|
return shop.use(path, use_pkg)
|
|
}
|
|
})
|
|
}
|
|
|
|
// Read and parse
|
|
var _read = function() {
|
|
src = text(fd.slurp(src_path))
|
|
ast = analyze(src, src_path)
|
|
} disruption {
|
|
push(results.errors, `failed to parse ${file_path}`)
|
|
return results
|
|
}
|
|
_read()
|
|
if (length(results.errors) > 0) return results
|
|
|
|
// Run optimized
|
|
var _run_opt = function() {
|
|
mod_opt = run_ast_fn(mod_path, ast, make_env())
|
|
} disruption {
|
|
opt_error = "disrupted"
|
|
}
|
|
_run_opt()
|
|
|
|
// Run unoptimized
|
|
var _run_noopt = function() {
|
|
mod_noopt = run_ast_noopt_fn(mod_path, ast, make_env())
|
|
} disruption {
|
|
noopt_error = "disrupted"
|
|
}
|
|
_run_noopt()
|
|
|
|
// Compare module-level behavior
|
|
if (opt_error != noopt_error) {
|
|
push(results.errors, `module load mismatch: opt=${opt_error != null ? opt_error : "ok"} noopt=${noopt_error != null ? noopt_error : "ok"}`)
|
|
results.failed = results.failed + 1
|
|
return results
|
|
}
|
|
if (opt_error != null) {
|
|
// Both disrupted during load — that's consistent
|
|
results.passed = results.passed + 1
|
|
push(results.tests, {name: "<module>", status: "passed"})
|
|
return results
|
|
}
|
|
|
|
// If module returns a record of functions, test each one
|
|
if (is_object(mod_opt) && is_object(mod_noopt)) {
|
|
keys = array(mod_opt)
|
|
while (i < length(keys)) {
|
|
k = keys[i]
|
|
if (is_function(mod_opt[k]) && is_function(mod_noopt[k])) {
|
|
opt_result = null
|
|
noopt_result = null
|
|
opt_err = null
|
|
noopt_err = null
|
|
|
|
_run_one_opt = function() {
|
|
opt_result = mod_opt[k]()
|
|
} disruption {
|
|
opt_err = "disrupted"
|
|
}
|
|
_run_one_opt()
|
|
|
|
_run_one_noopt = function() {
|
|
noopt_result = mod_noopt[k]()
|
|
} disruption {
|
|
noopt_err = "disrupted"
|
|
}
|
|
_run_one_noopt()
|
|
|
|
if (opt_err != noopt_err) {
|
|
push(results.tests, {name: k, status: "failed"})
|
|
push(results.errors, `${k}: disruption mismatch opt=${opt_err != null ? opt_err : "ok"} noopt=${noopt_err != null ? noopt_err : "ok"}`)
|
|
results.failed = results.failed + 1
|
|
} else if (!values_equal(opt_result, noopt_result)) {
|
|
push(results.tests, {name: k, status: "failed"})
|
|
push(results.errors, `${k}: result mismatch opt=${describe(opt_result)} noopt=${describe(noopt_result)}`)
|
|
results.failed = results.failed + 1
|
|
} else {
|
|
push(results.tests, {name: k, status: "passed"})
|
|
results.passed = results.passed + 1
|
|
}
|
|
}
|
|
i = i + 1
|
|
}
|
|
} else {
|
|
// Compare direct return values
|
|
if (!values_equal(mod_opt, mod_noopt)) {
|
|
push(results.tests, {name: "<return>", status: "failed"})
|
|
push(results.errors, `return value mismatch: opt=${describe(mod_opt)} noopt=${describe(mod_noopt)}`)
|
|
results.failed = results.failed + 1
|
|
} else {
|
|
push(results.tests, {name: "<return>", status: "passed"})
|
|
results.passed = results.passed + 1
|
|
}
|
|
}
|
|
|
|
return results
|
|
}
|
|
|
|
// Main
|
|
var test_files = collect_tests(target_test)
|
|
log.console(`Differential testing: ${text(length(test_files))} file(s)`)
|
|
|
|
var total_passed = 0
|
|
var total_failed = 0
|
|
var i = 0
|
|
var result = null
|
|
var j = 0
|
|
|
|
while (i < length(test_files)) {
|
|
result = diff_test_file(test_files[i])
|
|
log.console(` ${result.file}: ${text(result.passed)} passed, ${text(result.failed)} failed`)
|
|
j = 0
|
|
while (j < length(result.errors)) {
|
|
log.console(` MISMATCH: ${result.errors[j]}`)
|
|
j = j + 1
|
|
}
|
|
total_passed = total_passed + result.passed
|
|
total_failed = total_failed + result.failed
|
|
i = i + 1
|
|
}
|
|
|
|
log.console(`----------------------------------------`)
|
|
log.console(`Diff: ${text(total_passed)} passed, ${text(total_failed)} failed, ${text(total_passed + total_failed)} total`)
|
|
|
|
if (total_failed > 0) {
|
|
log.console(`DIFFERENTIAL FAILURES DETECTED`)
|
|
}
|
|
|
|
$stop()
|