comprehensive testing for regression analysis
This commit is contained in:
278
fuzz.ce
Normal file
278
fuzz.ce
Normal file
@@ -0,0 +1,278 @@
|
||||
// fuzz.ce — fuzzer driver: generates random programs, runs differential, saves failures
|
||||
//
|
||||
// Usage:
|
||||
// cell fuzz - run 100 iterations with a random seed
|
||||
// cell fuzz 500 - run 500 iterations with a random seed
|
||||
// cell fuzz --seed 42 - run 100 iterations starting at seed 42
|
||||
// cell fuzz 500 --seed 42 - run 500 iterations starting at seed 42
|
||||
//
|
||||
// Each iteration generates a random self-checking program, compiles it,
|
||||
// runs it through both optimized and unoptimized paths, and compares results.
|
||||
// Failures are saved to tests/fuzz_failures/ for reproduction.
|
||||
var fd = use('fd')
|
||||
var time = use('time')
|
||||
var json = use('json')
|
||||
|
||||
var os_ref = use('os')
|
||||
var analyze = os_ref.analyze
|
||||
var run_ast_fn = os_ref.run_ast_fn
|
||||
var run_ast_noopt_fn = os_ref.run_ast_noopt_fn
|
||||
|
||||
var fuzzgen = use('fuzzgen')
|
||||
|
||||
var _args = args == null ? [] : args
|
||||
|
||||
// Parse arguments: fuzz [iterations] [--seed N]
|
||||
var iterations = 100
|
||||
var start_seed = null
|
||||
var i = 0
|
||||
var n = null
|
||||
var run_err = null
|
||||
var _run_one = null
|
||||
|
||||
while (i < length(_args)) {
|
||||
if (_args[i] == '--seed' && i + 1 < length(_args)) {
|
||||
start_seed = number(_args[i + 1])
|
||||
i = i + 2
|
||||
} else {
|
||||
n = number(_args[i])
|
||||
if (n != null && n > 0) iterations = n
|
||||
i = i + 1
|
||||
}
|
||||
}
|
||||
|
||||
if (start_seed == null) {
|
||||
start_seed = floor(time.number() * 1000) % 1000000
|
||||
}
|
||||
|
||||
if (!run_ast_noopt_fn) {
|
||||
log.console("error: run_ast_noopt_fn not available (rebuild bootstrap)")
|
||||
$stop()
|
||||
return
|
||||
}
|
||||
|
||||
// Ensure failures directory exists
|
||||
var failures_dir = "tests/fuzz_failures"
|
||||
|
||||
function ensure_dir(path) {
|
||||
if (fd.is_dir(path)) return
|
||||
var parts = array(path, '/')
|
||||
var current = ''
|
||||
var j = 0
|
||||
while (j < length(parts)) {
|
||||
if (parts[j] != '') {
|
||||
current = current + parts[j] + '/'
|
||||
if (!fd.is_dir(current)) {
|
||||
fd.mkdir(current)
|
||||
}
|
||||
}
|
||||
j = j + 1
|
||||
}
|
||||
}
|
||||
|
||||
// Deep comparison
|
||||
function values_equal(a, b) {
|
||||
var j = 0
|
||||
if (a == b) return true
|
||||
if (is_null(a) && is_null(b)) return true
|
||||
if (is_null(a) || is_null(b)) return false
|
||||
if (is_array(a) && is_array(b)) {
|
||||
if (length(a) != length(b)) return false
|
||||
j = 0
|
||||
while (j < length(a)) {
|
||||
if (!values_equal(a[j], b[j])) return false
|
||||
j = j + 1
|
||||
}
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
function describe(val) {
|
||||
if (is_null(val)) return "null"
|
||||
if (is_text(val)) return `"${val}"`
|
||||
if (is_number(val)) return text(val)
|
||||
if (is_logical(val)) return text(val)
|
||||
if (is_function(val)) return "<function>"
|
||||
return "<other>"
|
||||
}
|
||||
|
||||
// Run a single fuzz iteration
|
||||
function run_fuzz(seed_val) {
|
||||
var src = fuzzgen.generate(seed_val)
|
||||
var name = "fuzz_" + text(seed_val)
|
||||
var ast = null
|
||||
var mod_opt = null
|
||||
var mod_noopt = null
|
||||
var opt_err = null
|
||||
var noopt_err = null
|
||||
var errors = []
|
||||
var keys = null
|
||||
var k = 0
|
||||
var key = null
|
||||
var ret = null
|
||||
var _run = null
|
||||
var run_err = null
|
||||
var keys2 = null
|
||||
var k2 = 0
|
||||
var key2 = null
|
||||
var opt_result = null
|
||||
var noopt_result = null
|
||||
var opt_fn_err = null
|
||||
var noopt_fn_err = null
|
||||
var _run_opt = null
|
||||
var _run_noopt = null
|
||||
|
||||
// Parse
|
||||
var _parse = function() {
|
||||
ast = analyze(src, name + ".cm")
|
||||
} disruption {
|
||||
push(errors, "parse error")
|
||||
}
|
||||
_parse()
|
||||
if (length(errors) > 0) return {seed: seed_val, errors: errors, src: src}
|
||||
|
||||
// Run optimized
|
||||
var _opt = function() {
|
||||
mod_opt = run_ast_fn(name, ast, {use: function(p) { return use(p) }})
|
||||
} disruption {
|
||||
opt_err = "disrupted"
|
||||
}
|
||||
_opt()
|
||||
|
||||
// Run unoptimized
|
||||
var _noopt = function() {
|
||||
mod_noopt = run_ast_noopt_fn(name + "_noopt", ast, {use: function(p) { return use(p) }})
|
||||
} disruption {
|
||||
noopt_err = "disrupted"
|
||||
}
|
||||
_noopt()
|
||||
|
||||
// Check module-level behavior
|
||||
if (opt_err != noopt_err) {
|
||||
push(errors, `module load: opt=${opt_err != null ? opt_err : "ok"} noopt=${noopt_err != null ? noopt_err : "ok"}`)
|
||||
return {seed: seed_val, errors: errors, src: src}
|
||||
}
|
||||
if (opt_err != null) {
|
||||
// Both failed to load — consistent
|
||||
return {seed: seed_val, errors: errors, src: src}
|
||||
}
|
||||
|
||||
// Run self-checks (optimized module)
|
||||
if (is_object(mod_opt)) {
|
||||
keys = array(mod_opt)
|
||||
k = 0
|
||||
while (k < length(keys)) {
|
||||
key = keys[k]
|
||||
if (is_function(mod_opt[key])) {
|
||||
ret = null
|
||||
run_err = null
|
||||
_run = function() {
|
||||
ret = mod_opt[key]()
|
||||
} disruption {
|
||||
run_err = "disrupted"
|
||||
}
|
||||
_run()
|
||||
|
||||
if (is_text(ret)) {
|
||||
push(errors, `self-check ${key}: ${ret}`)
|
||||
}
|
||||
if (run_err != null) {
|
||||
push(errors, `self-check ${key}: unexpected disruption`)
|
||||
}
|
||||
}
|
||||
k = k + 1
|
||||
}
|
||||
}
|
||||
|
||||
// Differential check on each function
|
||||
if (is_object(mod_opt) && is_object(mod_noopt)) {
|
||||
keys2 = array(mod_opt)
|
||||
k2 = 0
|
||||
while (k2 < length(keys2)) {
|
||||
key2 = keys2[k2]
|
||||
if (is_function(mod_opt[key2]) && is_function(mod_noopt[key2])) {
|
||||
opt_result = null
|
||||
noopt_result = null
|
||||
opt_fn_err = null
|
||||
noopt_fn_err = null
|
||||
|
||||
_run_opt = function() {
|
||||
opt_result = mod_opt[key2]()
|
||||
} disruption {
|
||||
opt_fn_err = "disrupted"
|
||||
}
|
||||
_run_opt()
|
||||
|
||||
_run_noopt = function() {
|
||||
noopt_result = mod_noopt[key2]()
|
||||
} disruption {
|
||||
noopt_fn_err = "disrupted"
|
||||
}
|
||||
_run_noopt()
|
||||
|
||||
if (opt_fn_err != noopt_fn_err) {
|
||||
push(errors, `diff ${key2}: opt=${opt_fn_err != null ? opt_fn_err : "ok"} noopt=${noopt_fn_err != null ? noopt_fn_err : "ok"}`)
|
||||
} else if (!values_equal(opt_result, noopt_result)) {
|
||||
push(errors, `diff ${key2}: opt=${describe(opt_result)} noopt=${describe(noopt_result)}`)
|
||||
}
|
||||
}
|
||||
k2 = k2 + 1
|
||||
}
|
||||
}
|
||||
|
||||
return {seed: seed_val, errors: errors, src: src}
|
||||
}
|
||||
|
||||
// Main loop
|
||||
log.console(`Fuzzing: ${text(iterations)} iterations, starting seed=${text(start_seed)}`)
|
||||
var total_pass = 0
|
||||
var total_fail = 0
|
||||
var result = null
|
||||
var j = 0
|
||||
var current_seed = 0
|
||||
var fail_path = null
|
||||
|
||||
i = 0
|
||||
while (i < iterations) {
|
||||
current_seed = start_seed + i
|
||||
run_err = null
|
||||
_run_one = function() {
|
||||
result = run_fuzz(current_seed)
|
||||
} disruption {
|
||||
run_err = "generator crashed"
|
||||
}
|
||||
_run_one()
|
||||
|
||||
if (run_err != null) {
|
||||
result = {seed: current_seed, errors: [run_err], src: "// generator crashed"}
|
||||
}
|
||||
|
||||
if (length(result.errors) > 0) {
|
||||
total_fail = total_fail + 1
|
||||
log.console(` FAIL seed=${text(current_seed)}: ${result.errors[0]}`)
|
||||
|
||||
// Save failure source for reproduction
|
||||
ensure_dir(failures_dir)
|
||||
fail_path = failures_dir + "/seed_" + text(current_seed) + ".cm"
|
||||
fd.slurpwrite(fail_path, stone(blob(result.src)))
|
||||
log.console(` saved to ${fail_path}`)
|
||||
} else {
|
||||
total_pass = total_pass + 1
|
||||
}
|
||||
|
||||
// Progress report every 100 iterations
|
||||
if ((i + 1) % 100 == 0) {
|
||||
log.console(` progress: ${text(i + 1)}/${text(iterations)} (${text(total_pass)} passed, ${text(total_fail)} failed)`)
|
||||
}
|
||||
|
||||
i = i + 1
|
||||
}
|
||||
|
||||
log.console(`----------------------------------------`)
|
||||
log.console(`Fuzz: ${text(total_pass)} passed, ${text(total_fail)} failed, ${text(iterations)} total`)
|
||||
if (total_fail > 0) {
|
||||
log.console(`Failures saved to ${failures_dir}/`)
|
||||
}
|
||||
|
||||
$stop()
|
||||
Reference in New Issue
Block a user