cell/fuzz.ce

// fuzz.ce — fuzzer driver: generates random programs, runs differential, saves failures
//
// Usage:
//   cell fuzz                - run 100 iterations with a random seed
//   cell fuzz 500            - run 500 iterations with a random seed
//   cell fuzz --seed 42      - run 100 iterations starting at seed 42
//   cell fuzz 500 --seed 42  - run 500 iterations starting at seed 42
//
// Each iteration generates a random self-checking program, compiles it,
// runs it through both optimized and unoptimized paths, and compares results.
// Failures are saved to tests/fuzz_failures/ for reproduction.
var fd = use('fd')
var time = use('time')
var json = use('json')

var os_ref = use('internal/os')
var analyze = os_ref.analyze
var run_ast_fn = os_ref.run_ast_fn
var run_ast_noopt_fn = os_ref.run_ast_noopt_fn

var fuzzgen = use('fuzzgen')

var _args = args == null ? [] : args

// Parse arguments: fuzz [iterations] [--seed N]
var iterations = 100
var start_seed = null
var i = 0
var n = null
var run_err = null
var _run_one = null

while (i < length(_args)) {
  if (_args[i] == '--seed' && i + 1 < length(_args)) {
    start_seed = number(_args[i + 1])
    i = i + 2
  } else {
    n = number(_args[i])
    if (n != null && n > 0) iterations = n
    i = i + 1
  }
}

if (start_seed == null) {
  start_seed = floor(time.number() * 1000) % 1000000
}

if (!run_ast_noopt_fn) {
  log.console("error: run_ast_noopt_fn not available (rebuild bootstrap)")
  $stop()
  return
}

// Ensure failures directory exists
var failures_dir = "tests/fuzz_failures"

function ensure_dir(path) {
  if (fd.is_dir(path)) return
  var parts = array(path, '/')
  var current = ''
  var j = 0
  while (j < length(parts)) {
    if (parts[j] != '') {
      current = current + parts[j] + '/'
      if (!fd.is_dir(current)) {
        fd.mkdir(current)
      }
    }
    j = j + 1
  }
}

// Deep comparison
function values_equal(a, b) {
  var j = 0
  if (a == b) return true
  if (is_null(a) && is_null(b)) return true
  if (is_null(a) || is_null(b)) return false
  if (is_array(a) && is_array(b)) {
    if (length(a) != length(b)) return false
    j = 0
    while (j < length(a)) {
      if (!values_equal(a[j], b[j])) return false
      j = j + 1
    }
    return true
  }
  return false
}

function describe(val) {
  if (is_null(val)) return "null"
  if (is_text(val)) return `"${val}"`
  if (is_number(val)) return text(val)
  if (is_logical(val)) return text(val)
  if (is_function(val)) return "<function>"
  return "<other>"
}

// Run a single fuzz iteration
function run_fuzz(seed_val) {
  var src = fuzzgen.generate(seed_val)
  var name = "fuzz_" + text(seed_val)
  var ast = null
  var mod_opt = null
  var mod_noopt = null
  var opt_err = null
  var noopt_err = null
  var errors = []
  var keys = null
  var k = 0
  var key = null
  var ret = null
  var _run = null
  var run_err = null
  var keys2 = null
  var k2 = 0
  var key2 = null
  var opt_result = null
  var noopt_result = null
  var opt_fn_err = null
  var noopt_fn_err = null
  var _run_opt = null
  var _run_noopt = null

  // Parse
  var _parse = function() {
    ast = analyze(src, name + ".cm")
  } disruption {
    push(errors, "parse error")
  }
  _parse()
  if (length(errors) > 0) return {seed: seed_val, errors: errors, src: src}

  // Run optimized
  var _opt = function() {
    mod_opt = run_ast_fn(name, ast, stone({use: function(p) { return use(p) }}))
  } disruption {
    opt_err = "disrupted"
  }
  _opt()

  // Run unoptimized
  var _noopt = function() {
    mod_noopt = run_ast_noopt_fn(name + "_noopt", ast, stone({use: function(p) { return use(p) }}))
  } disruption {
    noopt_err = "disrupted"
  }
  _noopt()

  // Check module-level behavior
  if (opt_err != noopt_err) {
    push(errors, `module load: opt=${opt_err != null ? opt_err : "ok"} noopt=${noopt_err != null ? noopt_err : "ok"}`)
    return {seed: seed_val, errors: errors, src: src}
  }
  if (opt_err != null) {
    // Both failed to load — consistent
    return {seed: seed_val, errors: errors, src: src}
  }

  // Run self-checks (optimized module)
  if (is_object(mod_opt)) {
    keys = array(mod_opt)
    k = 0
    while (k < length(keys)) {
      key = keys[k]
      if (is_function(mod_opt[key])) {
        ret = null
        run_err = null
        _run = function() {
          ret = mod_opt[key]()
        } disruption {
          run_err = "disrupted"
        }
        _run()

        if (is_text(ret)) {
          push(errors, `self-check ${key}: ${ret}`)
        }
        if (run_err != null) {
          push(errors, `self-check ${key}: unexpected disruption`)
        }
      }
      k = k + 1
    }
  }

  // Differential check on each function
  if (is_object(mod_opt) && is_object(mod_noopt)) {
    keys2 = array(mod_opt)
    k2 = 0
    while (k2 < length(keys2)) {
      key2 = keys2[k2]
      if (is_function(mod_opt[key2]) && is_function(mod_noopt[key2])) {
        opt_result = null
        noopt_result = null
        opt_fn_err = null
        noopt_fn_err = null

        _run_opt = function() {
          opt_result = mod_opt[key2]()
        } disruption {
          opt_fn_err = "disrupted"
        }
        _run_opt()

        _run_noopt = function() {
          noopt_result = mod_noopt[key2]()
        } disruption {
          noopt_fn_err = "disrupted"
        }
        _run_noopt()

        if (opt_fn_err != noopt_fn_err) {
          push(errors, `diff ${key2}: opt=${opt_fn_err != null ? opt_fn_err : "ok"} noopt=${noopt_fn_err != null ? noopt_fn_err : "ok"}`)
        } else if (!values_equal(opt_result, noopt_result)) {
          push(errors, `diff ${key2}: opt=${describe(opt_result)} noopt=${describe(noopt_result)}`)
        }
      }
      k2 = k2 + 1
    }
  }

  return {seed: seed_val, errors: errors, src: src}
}

// Main loop
log.console(`Fuzzing: ${text(iterations)} iterations, starting seed=${text(start_seed)}`)
var total_pass = 0
var total_fail = 0
var result = null
var j = 0
var current_seed = 0
var fail_path = null

i = 0
while (i < iterations) {
  current_seed = start_seed + i
  run_err = null
  _run_one = function() {
    result = run_fuzz(current_seed)
  } disruption {
    run_err = "generator crashed"
  }
  _run_one()

  if (run_err != null) {
    result = {seed: current_seed, errors: [run_err], src: "// generator crashed"}
  }

  if (length(result.errors) > 0) {
    total_fail = total_fail + 1
    log.console(`  FAIL seed=${text(current_seed)}: ${result.errors[0]}`)

    // Save failure source for reproduction
    ensure_dir(failures_dir)
    fail_path = failures_dir + "/seed_" + text(current_seed) + ".cm"
    fd.slurpwrite(fail_path, stone(blob(result.src)))
    log.console(`    saved to ${fail_path}`)
  } else {
    total_pass = total_pass + 1
  }

  // Progress report every 100 iterations
  if ((i + 1) % 100 == 0) {
    log.console(`  progress: ${text(i + 1)}/${text(iterations)} (${text(total_pass)} passed, ${text(total_fail)} failed)`)
  }

  i = i + 1
}

log.console(`----------------------------------------`)
log.console(`Fuzz: ${text(total_pass)} passed, ${text(total_fail)} failed, ${text(iterations)} total`)
if (total_fail > 0) {
  log.console(`Failures saved to ${failures_dir}/`)
}

$stop()