cell/bench.ce

var shop = use('internal/shop')
var pkg = use('package')
var fd = use('fd')
var time = use('time')
var json = use('json')
var utf8 = use('utf8')
var os = use('os')
var testlib = use('internal/testlib')
var math = use('math/radians')

if (!args) args = []

var target_pkg = null // null = current package
var target_bench = null // null = all benchmarks, otherwise specific bench file
var all_pkgs = false

// Benchmark configuration
def WARMUP_ITERATIONS = 5
def MIN_ITERATIONS = 10
def MAX_ITERATIONS = 1000
def TARGET_DURATION_NS = 1000000000 // 1 second target per benchmark

// Statistical functions
function median(arr) {
  if (arr.length == 0) return 0
  var sorted = arr.slice().sort(function(a, b) { return a - b })
  var mid = number.floor(arr.length / 2)
  if (arr.length % 2 == 0) {
    return (sorted[mid - 1] + sorted[mid]) / 2
  }
  return sorted[mid]
}

function mean(arr) {
  if (arr.length == 0) return 0
  var sum = 0
  for (var i = 0; i < arr.length; i++) {
    sum += arr[i]
  }
  return sum / arr.length
}

function stddev(arr, mean_val) {
  if (arr.length < 2) return 0
  var sum_sq_diff = 0
  for (var i = 0; i < arr.length; i++) {
    var diff = arr[i] - mean_val
    sum_sq_diff += diff * diff
  }
  return math.sqrt(sum_sq_diff / (arr.length - 1))
}

function percentile(arr, p) {
  if (arr.length == 0) return 0
  var sorted = arr.slice().sort(function(a, b) { return a - b })
  var idx = number.floor(arr.length * p / 100)
  if (idx >= arr.length) idx = arr.length - 1
  return sorted[idx]
}

function min_val(arr) {
  if (arr.length == 0) return 0
  var m = arr[0]
  for (var i = 1; i < arr.length; i++) {
    if (arr[i] < m) m = arr[i]
  }
  return m
}

function max_val(arr) {
  if (arr.length == 0) return 0
  var m = arr[0]
  for (var i = 1; i < arr.length; i++) {
    if (arr[i] > m) m = arr[i]
  }
  return m
}

// Parse arguments similar to test.ce
function parse_args() {
  if (args.length == 0) {
    if (!testlib.is_valid_package('.')) {
      log.console('No cell.toml found in current directory')
      return false
    }
    target_pkg = null
    return true
  }

  if (args[0] == 'all') {
    if (!testlib.is_valid_package('.')) {
      log.console('No cell.toml found in current directory')
      return false
    }
    target_pkg = null
    return true
  }

  if (args[0] == 'package') {
    if (args.length < 2) {
      log.console('Usage: cell bench package <name> [bench]')
      log.console('       cell bench package all')
      return false
    }

    if (args[1] == 'all') {
      all_pkgs = true
      log.console('Benchmarking all packages...')
      return true
    }

    var name = args[1]
    var lock = shop.load_lock()
    if (lock[name]) {
      target_pkg = name
    } else if (name.startsWith('/') && testlib.is_valid_package(name)) {
      target_pkg = name
    } else {
      if (testlib.is_valid_package('.')) {
        var resolved = pkg.alias_to_package(null, name)
        if (resolved) {
          target_pkg = resolved
        } else {
          log.console(`Package not found: ${name}`)
          return false
        }
      } else {
        log.console(`Package not found: ${name}`)
        return false
      }
    }

    if (args.length >= 3) {
      target_bench = args[2]
    }

    log.console(`Benchmarking package: ${target_pkg}`)
    return true
  }

  // cell bench benches/suite or cell bench <path>
  var bench_path = args[0]

  // Normalize path - add benches/ prefix if not present
  if (!bench_path.startsWith('benches/') && !bench_path.startsWith('/')) {
    if (!fd.is_file(bench_path + '.cm') && !fd.is_file(bench_path)) {
      if (fd.is_file('benches/' + bench_path + '.cm') || fd.is_file('benches/' + bench_path)) {
        bench_path = 'benches/' + bench_path
      }
    }
  }

  target_bench = bench_path
  target_pkg = null

  if (!testlib.is_valid_package('.')) {
    log.console('No cell.toml found in current directory')
    return false
  }

  return true
}

if (!parse_args()) {
  $stop()
  return
}

// Collect benchmark files from a package
function collect_benches(package_name, specific_bench) {
  var prefix = testlib.get_pkg_dir(package_name)
  var benches_dir = prefix + '/benches'

  if (!fd.is_dir(benches_dir)) return []

  var files = pkg.list_files(package_name)
  var bench_files = []
  for (var i = 0; i < files.length; i++) {
    var f = files[i]
    if (f.startsWith("benches/") && f.endsWith(".cm")) {
      if (specific_bench) {
        var bench_name = f.substring(0, f.length - 3)
        var match_name = specific_bench
        if (!match_name.startsWith('benches/')) match_name = 'benches/' + match_name
        var match_base = match_name.endsWith('.cm') ? match_name.substring(0, match_name.length - 3) : match_name
        if (bench_name != match_base) continue
      }
      bench_files.push(f)
    }
  }
  return bench_files
}

// Run a single benchmark function
function run_single_bench(bench_fn, bench_name) {
  var timings = []

  // Warmup phase
  for (var i = 0; i < WARMUP_ITERATIONS; i++) {
    bench_fn()
  }

  // Determine how many iterations to run
  var test_start = os.now()
  bench_fn()
  var test_duration = os.now() - test_start

  var iterations = MIN_ITERATIONS
  if (test_duration > 0) {
    iterations = number.floor(TARGET_DURATION_NS / test_duration)
    if (iterations < MIN_ITERATIONS) iterations = MIN_ITERATIONS
    if (iterations > MAX_ITERATIONS) iterations = MAX_ITERATIONS
  }

  // Measurement phase
  for (var i = 0; i < iterations; i++) {
    var start = os.now()
    bench_fn()
    var duration = os.now() - start
    timings.push(duration)
  }

  // Calculate statistics
  var mean_ns = mean(timings)
  var median_ns = median(timings)
  var min_ns = min_val(timings)
  var max_ns = max_val(timings)
  var stddev_ns = stddev(timings, mean_ns)
  var p95_ns = percentile(timings, 95)
  var p99_ns = percentile(timings, 99)

  // Calculate ops/s from median
  var ops_per_sec = 0
  if (median_ns > 0) {
    ops_per_sec = number.floor(1000000000 / median_ns)
  }

  return {
    name: bench_name,
    iterations: iterations,
    mean_ns: number.round(mean_ns),
    median_ns: number.round(median_ns),
    min_ns: number.round(min_ns),
    max_ns: number.round(max_ns),
    stddev_ns: number.round(stddev_ns),
    p95_ns: number.round(p95_ns),
    p99_ns: number.round(p99_ns),
    ops_per_sec: ops_per_sec
  }
}

// Format nanoseconds for display
function format_ns(ns) {
  if (ns < 1000) return `${ns}ns`
  if (ns < 1000000) return `${number.round(ns / 1000 * 100) / 100}µs`
  if (ns < 1000000000) return `${number.round(ns / 1000000 * 100) / 100}ms`
  return `${number.round(ns / 1000000000 * 100) / 100}s`
}

// Format ops/sec for display
function format_ops(ops) {
  if (ops < 1000) return `${ops} ops/s`
  if (ops < 1000000) return `${number.round(ops / 1000 * 100) / 100}K ops/s`
  if (ops < 1000000000) return `${number.round(ops / 1000000 * 100) / 100}M ops/s`
  return `${number.round(ops / 1000000000 * 100) / 100}G ops/s`
}

// Run benchmarks for a package
function run_benchmarks(package_name, specific_bench) {
  var bench_files = collect_benches(package_name, specific_bench)

  var pkg_result = {
    package: package_name || "local",
    files: [],
    total: 0
  }

  if (bench_files.length == 0) return pkg_result

  if (package_name) log.console(`Running benchmarks for ${package_name}`)
  else log.console(`Running benchmarks for local package`)

  for (var i = 0; i < bench_files.length; i++) {
    var f = bench_files[i]
    var mod_path = f.substring(0, f.length - 3)

    var file_result = {
      name: f,
      benchmarks: []
    }

    try {
      var bench_mod
      var use_pkg = package_name ? package_name : fd.realpath('.')
      bench_mod = shop.use(mod_path, use_pkg)

      var benches = []
      if (typeof bench_mod == 'function') {
        benches.push({name: 'main', fn: bench_mod})
      } else if (typeof bench_mod == 'object') {
        for (var k in bench_mod) {
          if (typeof bench_mod[k] == 'function') {
            benches.push({name: k, fn: bench_mod[k]})
          }
        }
      }

      if (benches.length > 0) {
        log.console(`  ${f}`)
        for (var j = 0; j < benches.length; j++) {
          var b = benches[j]
          try {
            var result = run_single_bench(b.fn, b.name)
            result.package = pkg_result.package
            file_result.benchmarks.push(result)
            pkg_result.total++

            log.console(`    ${result.name}`)
            log.console(`      ${format_ns(result.median_ns)}/op  ${format_ops(result.ops_per_sec)}`)
            log.console(`      min: ${format_ns(result.min_ns)}  max: ${format_ns(result.max_ns)}  stddev: ${format_ns(result.stddev_ns)}`)
          } catch (e) {
            log.console(`    ERROR ${b.name}: ${e}`)
            log.error(e)
            var error_result = {
              package: pkg_result.package,
              name: b.name,
              error: e.toString()
            }
            file_result.benchmarks.push(error_result)
            pkg_result.total++
          }
        }
      }
    } catch (e) {
      log.console(`  Error loading ${f}: ${e}`)
      var error_result = {
        package: pkg_result.package,
        name: "load_module",
        error: `Error loading module: ${e}`
      }
      file_result.benchmarks.push(error_result)
      pkg_result.total++
    }

    if (file_result.benchmarks.length > 0) {
      pkg_result.files.push(file_result)
    }
  }

  return pkg_result
}

// Run all benchmarks
var all_results = []

if (all_pkgs) {
  if (testlib.is_valid_package('.')) {
    all_results.push(run_benchmarks(null, null))
  }

  var packages = shop.list_packages()
  for (var i = 0; i < packages.length; i++) {
    all_results.push(run_benchmarks(packages[i], null))
  }
} else {
  all_results.push(run_benchmarks(target_pkg, target_bench))
}

// Calculate totals
var total_benches = 0
for (var i = 0; i < all_results.length; i++) {
  total_benches += all_results[i].total
}

log.console(`----------------------------------------`)
log.console(`Benchmarks: ${total_benches} total`)

// Generate reports
function generate_reports() {
  var timestamp = number.floor(time.number()).toString()
  var report_dir = shop.get_reports_dir() + '/bench_' + timestamp
  testlib.ensure_dir(report_dir)

  var txt_report = `BENCHMARK REPORT
Date: ${time.text(time.number())}
Total benchmarks: ${total_benches}

=== SUMMARY ===
`
  for (var i = 0; i < all_results.length; i++) {
    var pkg_res = all_results[i]
    if (pkg_res.total == 0) continue
    txt_report += `Package: ${pkg_res.package}\n`
    for (var j = 0; j < pkg_res.files.length; j++) {
      var f = pkg_res.files[j]
      txt_report += `  ${f.name}\n`
      for (var k = 0; k < f.benchmarks.length; k++) {
        var b = f.benchmarks[k]
        if (b.error) {
          txt_report += `    ERROR ${b.name}: ${b.error}\n`
        } else {
          txt_report += `    ${b.name}: ${format_ns(b.median_ns)}/op (${format_ops(b.ops_per_sec)})\n`
        }
      }
    }
  }

  txt_report += `\n=== DETAILED RESULTS ===\n`
  for (var i = 0; i < all_results.length; i++) {
    var pkg_res = all_results[i]
    if (pkg_res.total == 0) continue

    for (var j = 0; j < pkg_res.files.length; j++) {
      var f = pkg_res.files[j]
      for (var k = 0; k < f.benchmarks.length; k++) {
        var b = f.benchmarks[k]
        if (b.error) continue

        txt_report += `\n${pkg_res.package}::${b.name}\n`
        txt_report += `  iterations: ${b.iterations}\n`
        txt_report += `  median: ${format_ns(b.median_ns)}/op\n`
        txt_report += `  mean: ${format_ns(b.mean_ns)}/op\n`
        txt_report += `  min: ${format_ns(b.min_ns)}\n`
        txt_report += `  max: ${format_ns(b.max_ns)}\n`
        txt_report += `  stddev: ${format_ns(b.stddev_ns)}\n`
        txt_report += `  p95: ${format_ns(b.p95_ns)}\n`
        txt_report += `  p99: ${format_ns(b.p99_ns)}\n`
        txt_report += `  ops/s: ${format_ops(b.ops_per_sec)}\n`
      }
    }
  }

  testlib.ensure_dir(report_dir)
  fd.slurpwrite(`${report_dir}/bench.txt`, utf8.encode(txt_report))
  log.console(`Report written to ${report_dir}/bench.txt`)

  // Generate JSON per package
  for (var i = 0; i < all_results.length; i++) {
    var pkg_res = all_results[i]
    if (pkg_res.total == 0) continue

    var pkg_benches = []
    for (var j = 0; j < pkg_res.files.length; j++) {
      var f = pkg_res.files[j]
      for (var k = 0; k < f.benchmarks.length; k++) {
        pkg_benches.push(f.benchmarks[k])
      }
    }

    var json_path = `${report_dir}/${pkg_res.package.replace(/\//g, '_')}.json`
    fd.slurpwrite(json_path, utf8.encode(json.encode(pkg_benches)))
  }
}

generate_reports()
$stop()