john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

benchmarks

Browse Source
This commit is contained in:
John Alanbrook
2026-02-12 18:41:15 -06:00
parent 4aedb8b0c5
commit c1e101b24f
17 changed files with 2765 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

167
bench.ce
View File

@@ -8,7 +8,7 @@ var os = use('os')
var testlib = use('internal/testlib')
var math = use('math/radians')
if (!args) args = []
var _args = args == null ? [] : args
var target_pkg = null // null = current package
var target_bench = null // null = all benchmarks, otherwise specific bench file
@@ -55,14 +55,19 @@ function stddev(arr, mean_val) {
function percentile(arr, p) {
if (length(arr) == 0) return 0
var sorted = sort(arr)
var idx = floor(arr) * p / 100
var idx = floor(length(arr) * p / 100)
if (idx >= length(arr)) idx = length(arr) - 1
return sorted[idx]
}
// Parse arguments similar to test.ce
function parse_args() {
if (length(args) == 0) {
var name = null
var lock = null
var resolved = null
var bench_path = null
if (length(_args) == 0) {
if (!testlib.is_valid_package('.')) {
log.console('No cell.toml found in current directory')
return false
@@ -71,7 +76,7 @@ function parse_args() {
return true
}
if (args[0] == 'all') {
if (_args[0] == 'all') {
if (!testlib.is_valid_package('.')) {
log.console('No cell.toml found in current directory')
return false
@@ -80,28 +85,28 @@ function parse_args() {
return true
}
if (args[0] == 'package') {
if (length(args) < 2) {
if (_args[0] == 'package') {
if (length(_args) < 2) {
log.console('Usage: cell bench package <name> [bench]')
log.console(' cell bench package all')
return false
}
if (args[1] == 'all') {
if (_args[1] == 'all') {
all_pkgs = true
log.console('Benchmarking all packages...')
return true
}
var name = args[1]
var lock = shop.load_lock()
name = _args[1]
lock = shop.load_lock()
if (lock[name]) {
target_pkg = name
} else if (starts_with(name, '/') && testlib.is_valid_package(name)) {
target_pkg = name
} else {
if (testlib.is_valid_package('.')) {
var resolved = pkg.alias_to_package(null, name)
resolved = pkg.alias_to_package(null, name)
if (resolved) {
target_pkg = resolved
} else {
@@ -114,8 +119,8 @@ function parse_args() {
}
}
if (length(args) >= 3) {
target_bench = args[2]
if (length(_args) >= 3) {
target_bench = _args[2]
}
log.console(`Benchmarking package: ${target_pkg}`)
@@ -123,7 +128,7 @@ function parse_args() {
}
// cell bench benches/suite or cell bench <path>
var bench_path = args[0]
bench_path = _args[0]
// Normalize path - add benches/ prefix if not present
if (!starts_with(bench_path, 'benches/') && !starts_with(bench_path, '/')) {
@@ -160,12 +165,15 @@ function collect_benches(package_name, specific_bench) {
var files = pkg.list_files(package_name)
var bench_files = []
arrfor(files, function(f) {
var bench_name = null
var match_name = null
var match_base = null
if (starts_with(f, "benches/") && ends_with(f, ".cm")) {
if (specific_bench) {
var bench_name = text(f, 0, -3)
var match_name = specific_bench
bench_name = text(f, 0, -3)
match_name = specific_bench
if (!starts_with(match_name, 'benches/')) match_name = 'benches/' + match_name
var match_base = ends_with(match_name, '.cm') ? text(match_name, 0, -3) : match_name
match_base = ends_with(match_name, '.cm') ? text(match_name, 0, -3) : match_name
if (bench_name != match_base) return
}
push(bench_files, f)
@@ -180,24 +188,25 @@ function calibrate_batch_size(bench_fn, is_batch) {
var n = MIN_BATCH_SIZE
var dt = 0
var start = 0
var new_n = 0
var calc = 0
var target_n = 0
// Find a batch size that takes at least MIN_SAMPLE_NS
while (n < MAX_BATCH_SIZE) {
// Ensure n is a valid number before calling
if (!is_number(n) || n < 1) {
n = 1
break
}
var start = os.now()
start = os.now()
bench_fn(n)
dt = os.now() - start
if (dt >= MIN_SAMPLE_NS) break
// Double the batch size
var new_n = n * 2
// Check if multiplication produced a valid number
new_n = n * 2
if (!is_number(new_n) || new_n > MAX_BATCH_SIZE) {
n = MAX_BATCH_SIZE
break
@@ -207,10 +216,9 @@ function calibrate_batch_size(bench_fn, is_batch) {
// Adjust to target sample duration
if (dt > 0 && dt < TARGET_SAMPLE_NS && is_number(n) && is_number(dt)) {
var calc = n * TARGET_SAMPLE_NS / dt
calc = n * TARGET_SAMPLE_NS / dt
if (is_number(calc) && calc > 0) {
var target_n = floor(calc)
// Check if floor returned a valid number
target_n = floor(calc)
if (is_number(target_n) && target_n > 0) {
if (target_n > MAX_BATCH_SIZE) target_n = MAX_BATCH_SIZE
if (target_n < MIN_BATCH_SIZE) target_n = MIN_BATCH_SIZE
@@ -219,7 +227,6 @@ function calibrate_batch_size(bench_fn, is_batch) {
}
}
// Safety check - ensure we always return a valid batch size
if (!is_number(n) || n < 1) {
n = 1
}
@@ -230,72 +237,70 @@ function calibrate_batch_size(bench_fn, is_batch) {
// Run a single benchmark function
function run_single_bench(bench_fn, bench_name) {
var timings_per_op = []
// Detect benchmark format:
// 1. Object with { setup, run, teardown } - structured format
// 2. Function that accepts (n) - batch format
// 3. Function that accepts () - legacy format
var is_structured = is_object(bench_fn) && bench_fn.run
var is_batch = false
var batch_size = 1
var setup_fn = null
var run_fn = null
var teardown_fn = null
var calibrate_fn = null
var _detect = null
var i = 0
var state = null
var start = 0
var duration = 0
var ns_per_op = 0
if (is_structured) {
setup_fn = bench_fn.setup || function() { return null }
run_fn = bench_fn.run
teardown_fn = bench_fn.teardown || function(state) {}
teardown_fn = bench_fn.teardown || function(s) {}
// Check if run function accepts batch size
try {
_detect = function() {
var test_state = setup_fn()
run_fn(1, test_state)
is_batch = true
if (teardown_fn) teardown_fn(test_state)
} catch (e) {
} disruption {
is_batch = false
}
_detect()
// Create wrapper for calibration
var calibrate_fn = function(n) {
var state = setup_fn()
run_fn(n, state)
if (teardown_fn) teardown_fn(state)
calibrate_fn = function(n) {
var s = setup_fn()
run_fn(n, s)
if (teardown_fn) teardown_fn(s)
}
batch_size = calibrate_batch_size(calibrate_fn, is_batch)
// Safety check for structured benchmarks
if (!is_number(batch_size) || batch_size < 1) {
batch_size = 1
}
} else {
// Simple function format
try {
_detect = function() {
bench_fn(1)
is_batch = true
} catch (e) {
} disruption {
is_batch = false
}
_detect()
batch_size = calibrate_batch_size(bench_fn, is_batch)
}
// Safety check - ensure batch_size is valid
if (!batch_size || batch_size < 1) {
batch_size = 1
}
// Warmup phase
for (var i = 0; i < WARMUP_BATCHES; i++) {
// Ensure batch_size is valid before warmup
for (i = 0; i < WARMUP_BATCHES; i++) {
if (!is_number(batch_size) || batch_size < 1) {
var type_str = is_null(batch_size) ? 'null' : is_number(batch_size) ? 'number' : is_text(batch_size) ? 'text' : is_object(batch_size) ? 'object' : is_array(batch_size) ? 'array' : is_function(batch_size) ? 'function' : is_logical(batch_size) ? 'logical' : 'unknown'
log.console(`WARNING: batch_size became ${type_str} = ${batch_size}, resetting to 1`)
batch_size = 1
}
if (is_structured) {
var state = setup_fn()
state = setup_fn()
if (is_batch) {
run_fn(batch_size, state)
} else {
@@ -312,35 +317,34 @@ function run_single_bench(bench_fn, bench_name) {
}
// Measurement phase - collect SAMPLES timing samples
for (var i = 0; i < SAMPLES; i++) {
// Double-check batch_size is valid (should never happen, but defensive)
for (i = 0; i < SAMPLES; i++) {
if (!is_number(batch_size) || batch_size < 1) {
batch_size = 1
}
if (is_structured) {
var state = setup_fn()
var start = os.now()
state = setup_fn()
start = os.now()
if (is_batch) {
run_fn(batch_size, state)
} else {
run_fn(state)
}
var duration = os.now() - start
duration = os.now() - start
if (teardown_fn) teardown_fn(state)
var ns_per_op = is_batch ? duration / batch_size : duration
ns_per_op = is_batch ? duration / batch_size : duration
push(timings_per_op, ns_per_op)
} else {
var start = os.now()
start = os.now()
if (is_batch) {
bench_fn(batch_size)
} else {
bench_fn()
}
var duration = os.now() - start
duration = os.now() - start
var ns_per_op = is_batch ? duration / batch_size : duration
ns_per_op = is_batch ? duration / batch_size : duration
push(timings_per_op, ns_per_op)
}
}
@@ -354,7 +358,6 @@ function run_single_bench(bench_fn, bench_name) {
var p95_ns = percentile(timings_per_op, 95)
var p99_ns = percentile(timings_per_op, 99)
// Calculate ops/s from median
var ops_per_sec = 0
if (median_ns > 0) {
ops_per_sec = floor(1000000000 / median_ns)
@@ -408,18 +411,21 @@ function run_benchmarks(package_name, specific_bench) {
arrfor(bench_files, function(f) {
var mod_path = text(f, 0, -3)
var load_error = false
var bench_mod = null
var use_pkg = null
var benches = []
var error_result = null
var file_result = {
name: f,
benchmarks: []
}
try {
var bench_mod
var use_pkg = package_name ? package_name : fd.realpath('.')
var _load_file = function() {
use_pkg = package_name ? package_name : fd.realpath('.')
bench_mod = shop.use(mod_path, use_pkg)
var benches = []
if (is_function(bench_mod)) {
push(benches, {name: 'main', fn: bench_mod})
} else if (is_object(bench_mod)) {
@@ -432,8 +438,11 @@ function run_benchmarks(package_name, specific_bench) {
if (length(benches) > 0) {
log.console(` ${f}`)
arrfor(benches, function(b) {
try {
var result = run_single_bench(b.fn, b.name)
var bench_error = false
var result = null
var _run_bench = function() {
result = run_single_bench(b.fn, b.name)
result.package = pkg_result.package
push(file_result.benchmarks, result)
pkg_result.total++
@@ -444,25 +453,32 @@ function run_benchmarks(package_name, specific_bench) {
if (result.batch_size > 1) {
log.console(` batch: ${result.batch_size} samples: ${result.samples}`)
}
} catch (e) {
log.console(` ERROR ${b.name}: ${e}`)
log.error(e)
var error_result = {
} disruption {
bench_error = true
}
_run_bench()
if (bench_error) {
log.console(` ERROR ${b.name}`)
error_result = {
package: pkg_result.package,
name: b.name,
error: e.toString()
error: "benchmark disrupted"
}
push(file_result.benchmarks, error_result)
pkg_result.total++
}
})
}
} catch (e) {
log.console(` Error loading ${f}: ${e}`)
var error_result = {
} disruption {
load_error = true
}
_load_file()
if (load_error) {
log.console(` Error loading ${f}`)
error_result = {
package: pkg_result.package,
name: "load_module",
error: `Error loading module: ${e}`
error: "error loading module"
}
push(file_result.benchmarks, error_result)
pkg_result.total++
@@ -478,15 +494,16 @@ function run_benchmarks(package_name, specific_bench) {
// Run all benchmarks
var all_results = []
var packages = null
if (all_pkgs) {
if (testlib.is_valid_package('.')) {
push(all_results, run_benchmarks(null, null))
}
var packages = shop.list_packages()
arrfor(packages, function(pkg) {
push(all_results, run_benchmarks(pkg, null))
packages = shop.list_packages()
arrfor(packages, function(p) {
push(all_results, run_benchmarks(p, null))
})
} else {
push(all_results, run_benchmarks(target_pkg, target_bench))