diff --git a/bench.ce b/bench.ce new file mode 100644 index 00000000..6173fd47 --- /dev/null +++ b/bench.ce @@ -0,0 +1,456 @@ +var shop = use('internal/shop') +var pkg = use('package') +var fd = use('fd') +var time = use('time') +var json = use('json') +var utf8 = use('utf8') +var os = use('os') +var testlib = use('internal/testlib') +var math = use('math/radians') + +if (!args) args = [] + +var target_pkg = null // null = current package +var target_bench = null // null = all benchmarks, otherwise specific bench file +var all_pkgs = false + +// Benchmark configuration +def WARMUP_ITERATIONS = 5 +def MIN_ITERATIONS = 10 +def MAX_ITERATIONS = 1000 +def TARGET_DURATION_NS = 1000000000 // 1 second target per benchmark + +// Statistical functions +function median(arr) { + if (arr.length == 0) return 0 + var sorted = arr.slice().sort(function(a, b) { return a - b }) + var mid = number.floor(arr.length / 2) + if (arr.length % 2 == 0) { + return (sorted[mid - 1] + sorted[mid]) / 2 + } + return sorted[mid] +} + +function mean(arr) { + if (arr.length == 0) return 0 + var sum = 0 + for (var i = 0; i < arr.length; i++) { + sum += arr[i] + } + return sum / arr.length +} + +function stddev(arr, mean_val) { + if (arr.length < 2) return 0 + var sum_sq_diff = 0 + for (var i = 0; i < arr.length; i++) { + var diff = arr[i] - mean_val + sum_sq_diff += diff * diff + } + return math.sqrt(sum_sq_diff / (arr.length - 1)) +} + +function percentile(arr, p) { + if (arr.length == 0) return 0 + var sorted = arr.slice().sort(function(a, b) { return a - b }) + var idx = number.floor(arr.length * p / 100) + if (idx >= arr.length) idx = arr.length - 1 + return sorted[idx] +} + +function min_val(arr) { + if (arr.length == 0) return 0 + var m = arr[0] + for (var i = 1; i < arr.length; i++) { + if (arr[i] < m) m = arr[i] + } + return m +} + +function max_val(arr) { + if (arr.length == 0) return 0 + var m = arr[0] + for (var i = 1; i < arr.length; i++) { + if (arr[i] > m) m = arr[i] + } + return m +} + +// Parse arguments similar to test.ce +function parse_args() { + if (args.length == 0) { + if (!testlib.is_valid_package('.')) { + log.console('No cell.toml found in current directory') + return false + } + target_pkg = null + return true + } + + if (args[0] == 'all') { + if (!testlib.is_valid_package('.')) { + log.console('No cell.toml found in current directory') + return false + } + target_pkg = null + return true + } + + if (args[0] == 'package') { + if (args.length < 2) { + log.console('Usage: cell bench package [bench]') + log.console(' cell bench package all') + return false + } + + if (args[1] == 'all') { + all_pkgs = true + log.console('Benchmarking all packages...') + return true + } + + var name = args[1] + var lock = shop.load_lock() + if (lock[name]) { + target_pkg = name + } else if (name.startsWith('/') && testlib.is_valid_package(name)) { + target_pkg = name + } else { + if (testlib.is_valid_package('.')) { + var resolved = pkg.alias_to_package(null, name) + if (resolved) { + target_pkg = resolved + } else { + log.console(`Package not found: ${name}`) + return false + } + } else { + log.console(`Package not found: ${name}`) + return false + } + } + + if (args.length >= 3) { + target_bench = args[2] + } + + log.console(`Benchmarking package: ${target_pkg}`) + return true + } + + // cell bench benches/suite or cell bench + var bench_path = args[0] + + // Normalize path - add benches/ prefix if not present + if (!bench_path.startsWith('benches/') && !bench_path.startsWith('/')) { + if (!fd.is_file(bench_path + '.cm') && !fd.is_file(bench_path)) { + if (fd.is_file('benches/' + bench_path + '.cm') || fd.is_file('benches/' + bench_path)) { + bench_path = 'benches/' + bench_path + } + } + } + + target_bench = bench_path + target_pkg = null + + if (!testlib.is_valid_package('.')) { + log.console('No cell.toml found in current directory') + return false + } + + return true +} + +if (!parse_args()) { + $stop() + return +} + +// Collect benchmark files from a package +function collect_benches(package_name, specific_bench) { + var prefix = testlib.get_pkg_dir(package_name) + var benches_dir = prefix + '/benches' + + if (!fd.is_dir(benches_dir)) return [] + + var files = pkg.list_files(package_name) + var bench_files = [] + for (var i = 0; i < files.length; i++) { + var f = files[i] + if (f.startsWith("benches/") && f.endsWith(".cm")) { + if (specific_bench) { + var bench_name = f.substring(0, f.length - 3) + var match_name = specific_bench + if (!match_name.startsWith('benches/')) match_name = 'benches/' + match_name + var match_base = match_name.endsWith('.cm') ? match_name.substring(0, match_name.length - 3) : match_name + if (bench_name != match_base) continue + } + bench_files.push(f) + } + } + return bench_files +} + +// Run a single benchmark function +function run_single_bench(bench_fn, bench_name) { + var timings = [] + + // Warmup phase + for (var i = 0; i < WARMUP_ITERATIONS; i++) { + bench_fn() + } + + // Determine how many iterations to run + var test_start = os.now() + bench_fn() + var test_duration = os.now() - test_start + + var iterations = MIN_ITERATIONS + if (test_duration > 0) { + iterations = number.floor(TARGET_DURATION_NS / test_duration) + if (iterations < MIN_ITERATIONS) iterations = MIN_ITERATIONS + if (iterations > MAX_ITERATIONS) iterations = MAX_ITERATIONS + } + + // Measurement phase + for (var i = 0; i < iterations; i++) { + var start = os.now() + bench_fn() + var duration = os.now() - start + timings.push(duration) + } + + // Calculate statistics + var mean_ns = mean(timings) + var median_ns = median(timings) + var min_ns = min_val(timings) + var max_ns = max_val(timings) + var stddev_ns = stddev(timings, mean_ns) + var p95_ns = percentile(timings, 95) + var p99_ns = percentile(timings, 99) + + // Calculate ops/s from median + var ops_per_sec = 0 + if (median_ns > 0) { + ops_per_sec = number.floor(1000000000 / median_ns) + } + + return { + name: bench_name, + iterations: iterations, + mean_ns: number.round(mean_ns), + median_ns: number.round(median_ns), + min_ns: number.round(min_ns), + max_ns: number.round(max_ns), + stddev_ns: number.round(stddev_ns), + p95_ns: number.round(p95_ns), + p99_ns: number.round(p99_ns), + ops_per_sec: ops_per_sec + } +} + +// Format nanoseconds for display +function format_ns(ns) { + if (ns < 1000) return `${ns}ns` + if (ns < 1000000) return `${number.round(ns / 1000 * 100) / 100}µs` + if (ns < 1000000000) return `${number.round(ns / 1000000 * 100) / 100}ms` + return `${number.round(ns / 1000000000 * 100) / 100}s` +} + +// Format ops/sec for display +function format_ops(ops) { + if (ops < 1000) return `${ops} ops/s` + if (ops < 1000000) return `${number.round(ops / 1000 * 100) / 100}K ops/s` + if (ops < 1000000000) return `${number.round(ops / 1000000 * 100) / 100}M ops/s` + return `${number.round(ops / 1000000000 * 100) / 100}G ops/s` +} + +// Run benchmarks for a package +function run_benchmarks(package_name, specific_bench) { + var bench_files = collect_benches(package_name, specific_bench) + + var pkg_result = { + package: package_name || "local", + files: [], + total: 0 + } + + if (bench_files.length == 0) return pkg_result + + if (package_name) log.console(`Running benchmarks for ${package_name}`) + else log.console(`Running benchmarks for local package`) + + for (var i = 0; i < bench_files.length; i++) { + var f = bench_files[i] + var mod_path = f.substring(0, f.length - 3) + + var file_result = { + name: f, + benchmarks: [] + } + + try { + var bench_mod + var use_pkg = package_name ? package_name : fd.realpath('.') + bench_mod = shop.use(mod_path, use_pkg) + + var benches = [] + if (typeof bench_mod == 'function') { + benches.push({name: 'main', fn: bench_mod}) + } else if (typeof bench_mod == 'object') { + for (var k in bench_mod) { + if (typeof bench_mod[k] == 'function') { + benches.push({name: k, fn: bench_mod[k]}) + } + } + } + + if (benches.length > 0) { + log.console(` ${f}`) + for (var j = 0; j < benches.length; j++) { + var b = benches[j] + try { + var result = run_single_bench(b.fn, b.name) + result.package = pkg_result.package + file_result.benchmarks.push(result) + pkg_result.total++ + + log.console(` ${result.name}`) + log.console(` ${format_ns(result.median_ns)}/op ${format_ops(result.ops_per_sec)}`) + log.console(` min: ${format_ns(result.min_ns)} max: ${format_ns(result.max_ns)} stddev: ${format_ns(result.stddev_ns)}`) + } catch (e) { + log.console(` ERROR ${b.name}: ${e}`) + log.error(e) + var error_result = { + package: pkg_result.package, + name: b.name, + error: e.toString() + } + file_result.benchmarks.push(error_result) + pkg_result.total++ + } + } + } + } catch (e) { + log.console(` Error loading ${f}: ${e}`) + var error_result = { + package: pkg_result.package, + name: "load_module", + error: `Error loading module: ${e}` + } + file_result.benchmarks.push(error_result) + pkg_result.total++ + } + + if (file_result.benchmarks.length > 0) { + pkg_result.files.push(file_result) + } + } + + return pkg_result +} + +// Run all benchmarks +var all_results = [] + +if (all_pkgs) { + if (testlib.is_valid_package('.')) { + all_results.push(run_benchmarks(null, null)) + } + + var packages = shop.list_packages() + for (var i = 0; i < packages.length; i++) { + all_results.push(run_benchmarks(packages[i], null)) + } +} else { + all_results.push(run_benchmarks(target_pkg, target_bench)) +} + +// Calculate totals +var total_benches = 0 +for (var i = 0; i < all_results.length; i++) { + total_benches += all_results[i].total +} + +log.console(`----------------------------------------`) +log.console(`Benchmarks: ${total_benches} total`) + +// Generate reports +function generate_reports() { + var timestamp = number.floor(time.number()).toString() + var report_dir = shop.get_reports_dir() + '/bench_' + timestamp + testlib.ensure_dir(report_dir) + + var txt_report = `BENCHMARK REPORT +Date: ${time.text(time.number())} +Total benchmarks: ${total_benches} + +=== SUMMARY === +` + for (var i = 0; i < all_results.length; i++) { + var pkg_res = all_results[i] + if (pkg_res.total == 0) continue + txt_report += `Package: ${pkg_res.package}\n` + for (var j = 0; j < pkg_res.files.length; j++) { + var f = pkg_res.files[j] + txt_report += ` ${f.name}\n` + for (var k = 0; k < f.benchmarks.length; k++) { + var b = f.benchmarks[k] + if (b.error) { + txt_report += ` ERROR ${b.name}: ${b.error}\n` + } else { + txt_report += ` ${b.name}: ${format_ns(b.median_ns)}/op (${format_ops(b.ops_per_sec)})\n` + } + } + } + } + + txt_report += `\n=== DETAILED RESULTS ===\n` + for (var i = 0; i < all_results.length; i++) { + var pkg_res = all_results[i] + if (pkg_res.total == 0) continue + + for (var j = 0; j < pkg_res.files.length; j++) { + var f = pkg_res.files[j] + for (var k = 0; k < f.benchmarks.length; k++) { + var b = f.benchmarks[k] + if (b.error) continue + + txt_report += `\n${pkg_res.package}::${b.name}\n` + txt_report += ` iterations: ${b.iterations}\n` + txt_report += ` median: ${format_ns(b.median_ns)}/op\n` + txt_report += ` mean: ${format_ns(b.mean_ns)}/op\n` + txt_report += ` min: ${format_ns(b.min_ns)}\n` + txt_report += ` max: ${format_ns(b.max_ns)}\n` + txt_report += ` stddev: ${format_ns(b.stddev_ns)}\n` + txt_report += ` p95: ${format_ns(b.p95_ns)}\n` + txt_report += ` p99: ${format_ns(b.p99_ns)}\n` + txt_report += ` ops/s: ${format_ops(b.ops_per_sec)}\n` + } + } + } + + testlib.ensure_dir(report_dir) + fd.slurpwrite(`${report_dir}/bench.txt`, utf8.encode(txt_report)) + log.console(`Report written to ${report_dir}/bench.txt`) + + // Generate JSON per package + for (var i = 0; i < all_results.length; i++) { + var pkg_res = all_results[i] + if (pkg_res.total == 0) continue + + var pkg_benches = [] + for (var j = 0; j < pkg_res.files.length; j++) { + var f = pkg_res.files[j] + for (var k = 0; k < f.benchmarks.length; k++) { + pkg_benches.push(f.benchmarks[k]) + } + } + + var json_path = `${report_dir}/${pkg_res.package.replace(/\//g, '_')}.json` + fd.slurpwrite(json_path, utf8.encode(json.encode(pkg_benches))) + } +} + +generate_reports() +$stop() diff --git a/internal/shop.cm b/internal/shop.cm index 4b665e27..0b5b1bbf 100644 --- a/internal/shop.cm +++ b/internal/shop.cm @@ -143,8 +143,22 @@ function abs_path_to_package(package_dir) { if (!fd.is_file(package_dir + '/cell.toml')) throw new Error('Not a valid package directory (no cell.toml): ' + package_dir) - + var packages_prefix = get_packages_dir() + '/' + var core_dir = packages_prefix + core_package + + // Check if this is the core package directory (or its symlink target) + if (package_dir == core_dir) { + return 'core' + } + // Also check if core_dir is a symlink pointing to package_dir + if (fd.is_link(core_dir)) { + var core_target = fd.readlink(core_dir) + if (core_target == package_dir || fd.realpath(core_dir) == package_dir) { + return 'core' + } + } + if (package_dir.startsWith(packages_prefix)) return package_dir.substring(packages_prefix.length) @@ -475,7 +489,10 @@ function resolve_locator(path, ctx) if (fd.is_file(ctx_path)) { var fn = resolve_mod_fn(ctx_path, ctx) - return {path: ctx_path, scope: SCOPE_LOCAL, symbol: fn} + // Check if ctx is the core package (either by name or by path) + var is_core = (ctx == 'core') || (ctx_dir == Shop.get_core_dir()) + var scope = is_core ? SCOPE_CORE : SCOPE_LOCAL + return {path: ctx_path, scope: scope, symbol: fn} } if (is_internal_path(path)) diff --git a/internal/testlib.cm b/internal/testlib.cm new file mode 100644 index 00000000..b50e1d22 --- /dev/null +++ b/internal/testlib.cm @@ -0,0 +1,55 @@ +// Shared test/bench infrastructure +var fd = use('fd') +var pkg = use('package') + +// Check if current directory is a valid cell package +function is_valid_package(dir) { + if (!dir) dir = '.' + return fd.is_file(dir + '/cell.toml') +} + +// Get current package name from cell.toml or null +function get_current_package_name() { + if (!is_valid_package('.')) return null + try { + var config = pkg.load_config(null) + return config.package || 'local' + } catch (e) { + return 'local' + } +} + +// Get the directory for a package +function get_pkg_dir(package_name) { + if (!package_name) { + return fd.realpath('.') + } + if (package_name.startsWith('/')) { + return package_name + } + var shop = use('internal/shop') + return shop.get_package_dir(package_name) +} + +// Ensure directory exists +function ensure_dir(path) { + if (fd.is_dir(path)) return true + + var parts = path.split('/') + var current = path.startsWith('/') ? '/' : '' + for (var i = 0; i < parts.length; i++) { + if (parts[i] == '') continue + current += parts[i] + '/' + if (!fd.is_dir(current)) { + fd.mkdir(current) + } + } + return true +} + +return { + is_valid_package: is_valid_package, + get_current_package_name: get_current_package_name, + get_pkg_dir: get_pkg_dir, + ensure_dir: ensure_dir +}