From c02fbbd9e0bca8a7922fdc9c5ba2cd21ca6102ec Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Tue, 17 Feb 2026 13:37:17 -0600 Subject: [PATCH 1/3] tooling improvements --- build.ce | 8 +- build.cm | 27 +++++-- docs/cli.md | 19 ++++- docs/compiler-tools.md | 30 +++++++- docs/packages.md | 10 +-- docs/shop.md | 2 +- docs/spec/pipeline.md | 19 +++++ dump_stream.cm | 166 ----------------------------------------- seed.ce | 116 ++++++++++++++++++++++++++++ streamline.ce | 160 ++++++++++++++++++++++++++++++++++++++- 10 files changed, 371 insertions(+), 186 deletions(-) delete mode 100644 dump_stream.cm create mode 100644 seed.ce diff --git a/build.ce b/build.ce index 0bc20318..7d3b445a 100644 --- a/build.ce +++ b/build.ce @@ -6,6 +6,7 @@ // cell build Build dynamic library for specific package // cell build -t Cross-compile dynamic libraries for target platform // cell build -b Build type: release (default), debug, or minsize +// cell build --verbose Print resolved flags, commands, and cache status var build = use('build') var shop = use('internal/shop') @@ -15,6 +16,7 @@ var fd = use('fd') var target = null var target_package = null var buildtype = 'release' +var verbose = false var force_rebuild = false var dry_run = false var i = 0 @@ -55,6 +57,8 @@ for (i = 0; i < length(args); i++) { } } else if (args[i] == '--force') { force_rebuild = true + } else if (args[i] == '--verbose' || args[i] == '-v') { + verbose = true } else if (args[i] == '--dry-run') { dry_run = true } else if (args[i] == '--list-targets') { @@ -104,7 +108,7 @@ if (target_package) { // Build single package log.console('Building ' + target_package + '...') _build = function() { - lib = build.build_dynamic(target_package, target, buildtype) + lib = build.build_dynamic(target_package, target, buildtype, {verbose: verbose}) if (lib) { log.console(`Built ${text(length(lib))} module(s)`) } @@ -116,7 +120,7 @@ if (target_package) { } else { // Build all packages log.console('Building all packages...') - results = build.build_all_dynamic(target, buildtype) + results = build.build_all_dynamic(target, buildtype, {verbose: verbose}) success = 0 failed = 0 diff --git a/build.cm b/build.cm index 1b393c19..41ae6af0 100644 --- a/build.cm +++ b/build.cm @@ -172,6 +172,11 @@ Build.compile_file = function(pkg, file, target, opts) { var cmd_str = text(cmd_parts, ' ') + if (_opts.verbose) { + print('[verbose] CFLAGS: ' + text(cflags, ' ')) + print('[verbose] compile: ' + cmd_str) + } + // Content hash: command + file content var file_content = fd.slurp(src_path) var hash_input = cmd_str + '\n' + text(file_content) @@ -183,8 +188,10 @@ Build.compile_file = function(pkg, file, target, opts) { // Check if already compiled if (fd.is_file(obj_path)) { + if (_opts.verbose) print('[verbose] cache hit: ' + file) return obj_path } + if (_opts.verbose) print('[verbose] cache miss: ' + file) // Compile — capture stderr to detect missing-header vs real errors var err_path = '/tmp/cell_build_err_' + hash + '.log' @@ -308,6 +315,10 @@ Build.build_module_dylib = function(pkg, file, target, opts) { var cmd_str = null var ret = null + if (_opts.verbose) { + print('[verbose] LDFLAGS: ' + text(resolved_ldflags, ' ')) + } + if (!fd.is_file(dylib_path)) { cmd_parts = [cc, '-shared', '-fPIC'] @@ -340,6 +351,7 @@ Build.build_module_dylib = function(pkg, file, target, opts) { push(cmd_parts, '"' + dylib_path + '"') cmd_str = text(cmd_parts, ' ') + if (_opts.verbose) print('[verbose] link: ' + cmd_str) log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path)) ret = os.system(cmd_str) if (ret != 0) { @@ -361,6 +373,7 @@ Build.build_module_dylib = function(pkg, file, target, opts) { ensure_dir(install_dir) var install_path = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) + '/' + file_stem + dylib_ext fd.slurpwrite(install_path, fd.slurp(dylib_path)) + if (_opts.verbose) print('[verbose] install: ' + install_path) return dylib_path } @@ -368,9 +381,10 @@ Build.build_module_dylib = function(pkg, file, target, opts) { // Build a dynamic library for a package (one dylib per C file) // Returns array of {file, symbol, dylib} for each module // Also writes a manifest mapping symbols to dylib paths -Build.build_dynamic = function(pkg, target, buildtype) { +Build.build_dynamic = function(pkg, target, buildtype, opts) { var _target = target || Build.detect_host_target() var _buildtype = buildtype || 'release' + var _opts = opts || {} var c_files = pkg_tools.get_c_files(pkg, _target, true) var results = [] @@ -382,13 +396,13 @@ Build.build_dynamic = function(pkg, target, buildtype) { var sources = pkg_tools.get_sources(pkg) var support_objects = [] arrfor(sources, function(src_file) { - var obj = Build.compile_file(pkg, src_file, _target, {buildtype: _buildtype, cflags: cached_cflags}) + var obj = Build.compile_file(pkg, src_file, _target, {buildtype: _buildtype, cflags: cached_cflags, verbose: _opts.verbose}) push(support_objects, obj) }) arrfor(c_files, function(file) { var sym_name = shop.c_symbol_for_file(pkg, file) - var dylib = Build.build_module_dylib(pkg, file, _target, {buildtype: _buildtype, extra_objects: support_objects, cflags: cached_cflags}) + var dylib = Build.build_module_dylib(pkg, file, _target, {buildtype: _buildtype, extra_objects: support_objects, cflags: cached_cflags, verbose: _opts.verbose}) if (dylib) { push(results, {file: file, symbol: sym_name, dylib: dylib}) } @@ -847,9 +861,10 @@ Build.generate_module_table = function(modules, output) { // ============================================================================ // Build dynamic libraries for all installed packages -Build.build_all_dynamic = function(target, buildtype) { +Build.build_all_dynamic = function(target, buildtype, opts) { var _target = target || Build.detect_host_target() var _buildtype = buildtype || 'release' + var _opts = opts || {} var packages = shop.list_packages() var results = [] @@ -860,14 +875,14 @@ Build.build_all_dynamic = function(target, buildtype) { // Build core first if (find(packages, function(p) { return p == 'core' }) != null) { - core_mods = Build.build_dynamic('core', _target, _buildtype) + core_mods = Build.build_dynamic('core', _target, _buildtype, _opts) push(results, {package: 'core', modules: core_mods}) } // Build other packages arrfor(packages, function(pkg) { if (pkg == 'core') return - var pkg_mods = Build.build_dynamic(pkg, _target, _buildtype) + var pkg_mods = Build.build_dynamic(pkg, _target, _buildtype, _opts) push(results, {package: pkg, modules: pkg_mods}) }) diff --git a/docs/cli.md b/docs/cli.md index 80466131..3d7f9f9c 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -65,6 +65,7 @@ pit build -b debug # build type: release (default), debug, minsiz pit build --list-targets # list available targets pit build --force # force rebuild pit build --dry-run # show what would be built +pit build --verbose # print resolved flags, commands, cache status ``` ### pit test @@ -330,9 +331,14 @@ pit mcode Apply the full optimization pipeline to a source file and output optimized mcode as JSON. ```bash -pit streamline +pit streamline # full optimized IR as JSON (default) +pit streamline --stats # summary stats per function +pit streamline --ir # human-readable IR +pit streamline --check # warnings only (e.g. high slot count) ``` +Flags can be combined. `--stats` output includes function name, args, slots, instruction counts by category, and nops eliminated. `--check` warns when `nr_slots > 200` (approaching the 255 limit). + ### pit qbe Compile a source file to QBE intermediate language (for native code generation). @@ -367,6 +373,17 @@ Ahead-of-time compile and execute a program natively. pit run_aot ``` +### pit seed + +Regenerate the boot seed files in `boot/`. Seeds are pre-compiled mcode IR (JSON) that bootstrap the compilation pipeline on cold start. They only need regenerating when the pipeline source changes in a way the existing seeds can't compile, or before distribution. + +```bash +pit seed # regenerate all boot seeds +pit seed --clean # also clear the build cache after +``` + +The engine recompiles pipeline modules automatically when source changes (via content-addressed cache). Seeds are a fallback for cold start when no cache exists. + ### Analysis ### pit explain diff --git a/docs/compiler-tools.md b/docs/compiler-tools.md index 8479dfb9..f6217e8b 100644 --- a/docs/compiler-tools.md +++ b/docs/compiler-tools.md @@ -81,9 +81,37 @@ Shows the optimized IR with type annotations. Each instruction is followed by th Runs the full pipeline (tokenize, parse, fold, mcode, streamline) and outputs the optimized IR as JSON. Useful for piping to `jq` or saving for comparison. ```bash -./cell --core . streamline.ce +./cell --core . streamline.ce # full JSON (default) +./cell --core . streamline.ce --stats # summary stats per function +./cell --core . streamline.ce --ir # human-readable IR +./cell --core . streamline.ce --check # warnings only ``` +| Flag | Description | +|------|-------------| +| (none) | Full optimized IR as JSON (backward compatible) | +| `--stats` | Per-function summary: args, slots, instruction counts by category, nops eliminated | +| `--ir` | Human-readable canonical IR (same format as `ir_report.ce`) | +| `--check` | Warnings only (e.g. `nr_slots > 200` approaching 255 limit) | + +Flags can be combined. + +## seed.ce + +Regenerates the boot seed files in `boot/`. These are pre-compiled mcode IR (JSON) files that bootstrap the compilation pipeline on cold start. + +```bash +./cell --core . seed.ce # regenerate all boot seeds +./cell --core . seed.ce --clean # also clear the build cache after +``` + +The script compiles each pipeline module (tokenize, parse, fold, mcode, streamline) and `internal/bootstrap.cm` through the current pipeline, encodes the output as JSON, and writes it to `boot/.cm.mcode`. + +**When to regenerate seeds:** +- Before a release or distribution +- When the pipeline source changes in a way the existing seeds can't compile the new source (e.g. language-level changes) +- Seeds do NOT need regenerating for normal development — the engine recompiles pipeline modules from source automatically via the content-addressed cache + ## ir_report.ce The optimizer flight recorder. Runs the full pipeline with structured logging and outputs machine-readable, diff-friendly JSON. This is the most detailed tool for understanding what the optimizer did and why. diff --git a/docs/packages.md b/docs/packages.md index 154a1274..6e44b203 100644 --- a/docs/packages.md +++ b/docs/packages.md @@ -9,11 +9,11 @@ Packages are the fundamental unit of code organization and sharing in ƿit. ## Package Structure -A package is a directory containing a `pit.toml` manifest: +A package is a directory containing a `cell.toml` manifest: ``` mypackage/ -├── pit.toml # package manifest +├── cell.toml # package manifest ├── main.ce # entry point (optional) ├── utils.cm # module ├── helper/ @@ -23,7 +23,7 @@ mypackage/ └── helpers.cm # private module (internal/ only) ``` -## pit.toml +## cell.toml The package manifest declares metadata and dependencies: @@ -47,7 +47,7 @@ mylib = "/Users/john/work/mylib" When importing with `use()`, ƿit searches in order: 1. **Local package** — relative to package root -2. **Dependencies** — via aliases in `pit.toml` +2. **Dependencies** — via aliases in `cell.toml` 3. **Core** — built-in ƿit modules ```javascript @@ -179,7 +179,7 @@ C files in a package are compiled into per-file dynamic libraries: ``` mypackage/ -├── pit.toml +├── cell.toml ├── render.c # compiled to lib/mypackage/render.dylib └── physics.c # compiled to lib/mypackage/physics.dylib ``` diff --git a/docs/shop.md b/docs/shop.md index ed8a8291..0a94cb16 100644 --- a/docs/shop.md +++ b/docs/shop.md @@ -48,7 +48,7 @@ When `use('path')` is called from a package context, the shop resolves the modul For a call like `use('sprite')` from package `myapp`: 1. **Own package** — `~/.pit/packages/myapp/sprite.cm` and C symbol `js_myapp_sprite_use` -2. **Aliased dependencies** — if `myapp/pit.toml` has `renderer = "gitea.pockle.world/john/renderer"`, checks `renderer/sprite.cm` and its C symbols +2. **Aliased dependencies** — if `myapp/cell.toml` has `renderer = "gitea.pockle.world/john/renderer"`, checks `renderer/sprite.cm` and its C symbols 3. **Core** — built-in core modules and internal C symbols For calls without a package context (from core modules), only core is searched. diff --git a/docs/spec/pipeline.md b/docs/spec/pipeline.md index 1b02dcf1..4cd2bfca 100644 --- a/docs/spec/pipeline.md +++ b/docs/spec/pipeline.md @@ -93,6 +93,25 @@ String constants are interned in a data section. Integer constants are encoded i pit --emit-qbe script.ce > output.ssa ``` +## Boot Seeds + +The `boot/` directory contains pre-compiled mcode IR (JSON) seed files for the pipeline modules: + +``` +boot/tokenize.cm.mcode +boot/parse.cm.mcode +boot/fold.cm.mcode +boot/mcode.cm.mcode +boot/streamline.cm.mcode +boot/bootstrap.cm.mcode +``` + +Seeds are used during cold start (empty cache) to compile the pipeline modules from source. The engine's `load_pipeline_module()` hashes the **source file** content — if the source changes, the hash changes, the cache misses, and the module is recompiled from source using the boot seeds. This means: + +- Editing a pipeline module (e.g. `tokenize.cm`) takes effect on the next run automatically +- Seeds only need regenerating if the pipeline changes in a way the existing seeds can't compile the new source, or before distribution +- Use `pit seed` to regenerate all seeds, and `pit seed --clean` to also clear the build cache + ## Files | File | Role | diff --git a/dump_stream.cm b/dump_stream.cm deleted file mode 100644 index e74a0443..00000000 --- a/dump_stream.cm +++ /dev/null @@ -1,166 +0,0 @@ -// dump_stream.cm — show mcode IR before and after streamlining -// -// Usage: ./cell --core . dump_stream.cm - -var fd = use("fd") -var json = use("json") -var tokenize = use("tokenize") -var parse = use("parse") -var fold = use("fold") -var mcode = use("mcode") -var streamline = use("streamline") - -if (length(args) < 1) { - print("usage: cell --core . dump_stream.cm ") - return -} - -var filename = args[0] -var src = text(fd.slurp(filename)) -var tok = tokenize(src, filename) -var ast = parse(tok.tokens, src, filename, tokenize) -var folded = fold(ast) -var compiled = mcode(folded) - -// Deep copy IR for before snapshot -var before = json.decode(json.encode(compiled)) - -var optimized = streamline(compiled) - -var pad_right = function(s, w) { - var r = s - while (length(r) < w) { - r = r + " " - } - return r -} - -var fmt_val = function(v) { - if (is_null(v)) { - return "null" - } - if (is_number(v)) { - return text(v) - } - if (is_text(v)) { - return `"${v}"` - } - if (is_object(v)) { - return json.encode(v) - } - if (is_logical(v)) { - return v ? "true" : "false" - } - return text(v) -} - -var count_stats = function(func) { - var instrs = func.instructions - var total = 0 - var nops = 0 - var calls = 0 - var i = 0 - var instr = null - if (instrs == null) { - return {total: 0, nops: 0, real: 0, calls: 0} - } - while (i < length(instrs)) { - instr = instrs[i] - if (is_text(instr)) { - if (starts_with(instr, "_nop_")) { - nops = nops + 1 - } - } else if (is_array(instr)) { - total = total + 1 - if (instr[0] == "invoke") { - calls = calls + 1 - } - } - i = i + 1 - } - return {total: total, nops: nops, real: total - nops, calls: calls} -} - -var dump_function = function(func, show_nops) { - var instrs = func.instructions - var i = 0 - var pc = 0 - var instr = null - var op = null - var n = 0 - var parts = null - var j = 0 - var operands = null - var pc_str = null - var op_str = null - if (instrs == null || length(instrs) == 0) { - return null - } - while (i < length(instrs)) { - instr = instrs[i] - if (is_text(instr)) { - if (starts_with(instr, "_nop_")) { - if (show_nops) { - print(` ${pad_right(text(pc), 5)} --- nop ---`) - pc = pc + 1 - } - } else { - print(`${instr}:`) - } - } else if (is_array(instr)) { - op = instr[0] - n = length(instr) - parts = [] - j = 1 - while (j < n - 2) { - push(parts, fmt_val(instr[j])) - j = j + 1 - } - operands = text(parts, ", ") - pc_str = pad_right(text(pc), 5) - op_str = pad_right(op, 14) - print(` ${pc_str} ${op_str} ${operands}`) - pc = pc + 1 - } - i = i + 1 - } - return null -} - -var dump_pair = function(before_func, after_func, name) { - var nr_args = after_func.nr_args != null ? after_func.nr_args : 0 - var nr_slots = after_func.nr_slots != null ? after_func.nr_slots : 0 - var b_stats = count_stats(before_func) - var a_stats = count_stats(after_func) - var eliminated = a_stats.nops - print(`\n=== ${name} (args=${text(nr_args)}, slots=${text(nr_slots)}) ===`) - print(` before: ${text(b_stats.total)} instructions, ${text(b_stats.calls)} invokes`) - print(` after: ${text(a_stats.real)} instructions (${text(eliminated)} eliminated), ${text(a_stats.calls)} invokes`) - print("\n -- streamlined --") - dump_function(after_func, false) - return null -} - -var main_name = null -var fi = 0 -var func = null -var bfunc = null -var fname = null - -// Dump main -if (optimized.main != null && before.main != null) { - main_name = optimized.name != null ? optimized.name : "
" - dump_pair(before.main, optimized.main, main_name) -} - -// Dump sub-functions -if (optimized.functions != null && before.functions != null) { - fi = 0 - while (fi < length(optimized.functions)) { - func = optimized.functions[fi] - bfunc = before.functions[fi] - fname = func.name != null ? func.name : `` - dump_pair(bfunc, func, `[${text(fi)}] ${fname}`) - fi = fi + 1 - } -} diff --git a/seed.ce b/seed.ce new file mode 100644 index 00000000..148126fa --- /dev/null +++ b/seed.ce @@ -0,0 +1,116 @@ +// seed.ce — regenerate boot seed files in boot/ +// +// Usage: +// pit seed Regenerate all boot seeds +// pit seed --clean Also clear the build cache after +// +// Seeds are the pre-compiled mcode IR (JSON) files that bootstrap the +// compilation pipeline on cold start. They only need regenerating when: +// - The pipeline source (tokenize, parse, fold, mcode, streamline) changes +// in a way that the boot seeds can't compile the new source +// - You want the seed files to match the current pipeline output +// (e.g. before a release or distribution) +// +// The engine already recompiles pipeline modules from source when their +// content changes (content-addressed cache). Seeds are a fallback for +// cold start when no cache exists. + +var fd = use("fd") +var json = use("json") +var os = use("os") +var shop = use("internal/shop") +var tokenize = use("tokenize") +var parse = use("parse") +var fold = use("fold") +var mcode = use("mcode") +var streamline = use("streamline") + +var clean = false +var i = 0 + +for (i = 0; i < length(args); i++) { + if (args[i] == '--clean') { + clean = true + } else if (args[i] == '--help' || args[i] == '-h') { + print("usage: pit seed [--clean]") + print("") + print(" Regenerate boot seed files in boot/") + print(" --clean Also clear the build cache after") + $stop() + } +} + +var core_dir = shop.get_package_dir('core') +var boot_dir = core_dir + '/boot' +var pipeline_modules = ['tokenize', 'parse', 'fold', 'mcode', 'streamline'] +var generated = 0 +var name = null +var src_path = null +var src = null +var tok = null +var ast = null +var folded = null +var compiled = null +var optimized = null +var mcode_json = null +var out_path = null + +// Regenerate pipeline module seeds +for (i = 0; i < length(pipeline_modules); i++) { + name = pipeline_modules[i] + src_path = core_dir + '/' + name + '.cm' + + if (!fd.is_file(src_path)) { + print('WARNING: source not found: ' + src_path) + continue + } + + print('Seeding ' + name + '.cm ...') + src = text(fd.slurp(src_path)) + tok = tokenize(src, src_path) + ast = parse(tok.tokens, src, src_path, tokenize) + folded = fold(ast) + compiled = mcode(folded) + optimized = streamline(compiled) + mcode_json = json.encode(optimized) + + out_path = boot_dir + '/' + name + '.cm.mcode' + fd.slurpwrite(out_path, stone(blob(mcode_json))) + print(' -> ' + out_path + ' (' + text(length(mcode_json)) + ' bytes)') + generated = generated + 1 +} + +// Regenerate bootstrap.cm seed +var bootstrap_path = core_dir + '/internal/bootstrap.cm' +if (fd.is_file(bootstrap_path)) { + print('Seeding bootstrap.cm ...') + src = text(fd.slurp(bootstrap_path)) + tok = tokenize(src, bootstrap_path) + ast = parse(tok.tokens, src, bootstrap_path, tokenize) + folded = fold(ast) + compiled = mcode(folded) + optimized = streamline(compiled) + mcode_json = json.encode(optimized) + + out_path = boot_dir + '/bootstrap.cm.mcode' + fd.slurpwrite(out_path, stone(blob(mcode_json))) + print(' -> ' + out_path + ' (' + text(length(mcode_json)) + ' bytes)') + generated = generated + 1 +} else { + print('WARNING: bootstrap source not found: ' + bootstrap_path) +} + +print('\nRegenerated ' + text(generated) + ' seed(s)') + +if (clean) { + var build_dir = shop.get_build_dir() + if (fd.is_dir(build_dir)) { + print('Clearing build cache: ' + build_dir) + os.system('rm -rf "' + build_dir + '"') + print('Build cache cleared. Next run will recompile from new seeds.') + } else { + print('No build cache to clear.') + } +} + +$stop() diff --git a/streamline.ce b/streamline.ce index 334a86e4..ae011d32 100644 --- a/streamline.ce +++ b/streamline.ce @@ -1,6 +1,10 @@ -// streamline.ce — run the full compile + optimize pipeline, output JSON +// streamline.ce — run the full compile + optimize pipeline // -// Usage: ./cell --core . streamline.ce +// Usage: +// pit streamline Full optimized IR as JSON (default) +// pit streamline --stats Summary stats per function +// pit streamline --ir Human-readable IR +// pit streamline --check Warnings only (e.g. high slot count) var fd = use("fd") var json = use("json") @@ -9,11 +13,159 @@ var parse = use("parse") var fold = use("fold") var mcode = use("mcode") var streamline = use("streamline") -var filename = args[0] + +var show_stats = false +var show_ir = false +var show_check = false +var filename = null +var i = 0 + +for (i = 0; i < length(args); i++) { + if (args[i] == '--stats') { + show_stats = true + } else if (args[i] == '--ir') { + show_ir = true + } else if (args[i] == '--check') { + show_check = true + } else if (!starts_with(args[i], '-')) { + filename = args[i] + } +} + +if (!filename) { + print("usage: pit streamline [--stats] [--ir] [--check] ") + $stop() +} + var src = text(fd.slurp(filename)) var result = tokenize(src, filename) var ast = parse(result.tokens, src, filename, tokenize) var folded = fold(ast) var compiled = mcode(folded) + +// Deep copy for before snapshot (needed by --stats) +var before = null +if (show_stats) { + before = json.decode(json.encode(compiled)) +} + var optimized = streamline(compiled) -print(json.encode(optimized, true)) + +// If no flags, default to full JSON output +if (!show_stats && !show_ir && !show_check) { + print(json.encode(optimized, true)) + $stop() +} + +// --- Helpers --- + +var ir_stats = use("ir_stats") + +var pad_right = function(s, w) { + var r = s + while (length(r) < w) { + r = r + " " + } + return r +} + +var count_nops = function(func) { + var instrs = func.instructions + var nops = 0 + var i = 0 + if (instrs == null) return 0 + while (i < length(instrs)) { + if (is_text(instrs[i]) && starts_with(instrs[i], "_nop_")) { + nops = nops + 1 + } + i = i + 1 + } + return nops +} + +var print_func_stats = function(func, before_func, name) { + var nr_args = func.nr_args != null ? func.nr_args : 0 + var nr_slots = func.nr_slots != null ? func.nr_slots : 0 + var nr_close = func.nr_close_slots != null ? func.nr_close_slots : 0 + var stats = ir_stats.detailed_stats(func) + var nops = count_nops(func) + var before_stats = before_func ? ir_stats.detailed_stats(before_func) : null + var before_total = before_stats ? before_stats.instr : stats.instr + + print(` ${name}`) + print(` args=${text(nr_args)} slots=${text(nr_slots)} close_slots=${text(nr_close)}`) + print(` instructions: ${text(stats.instr)} total, ${text(nops)} nops eliminated`) + if (before_stats) { + print(` before: ${text(before_total)} after: ${text(stats.instr - nops)}`) + } + print(` load=${text(stats.load)} store=${text(stats.store)} branch=${text(stats.branch)} call=${text(stats.call)}`) + print(` guard=${text(stats.guard)} arith=${text(stats.arith)} move=${text(stats.move)} const=${text(stats.const)}`) + + if (nr_slots > 200) { + print(` WARNING: nr_slots=${text(nr_slots)} approaching 255 limit`) + } +} + +var print_func_ir = function(func, name) { + var ir_text = ir_stats.canonical_ir(func, name, {show_nops: true}) + print(ir_text) +} + +var check_func = function(func, name) { + var nr_slots = func.nr_slots != null ? func.nr_slots : 0 + if (nr_slots > 200) { + print(`WARNING: ${name} has ${text(nr_slots)} slots (approaching 255 limit)`) + } +} + +// --- Process functions --- + +var main_name = optimized.name != null ? optimized.name : "
" +var fi = 0 +var func = null +var bfunc = null +var fname = null + +if (show_stats) { + print(`\n--- Stats for ${filename} ---`) +} + +// Main function +if (optimized.main != null) { + if (show_stats) { + print_func_stats(optimized.main, before ? before.main : null, main_name) + } + if (show_ir) { + print_func_ir(optimized.main, main_name) + } + if (show_check) { + check_func(optimized.main, main_name) + } +} + +// Sub-functions +if (optimized.functions != null) { + fi = 0 + while (fi < length(optimized.functions)) { + func = optimized.functions[fi] + bfunc = before ? before.functions[fi] : null + fname = func.name != null ? func.name : `` + + if (show_stats) { + print_func_stats(func, bfunc, fname) + } + if (show_ir) { + print_func_ir(func, fname) + } + if (show_check) { + check_func(func, fname) + } + fi = fi + 1 + } +} + +if (show_stats) { + print('---') +} + +$stop() From 51815b66d84be2bb64f16b174add27c69b4fcc00 Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Tue, 17 Feb 2026 14:00:23 -0600 Subject: [PATCH 2/3] json rooting fix --- source/runtime.c | 23 +++++++--- source/suite.c | 116 ++++++++++++++++++++++++++++++++++++++++++++++- vm_suite.ce | 89 ++++++++++++++++++++++++++++++++++++ 3 files changed, 219 insertions(+), 9 deletions(-) diff --git a/source/runtime.c b/source/runtime.c index 9313ad6b..08be4ee9 100644 --- a/source/runtime.c +++ b/source/runtime.c @@ -5749,6 +5749,18 @@ exception: return JS_EXCEPTION; } +/* Check if val is already on the visited stack (circular reference detection). + Uses identity comparison (===) since we're checking for the same object. */ +static BOOL json_stack_has (JSContext *ctx, JSValue stack, JSValue val) { + if (!JS_IsArray (stack)) return FALSE; + JSArray *arr = JS_VALUE_GET_ARRAY (stack); + for (word_t i = 0; i < arr->len; i++) { + if (JS_StrictEq (ctx, arr->values[i], val)) + return TRUE; + } + return FALSE; +} + static int js_json_to_str (JSContext *ctx, JSONStringifyContext *jsc, JSValue holder, JSValue val, JSValue indent) { JSValue v; int64_t i, len; @@ -5784,9 +5796,7 @@ static int js_json_to_str (JSContext *ctx, JSONStringifyContext *jsc, JSValue ho if (mist_is_gc_object ( val_ref.val)) { /* includes arrays (OBJ_ARRAY) since they have JS_TAG_PTR */ - v = js_array_includes (ctx, jsc->stack, 1, &val_ref.val); - if (JS_IsException (v)) goto exception; - if (JS_ToBool (ctx, v)) { + if (json_stack_has (ctx, jsc->stack, val_ref.val)) { JS_ThrowTypeError (ctx, "circular reference"); goto exception; } @@ -5801,8 +5811,7 @@ static int js_json_to_str (JSContext *ctx, JSONStringifyContext *jsc, JSValue ho sep_ref.val = jsc->empty; sep1_ref.val = jsc->empty; } - v = js_cell_push (ctx, jsc->stack, 1, &val_ref.val); - if (check_exception_free (ctx, v)) goto exception; + if (JS_ArrayPush (ctx, &jsc->stack, val_ref.val) < 0) goto exception; ret = JS_IsArray (val_ref.val); if (ret < 0) goto exception; if (ret) { @@ -5890,8 +5899,8 @@ static int js_json_to_str (JSContext *ctx, JSONStringifyContext *jsc, JSValue ho } JSC_B_PUTC (jsc, '}'); } - if (check_exception_free (ctx, js_cell_pop (ctx, jsc->stack, 0, NULL))) - goto exception; + v = JS_ArrayPop (ctx, jsc->stack); + if (JS_IsException (v)) goto exception; goto done; } switch (JS_VALUE_GET_NORM_TAG (val_ref.val)) { diff --git a/source/suite.c b/source/suite.c index a64385ab..bfb791c6 100644 --- a/source/suite.c +++ b/source/suite.c @@ -1847,7 +1847,25 @@ TEST(is_integer_vs_number) { /* JSON Tests */ TEST(json_encode_object) { - /* Skip - requires GC rooting fixes in JS_JSONStringify */ + /* Build an object with several properties and stringify with pretty=true */ + JSGCRef obj_ref; + JS_PushGCRef(ctx, &obj_ref); + obj_ref.val = JS_NewObject(ctx); + JS_SetPropertyStr(ctx, obj_ref.val, "name", JS_NewString(ctx, "test")); + JS_SetPropertyStr(ctx, obj_ref.val, "value", JS_NewInt32(ctx, 42)); + JS_SetPropertyStr(ctx, obj_ref.val, "active", JS_NewBool(ctx, 1)); + JS_SetPropertyStr(ctx, obj_ref.val, "tag", JS_NewString(ctx, "hello world")); + JSValue space = JS_NewInt32(ctx, 2); + JSValue str = JS_JSONStringify(ctx, obj_ref.val, JS_NULL, space, 1); + JS_PopGCRef(ctx, &obj_ref); + ASSERT(!JS_IsException(str)); + ASSERT(JS_IsText(str)); + const char *s = JS_ToCString(ctx, str); + ASSERT(s != NULL); + ASSERT(strstr(s, "\"name\"") != NULL); + ASSERT(strstr(s, "\"test\"") != NULL); + ASSERT(strstr(s, "42") != NULL); + JS_FreeCString(ctx, s); return 1; } @@ -1867,7 +1885,98 @@ TEST(json_decode_object) { } TEST(json_roundtrip_array) { - /* Skip - requires GC rooting fixes in JS_JSONStringify */ + JSGCRef arr_ref; + JS_PushGCRef(ctx, &arr_ref); + arr_ref.val = JS_NewArray(ctx); + JS_ArrayPush(ctx, &arr_ref.val, JS_NewInt32(ctx, 10)); + JS_ArrayPush(ctx, &arr_ref.val, JS_NewString(ctx, "two")); + JS_ArrayPush(ctx, &arr_ref.val, JS_NewBool(ctx, 0)); + JSValue str = JS_JSONStringify(ctx, arr_ref.val, JS_NULL, JS_NULL, 0); + JS_PopGCRef(ctx, &arr_ref); + ASSERT(!JS_IsException(str)); + ASSERT(JS_IsText(str)); + const char *s = JS_ToCString(ctx, str); + ASSERT(s != NULL); + ASSERT(strstr(s, "10") != NULL); + ASSERT(strstr(s, "\"two\"") != NULL); + ASSERT(strstr(s, "false") != NULL); + JS_FreeCString(ctx, s); + return 1; +} + +TEST(json_encode_large_object) { + /* Stress test: build object with many properties, stringify with pretty print. + Under FORCE_GC_AT_MALLOC this will trigger GC on every allocation, + exposing any GC rooting bugs in the JSON encoder. */ + JSGCRef obj_ref, str_ref; + JS_PushGCRef(ctx, &obj_ref); + JS_PushGCRef(ctx, &str_ref); + obj_ref.val = JS_NewObject(ctx); + char key[32], val[64]; + for (int i = 0; i < 50; i++) { + snprintf(key, sizeof(key), "key_%d", i); + snprintf(val, sizeof(val), "value_%d_with_some_padding_text", i); + JS_SetPropertyStr(ctx, obj_ref.val, key, JS_NewString(ctx, val)); + } + JSValue space = JS_NewInt32(ctx, 2); + str_ref.val = JS_JSONStringify(ctx, obj_ref.val, JS_NULL, space, 1); + ASSERT(!JS_IsException(str_ref.val)); + ASSERT(JS_IsText(str_ref.val)); + const char *s = JS_ToCString(ctx, str_ref.val); + ASSERT(s != NULL); + ASSERT(strstr(s, "\"key_0\"") != NULL); + ASSERT(strstr(s, "\"key_49\"") != NULL); + JS_FreeCString(ctx, s); + JS_PopGCRef(ctx, &str_ref); + JS_PopGCRef(ctx, &obj_ref); + return 1; +} + +TEST(json_encode_nested) { + /* Nested objects stress test */ + JSGCRef outer_ref, inner_ref, str_ref; + JS_PushGCRef(ctx, &outer_ref); + JS_PushGCRef(ctx, &inner_ref); + JS_PushGCRef(ctx, &str_ref); + outer_ref.val = JS_NewObject(ctx); + char key[32], val[64]; + for (int i = 0; i < 20; i++) { + inner_ref.val = JS_NewObject(ctx); + for (int j = 0; j < 10; j++) { + snprintf(key, sizeof(key), "f%d", j); + snprintf(val, sizeof(val), "v_%d_%d", i, j); + JS_SetPropertyStr(ctx, inner_ref.val, key, JS_NewString(ctx, val)); + } + snprintf(key, sizeof(key), "obj_%d", i); + JS_SetPropertyStr(ctx, outer_ref.val, key, inner_ref.val); + } + JSValue space = JS_NewInt32(ctx, 2); + str_ref.val = JS_JSONStringify(ctx, outer_ref.val, JS_NULL, space, 1); + ASSERT(!JS_IsException(str_ref.val)); + ASSERT(JS_IsText(str_ref.val)); + const char *s = JS_ToCString(ctx, str_ref.val); + ASSERT(s != NULL); + ASSERT(strstr(s, "\"obj_0\"") != NULL); + ASSERT(strstr(s, "\"f0\"") != NULL); + JS_FreeCString(ctx, s); + JS_PopGCRef(ctx, &str_ref); + JS_PopGCRef(ctx, &inner_ref); + JS_PopGCRef(ctx, &outer_ref); + return 1; +} + +TEST(json_circular_reference) { + /* Circular reference should throw, not infinite recurse */ + JSGCRef obj_ref; + JS_PushGCRef(ctx, &obj_ref); + obj_ref.val = JS_NewObject(ctx); + JS_SetPropertyStr(ctx, obj_ref.val, "name", JS_NewString(ctx, "root")); + /* Create circular reference: obj.self = obj */ + JS_SetPropertyStr(ctx, obj_ref.val, "self", obj_ref.val); + JSValue str = JS_JSONStringify(ctx, obj_ref.val, JS_NULL, JS_NULL, 0); + JS_PopGCRef(ctx, &obj_ref); + /* Should be an exception (circular reference), not a crash */ + ASSERT(JS_IsException(str)); return 1; } @@ -2196,6 +2305,9 @@ int run_c_test_suite(JSContext *ctx) RUN_TEST(json_encode_object); RUN_TEST(json_decode_object); RUN_TEST(json_roundtrip_array); + RUN_TEST(json_encode_large_object); + RUN_TEST(json_encode_nested); + RUN_TEST(json_circular_reference); printf("\nSerialization - NOTA:\n"); RUN_TEST(nota_encode_int); diff --git a/vm_suite.ce b/vm_suite.ce index ec6c566f..c14dd11c 100644 --- a/vm_suite.ce +++ b/vm_suite.ce @@ -5011,6 +5011,95 @@ run("nested function used after definition", function() { assert_eq(result[1], 'b = "hi"', "nested fn encode text") }) +// ============================================================================ +// JSON ENCODING +// ============================================================================ + +def json = use("json") + +run("json encode flat object", function() { + var obj = {} + var i = 0 + for (i = 0; i < 500; i++) { + obj[text(i)] = "value_" + text(i) + } + var result = json.encode(obj) + assert_eq(is_text(result), true, "encode returns text") + var decoded = json.decode(result) + assert_eq(decoded["0"], "value_0", "first property survives roundtrip") + assert_eq(decoded["499"], "value_499", "last property survives roundtrip") +}) + +run("json encode nested objects", function() { + var outer = {} + var i = 0 + var j = 0 + var inner = null + for (i = 0; i < 50; i++) { + inner = {} + for (j = 0; j < 20; j++) { + inner[text(j)] = i * 20 + j + } + outer[text(i)] = inner + } + var result = json.encode(outer) + var decoded = json.decode(result) + assert_eq(decoded["0"]["0"], 0, "nested first value") + assert_eq(decoded["49"]["19"], 999, "nested last value") +}) + +run("json encode array", function() { + var arr = [1, "two", true, null, 3.14] + var result = json.encode(arr) + var decoded = json.decode(result) + assert_eq(decoded[0], 1, "array number") + assert_eq(decoded[1], "two", "array text") + assert_eq(decoded[2], true, "array logical") + assert_eq(decoded[3], null, "array null") + assert_eq(decoded[4], 3.14, "array float") +}) + +run("json circular reference detected", function() { + var circ = {} + circ.name = "root" + circ.self = circ + if (!should_disrupt(function() { json.encode(circ) })) { + fail("circular reference not detected") + } +}) + +run("json deeply nested circular reference", function() { + var a = {} + var b = {} + var c = {} + a.child = b + b.child = c + c.child = a + if (!should_disrupt(function() { json.encode(a) })) { + fail("deep circular reference not detected") + } +}) + +run("json roundtrip preserves types", function() { + var obj = { + "num": 42, + "txt": "hello", + "yes": true, + "no": false, + "nil": null, + "arr": [1, 2, 3], + "sub": {"a": 1} + } + var decoded = json.decode(json.encode(obj)) + assert_eq(decoded.num, 42, "number preserved") + assert_eq(decoded.txt, "hello", "text preserved") + assert_eq(decoded.yes, true, "true preserved") + assert_eq(decoded.no, false, "false preserved") + assert_eq(decoded.nil, null, "null preserved") + assert_eq(decoded.arr[2], 3, "array preserved") + assert_eq(decoded.sub.a, 1, "sub-object preserved") +}) + // ============================================================================ // SUMMARY // ============================================================================ From b3573dbf267979c8cbb8f0fa43bb2b2fcca8bf4c Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Tue, 17 Feb 2026 15:48:49 -0600 Subject: [PATCH 3/3] native flag --- compile_seed.ce | 98 ---------------------------------------------- internal/engine.cm | 41 ++++++++++++++++++- internal/shop.cm | 79 ++++++++++++++++++++++++++++++++----- run_native.ce | 44 ++++++++------------- source/cell.c | 16 +++++++- 5 files changed, 139 insertions(+), 139 deletions(-) delete mode 100644 compile_seed.ce diff --git a/compile_seed.ce b/compile_seed.ce deleted file mode 100644 index 46c59295..00000000 --- a/compile_seed.ce +++ /dev/null @@ -1,98 +0,0 @@ -// compile_seed.ce — compile a .cm module to native .dylib via QBE (seed mode) -// Usage: ./cell --dev --seed compile_seed - -var fd = use("fd") -var os = use("os") -var tokenize = use("tokenize") -var parse = use("parse") -var fold = use("fold") -var mcode = use("mcode") -var streamline = use("streamline") -var qbe_macros = use("qbe") -var qbe_emit = use("qbe_emit") - -if (length(args) < 1) { - print("usage: cell --dev --seed compile_seed ") - disrupt -} - -var file = args[0] -var base = file -if (ends_with(base, ".cm")) { - base = text(base, 0, length(base) - 3) -} else if (ends_with(base, ".ce")) { - base = text(base, 0, length(base) - 3) -} - -var safe = replace(replace(replace(base, "/", "_"), "-", "_"), ".", "_") -var symbol = "js_" + safe + "_use" -var tmp = "/tmp/qbe_" + safe -var ssa_path = tmp + ".ssa" -var s_path = tmp + ".s" -var o_path = tmp + ".o" -var rt_o_path = "/tmp/qbe_rt.o" -var dylib_path = file + ".dylib" -var rc = 0 - -// Step 1: compile to QBE IL -print("compiling " + file + " to QBE IL...") -var src = text(fd.slurp(file)) -var result = tokenize(src, file) -var ast = parse(result.tokens, src, file, tokenize) -var folded = fold(ast) -var compiled = mcode(folded) -var optimized = streamline(compiled) -var il = qbe_emit(optimized, qbe_macros) - -// Step 2: append wrapper function -var wrapper = ` -export function l $${symbol}(l %ctx) { -@entry - %result =l call $cell_rt_module_entry(l %ctx) - ret %result -} -` -il = il + wrapper - -// Write IL to file — remove old file first to avoid leftover content -if (fd.is_file(ssa_path)) fd.unlink(ssa_path) -var out_fd = fd.open(ssa_path, 1537, 420) -fd.write(out_fd, il) -fd.close(out_fd) -print("wrote " + ssa_path + " (" + text(length(il)) + " bytes)") - -// Step 3: compile QBE IL to assembly -print("qbe compile...") -rc = os.system("qbe -o " + s_path + " " + ssa_path) -if (rc != 0) { - print("qbe compilation failed") - disrupt -} - -// Step 4: assemble -print("assemble...") -rc = os.system("cc -c " + s_path + " -o " + o_path) -if (rc != 0) { - print("assembly failed") - disrupt -} - -// Step 5: compile runtime stubs -if (!fd.is_file(rt_o_path)) { - print("compile runtime stubs...") - rc = os.system("cc -c source/qbe_helpers.c -o " + rt_o_path + " -fPIC -Isource") - if (rc != 0) { - print("runtime stubs compilation failed") - disrupt - } -} - -// Step 6: link dylib -print("link...") -rc = os.system("cc -shared -fPIC -undefined dynamic_lookup " + o_path + " " + rt_o_path + " -o " + dylib_path) -if (rc != 0) { - print("linking failed") - disrupt -} - -print("built: " + dylib_path) diff --git a/internal/engine.cm b/internal/engine.cm index 221a42fd..4d4fec46 100644 --- a/internal/engine.cm +++ b/internal/engine.cm @@ -1,6 +1,7 @@ // Hidden vars (os, actorsym, init, core_path, shop_path, json, args) come from env // Engine is self-sufficient: defines its own compilation pipeline var ACTORDATA = actorsym +var native_mode = false var SYSYM = '__SYSTEM__' var _cell = {} @@ -216,13 +217,24 @@ var _program = null var _user_args = [] var _j = 1 var _init = init -if (args != null && _init == null) { + +// Inherit native_mode from init (set by C for --native, or by parent actor) +if (_init != null && _init.native_mode) + native_mode = true + +// CLI path: convert args to init record +if (args != null && (_init == null || !_init.program)) { _program = args[0] while (_j < length(args)) { push(_user_args, args[_j]) _j = _j + 1 } - _init = {program: _program, arg: _user_args} + if (_init == null) { + _init = {program: _program, arg: _user_args} + } else { + _init.program = _program + _init.arg = _user_args + } } use_cache['core/os'] = os @@ -413,9 +425,11 @@ core_extras.content_hash = content_hash core_extras.cache_path = cache_path core_extras.ensure_build_dir = ensure_build_dir core_extras.compile_to_blob = compile_to_blob +core_extras.native_mode = native_mode // NOW load shop -- it receives all of the above via env var shop = use_core('internal/shop') +if (native_mode) use_core('build') var time = use_core('time') var pronto = use_core('pronto') @@ -690,6 +704,7 @@ $_.start = function start(cb, program) { overling_id: oid, root_id: root ? root[ACTORDATA].id : null, program, + native_mode: native_mode, } greeters[id] = cb push(message_queue, { startup }) @@ -1082,6 +1097,28 @@ $_.clock(_ => { env.log = log env = stone(env) + var native_build = null + var native_dylib_path = null + var native_handle = null + var native_parts = null + var native_basename = null + var native_sym = null + + // Native execution path: compile to dylib and run + if (native_mode) { + native_build = use_core('build') + native_dylib_path = native_build.compile_native(prog_path, null, null, pkg) + native_handle = os.dylib_open(native_dylib_path) + native_parts = array(prog_path, '/') + native_basename = native_parts[length(native_parts) - 1] + native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null + if (native_sym) + os.native_module_load_named(native_handle, native_sym, env) + else + os.native_module_load(native_handle, env) + return + } + var source_blob = fd.slurp(prog_path) var hash = content_hash(source_blob) var cached_path = cache_path(hash) diff --git a/internal/shop.cm b/internal/shop.cm index 5bb1ac2d..66aa02f1 100644 --- a/internal/shop.cm +++ b/internal/shop.cm @@ -303,7 +303,8 @@ var _default_policy = { allow_dylib: true, allow_static: true, allow_mach: true, - allow_compile: true + allow_compile: true, + native: false } Shop.load_config = function() { @@ -336,6 +337,7 @@ Shop.load_config = function() { function get_policy() { var config = Shop.load_config() + if (native_mode) config.policy.native = true return config.policy } @@ -433,14 +435,37 @@ function detect_host_target() { var host_target = detect_host_target() // Check for a native .cm dylib at the deterministic lib path -// Returns the loaded module value, or null if no native dylib exists +// Returns a native descriptor {_native, _handle, _sym}, or null if no native dylib exists +// Also checks staleness: if source has changed, the content-addressed build artifact +// won't exist for the new hash, so the installed dylib is treated as stale. function try_native_mod_dylib(pkg, stem) { var dylib_path = get_dylib_path(pkg, stem) + var src_path = null + var src = null + var host = null + var hash = null + var tc_ext = null + var build_path = null + var handle = null + var sym = null + if (!fd.is_file(dylib_path)) return null - var handle = os.dylib_open(dylib_path) + + // Staleness check: verify the content-addressed build artifact exists + src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem + if (fd.is_file(src_path)) { + src = text(fd.slurp(src_path)) + host = detect_host_target() + hash = content_hash(src + '\n' + host + '\nnative') + tc_ext = dylib_ext + build_path = global_shop_path + '/build/' + hash + '.' + host + tc_ext + if (!fd.is_file(build_path)) return null + } + + handle = os.dylib_open(dylib_path) if (!handle) return null - var sym = Shop.c_symbol_for_file(pkg, stem) - return os.native_module_load_named(handle, sym) + sym = Shop.c_symbol_for_file(pkg, stem) + return {_native: true, _handle: handle, _sym: sym} } // Default capabilities injected into scripts @@ -511,6 +536,10 @@ function resolve_mod_fn(path, pkg) { var _pkg_dir = null var _stem = null var policy = null + var build_mod = null + var dylib_path = null + var handle = null + var sym = null policy = get_policy() @@ -525,8 +554,21 @@ function resolve_mod_fn(path, pkg) { // Check for native .cm dylib at deterministic path first if (policy.allow_dylib && pkg && _stem) { native_result = try_native_mod_dylib(pkg, _stem) - if (native_result != null) { - return {_native: true, value: native_result} + if (native_result != null) return native_result + } + + // Native compilation path: compile to native dylib instead of mach + if (policy.native && policy.allow_compile) { + build_mod = use_cache['core/build'] + if (build_mod) { + dylib_path = build_mod.compile_native(path, null, null, pkg) + if (dylib_path) { + handle = os.dylib_open(dylib_path) + if (handle) { + sym = pkg && _stem ? Shop.c_symbol_for_file(pkg, _stem) : null + return {_native: true, _handle: handle, _sym: sym} + } + } } } @@ -950,9 +992,16 @@ function execute_module(info) var pkg = null if (mod_resolve.scope < 900) { - // Check if native dylib was resolved + // Check if native dylib was resolved (descriptor with _handle and _sym) if (is_object(mod_resolve.symbol) && mod_resolve.symbol._native) { - used = mod_resolve.symbol.value + file_info = Shop.file_info(mod_resolve.path) + inject = Shop.script_inject_for(file_info) + env = inject_env(inject) + pkg = file_info.package + env.use = make_use_fn(pkg) + env = stone(env) + used = os.native_module_load_named( + mod_resolve.symbol._handle, mod_resolve.symbol._sym, env) } else { // Build env with runtime fns, capabilities, and use function file_info = Shop.file_info(mod_resolve.path) @@ -1607,6 +1656,8 @@ Shop.load_as_dylib = function(path, pkg) { var stem = null var result = null var real_pkg = pkg + var inject = null + var env = null if (!locator) { print('Module ' + path + ' not found'); disrupt } @@ -1621,7 +1672,15 @@ Shop.load_as_dylib = function(path, pkg) { if (!starts_with(file_path, pkg_dir + '/')) return null stem = text(file_path, length(pkg_dir) + 1) result = try_native_mod_dylib(real_pkg, stem) - return result + if (!result) return null + + // Build env and load the native module + if (!file_info) file_info = Shop.file_info(file_path) + inject = Shop.script_inject_for(file_info) + env = inject_env(inject) + env.use = make_use_fn(real_pkg) + env = stone(env) + return os.native_module_load_named(result._handle, result._sym, env) } Shop.audit_packages = function() { diff --git a/run_native.ce b/run_native.ce index 90408eff..0d803521 100644 --- a/run_native.ce +++ b/run_native.ce @@ -1,16 +1,18 @@ // run_native.ce — load a module both interpreted and native, compare speed // // Usage: -// cell --dev run_native.ce +// cell --dev run_native // -// Loads .cm via use() (interpreted) and .cm.dylib (native), +// Loads .cm via use() (interpreted) and via shop.use_native() (native), // runs both and compares results and timing. var os = use('os') +var fd = use('fd') +var shop = use('internal/shop') if (length(args) < 1) { - print('usage: cell --dev run_native.ce ') - print(' e.g. cell --dev run_native.ce num_torture') + print('usage: cell --dev run_native ') + print(' e.g. cell --dev run_native num_torture') return } @@ -19,11 +21,6 @@ if (ends_with(name, '.cm')) { name = text(name, 0, length(name) - 3) } -var safe = replace(replace(name, '/', '_'), '-', '_') -var symbol = 'js_' + safe + '_use' -var dylib_path = './' + name + '.cm.dylib' -var fd = use('fd') - // --- Test argument for function-returning modules --- var test_arg = 5000000 if (length(args) > 1) { @@ -48,44 +45,35 @@ print('result: ' + text(result_interp)) print('time: ' + text(ms_interp) + ' ms') // --- Native run --- -if (!fd.is_file(dylib_path)) { - print('\nno ' + dylib_path + ' found — run compile.ce first') +// Resolve to .cm path for shop.use_native() +var mod_path = name + '.cm' +if (!fd.is_file(mod_path)) { + print('\nno ' + mod_path + ' found') return } print('\n--- native ---') var t3 = os.now() -var lib = os.dylib_open(dylib_path) +var mod_native = shop.use_native(mod_path) var t4 = os.now() -var mod_native = os.dylib_symbol(lib, symbol) -var t5 = os.now() var result_native = null if (is_function(mod_native)) { print('module returns a function, calling with ' + text(test_arg)) - t4 = os.now() + t3 = os.now() result_native = mod_native(test_arg) - t5 = os.now() + t4 = os.now() } result_native = result_native != null ? result_native : mod_native -var ms_load = (t4 - t3) / 1000000 -var ms_exec = (t5 - t4) / 1000000 -var ms_native = (t5 - t3) / 1000000 +var ms_native = (t4 - t3) / 1000000 print('result: ' + text(result_native)) -print('load: ' + text(ms_load) + ' ms') -print('exec: ' + text(ms_exec) + ' ms') -print('total: ' + text(ms_native) + ' ms') +print('time: ' + text(ms_native) + ' ms') // --- Comparison --- print('\n--- comparison ---') var match = result_interp == result_native var speedup = 0 -var speedup_exec = 0 print('match: ' + text(match)) if (ms_native > 0) { speedup = ms_interp / ms_native - print('speedup: ' + text(speedup) + 'x (total)') -} -if (ms_exec > 0) { - speedup_exec = ms_interp / ms_exec - print('speedup: ' + text(speedup_exec) + 'x (exec only)') + print('speedup: ' + text(speedup) + 'x') } diff --git a/source/cell.c b/source/cell.c index ee47027d..ad91899a 100644 --- a/source/cell.c +++ b/source/cell.c @@ -29,6 +29,7 @@ static int run_test_suite(size_t heap_size); cell_rt *root_cell = NULL; static char *shop_path = NULL; static char *core_path = NULL; +static int native_mode = 0; static JSRuntime *g_runtime = NULL; // Compute blake2b hash of data and return hex string (caller must free) @@ -434,6 +435,7 @@ static void print_usage(const char *prog) printf(" --core Set core path directly (overrides CELL_CORE)\n"); printf(" --shop Set shop path (overrides CELL_SHOP)\n"); printf(" --dev Dev mode (shop=.cell, core=.)\n"); + printf(" --native Use AOT native code instead of bytecode\n"); printf(" --heap Initial heap size (e.g. 256MB, 1GB)\n"); printf(" --test [heap_size] Run C test suite\n"); printf(" -h, --help Show this help message\n"); @@ -510,6 +512,9 @@ int cell_init(int argc, char **argv) if (lstat(".cell/packages/core", &lst) != 0) symlink("../..", ".cell/packages/core"); arg_start++; + } else if (strcmp(argv[arg_start], "--native") == 0) { + native_mode = 1; + arg_start++; } else { break; } @@ -648,7 +653,16 @@ int cell_init(int argc, char **argv) JS_SetPropertyStr(ctx, env_ref.val, "actorsym", JS_DupValue(ctx, cli_rt->actor_sym_ref.val)); tmp = js_core_json_use(ctx); JS_SetPropertyStr(ctx, env_ref.val, "json", tmp); - JS_SetPropertyStr(ctx, env_ref.val, "init", JS_NULL); + if (native_mode) { + JSGCRef init_ref; + JS_AddGCRef(ctx, &init_ref); + init_ref.val = JS_NewObject(ctx); + JS_SetPropertyStr(ctx, init_ref.val, "native_mode", JS_NewBool(ctx, 1)); + JS_SetPropertyStr(ctx, env_ref.val, "init", init_ref.val); + JS_DeleteGCRef(ctx, &init_ref); + } else { + JS_SetPropertyStr(ctx, env_ref.val, "init", JS_NULL); + } JSGCRef args_ref; JS_AddGCRef(ctx, &args_ref); args_ref.val = JS_NewArray(ctx);