diff --git a/compile_seed.ce b/compile_seed.ce deleted file mode 100644 index 46c59295..00000000 --- a/compile_seed.ce +++ /dev/null @@ -1,98 +0,0 @@ -// compile_seed.ce — compile a .cm module to native .dylib via QBE (seed mode) -// Usage: ./cell --dev --seed compile_seed - -var fd = use("fd") -var os = use("os") -var tokenize = use("tokenize") -var parse = use("parse") -var fold = use("fold") -var mcode = use("mcode") -var streamline = use("streamline") -var qbe_macros = use("qbe") -var qbe_emit = use("qbe_emit") - -if (length(args) < 1) { - print("usage: cell --dev --seed compile_seed ") - disrupt -} - -var file = args[0] -var base = file -if (ends_with(base, ".cm")) { - base = text(base, 0, length(base) - 3) -} else if (ends_with(base, ".ce")) { - base = text(base, 0, length(base) - 3) -} - -var safe = replace(replace(replace(base, "/", "_"), "-", "_"), ".", "_") -var symbol = "js_" + safe + "_use" -var tmp = "/tmp/qbe_" + safe -var ssa_path = tmp + ".ssa" -var s_path = tmp + ".s" -var o_path = tmp + ".o" -var rt_o_path = "/tmp/qbe_rt.o" -var dylib_path = file + ".dylib" -var rc = 0 - -// Step 1: compile to QBE IL -print("compiling " + file + " to QBE IL...") -var src = text(fd.slurp(file)) -var result = tokenize(src, file) -var ast = parse(result.tokens, src, file, tokenize) -var folded = fold(ast) -var compiled = mcode(folded) -var optimized = streamline(compiled) -var il = qbe_emit(optimized, qbe_macros) - -// Step 2: append wrapper function -var wrapper = ` -export function l $${symbol}(l %ctx) { -@entry - %result =l call $cell_rt_module_entry(l %ctx) - ret %result -} -` -il = il + wrapper - -// Write IL to file — remove old file first to avoid leftover content -if (fd.is_file(ssa_path)) fd.unlink(ssa_path) -var out_fd = fd.open(ssa_path, 1537, 420) -fd.write(out_fd, il) -fd.close(out_fd) -print("wrote " + ssa_path + " (" + text(length(il)) + " bytes)") - -// Step 3: compile QBE IL to assembly -print("qbe compile...") -rc = os.system("qbe -o " + s_path + " " + ssa_path) -if (rc != 0) { - print("qbe compilation failed") - disrupt -} - -// Step 4: assemble -print("assemble...") -rc = os.system("cc -c " + s_path + " -o " + o_path) -if (rc != 0) { - print("assembly failed") - disrupt -} - -// Step 5: compile runtime stubs -if (!fd.is_file(rt_o_path)) { - print("compile runtime stubs...") - rc = os.system("cc -c source/qbe_helpers.c -o " + rt_o_path + " -fPIC -Isource") - if (rc != 0) { - print("runtime stubs compilation failed") - disrupt - } -} - -// Step 6: link dylib -print("link...") -rc = os.system("cc -shared -fPIC -undefined dynamic_lookup " + o_path + " " + rt_o_path + " -o " + dylib_path) -if (rc != 0) { - print("linking failed") - disrupt -} - -print("built: " + dylib_path) diff --git a/internal/engine.cm b/internal/engine.cm index 221a42fd..4d4fec46 100644 --- a/internal/engine.cm +++ b/internal/engine.cm @@ -1,6 +1,7 @@ // Hidden vars (os, actorsym, init, core_path, shop_path, json, args) come from env // Engine is self-sufficient: defines its own compilation pipeline var ACTORDATA = actorsym +var native_mode = false var SYSYM = '__SYSTEM__' var _cell = {} @@ -216,13 +217,24 @@ var _program = null var _user_args = [] var _j = 1 var _init = init -if (args != null && _init == null) { + +// Inherit native_mode from init (set by C for --native, or by parent actor) +if (_init != null && _init.native_mode) + native_mode = true + +// CLI path: convert args to init record +if (args != null && (_init == null || !_init.program)) { _program = args[0] while (_j < length(args)) { push(_user_args, args[_j]) _j = _j + 1 } - _init = {program: _program, arg: _user_args} + if (_init == null) { + _init = {program: _program, arg: _user_args} + } else { + _init.program = _program + _init.arg = _user_args + } } use_cache['core/os'] = os @@ -413,9 +425,11 @@ core_extras.content_hash = content_hash core_extras.cache_path = cache_path core_extras.ensure_build_dir = ensure_build_dir core_extras.compile_to_blob = compile_to_blob +core_extras.native_mode = native_mode // NOW load shop -- it receives all of the above via env var shop = use_core('internal/shop') +if (native_mode) use_core('build') var time = use_core('time') var pronto = use_core('pronto') @@ -690,6 +704,7 @@ $_.start = function start(cb, program) { overling_id: oid, root_id: root ? root[ACTORDATA].id : null, program, + native_mode: native_mode, } greeters[id] = cb push(message_queue, { startup }) @@ -1082,6 +1097,28 @@ $_.clock(_ => { env.log = log env = stone(env) + var native_build = null + var native_dylib_path = null + var native_handle = null + var native_parts = null + var native_basename = null + var native_sym = null + + // Native execution path: compile to dylib and run + if (native_mode) { + native_build = use_core('build') + native_dylib_path = native_build.compile_native(prog_path, null, null, pkg) + native_handle = os.dylib_open(native_dylib_path) + native_parts = array(prog_path, '/') + native_basename = native_parts[length(native_parts) - 1] + native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null + if (native_sym) + os.native_module_load_named(native_handle, native_sym, env) + else + os.native_module_load(native_handle, env) + return + } + var source_blob = fd.slurp(prog_path) var hash = content_hash(source_blob) var cached_path = cache_path(hash) diff --git a/internal/shop.cm b/internal/shop.cm index 5bb1ac2d..66aa02f1 100644 --- a/internal/shop.cm +++ b/internal/shop.cm @@ -303,7 +303,8 @@ var _default_policy = { allow_dylib: true, allow_static: true, allow_mach: true, - allow_compile: true + allow_compile: true, + native: false } Shop.load_config = function() { @@ -336,6 +337,7 @@ Shop.load_config = function() { function get_policy() { var config = Shop.load_config() + if (native_mode) config.policy.native = true return config.policy } @@ -433,14 +435,37 @@ function detect_host_target() { var host_target = detect_host_target() // Check for a native .cm dylib at the deterministic lib path -// Returns the loaded module value, or null if no native dylib exists +// Returns a native descriptor {_native, _handle, _sym}, or null if no native dylib exists +// Also checks staleness: if source has changed, the content-addressed build artifact +// won't exist for the new hash, so the installed dylib is treated as stale. function try_native_mod_dylib(pkg, stem) { var dylib_path = get_dylib_path(pkg, stem) + var src_path = null + var src = null + var host = null + var hash = null + var tc_ext = null + var build_path = null + var handle = null + var sym = null + if (!fd.is_file(dylib_path)) return null - var handle = os.dylib_open(dylib_path) + + // Staleness check: verify the content-addressed build artifact exists + src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem + if (fd.is_file(src_path)) { + src = text(fd.slurp(src_path)) + host = detect_host_target() + hash = content_hash(src + '\n' + host + '\nnative') + tc_ext = dylib_ext + build_path = global_shop_path + '/build/' + hash + '.' + host + tc_ext + if (!fd.is_file(build_path)) return null + } + + handle = os.dylib_open(dylib_path) if (!handle) return null - var sym = Shop.c_symbol_for_file(pkg, stem) - return os.native_module_load_named(handle, sym) + sym = Shop.c_symbol_for_file(pkg, stem) + return {_native: true, _handle: handle, _sym: sym} } // Default capabilities injected into scripts @@ -511,6 +536,10 @@ function resolve_mod_fn(path, pkg) { var _pkg_dir = null var _stem = null var policy = null + var build_mod = null + var dylib_path = null + var handle = null + var sym = null policy = get_policy() @@ -525,8 +554,21 @@ function resolve_mod_fn(path, pkg) { // Check for native .cm dylib at deterministic path first if (policy.allow_dylib && pkg && _stem) { native_result = try_native_mod_dylib(pkg, _stem) - if (native_result != null) { - return {_native: true, value: native_result} + if (native_result != null) return native_result + } + + // Native compilation path: compile to native dylib instead of mach + if (policy.native && policy.allow_compile) { + build_mod = use_cache['core/build'] + if (build_mod) { + dylib_path = build_mod.compile_native(path, null, null, pkg) + if (dylib_path) { + handle = os.dylib_open(dylib_path) + if (handle) { + sym = pkg && _stem ? Shop.c_symbol_for_file(pkg, _stem) : null + return {_native: true, _handle: handle, _sym: sym} + } + } } } @@ -950,9 +992,16 @@ function execute_module(info) var pkg = null if (mod_resolve.scope < 900) { - // Check if native dylib was resolved + // Check if native dylib was resolved (descriptor with _handle and _sym) if (is_object(mod_resolve.symbol) && mod_resolve.symbol._native) { - used = mod_resolve.symbol.value + file_info = Shop.file_info(mod_resolve.path) + inject = Shop.script_inject_for(file_info) + env = inject_env(inject) + pkg = file_info.package + env.use = make_use_fn(pkg) + env = stone(env) + used = os.native_module_load_named( + mod_resolve.symbol._handle, mod_resolve.symbol._sym, env) } else { // Build env with runtime fns, capabilities, and use function file_info = Shop.file_info(mod_resolve.path) @@ -1607,6 +1656,8 @@ Shop.load_as_dylib = function(path, pkg) { var stem = null var result = null var real_pkg = pkg + var inject = null + var env = null if (!locator) { print('Module ' + path + ' not found'); disrupt } @@ -1621,7 +1672,15 @@ Shop.load_as_dylib = function(path, pkg) { if (!starts_with(file_path, pkg_dir + '/')) return null stem = text(file_path, length(pkg_dir) + 1) result = try_native_mod_dylib(real_pkg, stem) - return result + if (!result) return null + + // Build env and load the native module + if (!file_info) file_info = Shop.file_info(file_path) + inject = Shop.script_inject_for(file_info) + env = inject_env(inject) + env.use = make_use_fn(real_pkg) + env = stone(env) + return os.native_module_load_named(result._handle, result._sym, env) } Shop.audit_packages = function() { diff --git a/run_native.ce b/run_native.ce index 90408eff..0d803521 100644 --- a/run_native.ce +++ b/run_native.ce @@ -1,16 +1,18 @@ // run_native.ce — load a module both interpreted and native, compare speed // // Usage: -// cell --dev run_native.ce +// cell --dev run_native // -// Loads .cm via use() (interpreted) and .cm.dylib (native), +// Loads .cm via use() (interpreted) and via shop.use_native() (native), // runs both and compares results and timing. var os = use('os') +var fd = use('fd') +var shop = use('internal/shop') if (length(args) < 1) { - print('usage: cell --dev run_native.ce ') - print(' e.g. cell --dev run_native.ce num_torture') + print('usage: cell --dev run_native ') + print(' e.g. cell --dev run_native num_torture') return } @@ -19,11 +21,6 @@ if (ends_with(name, '.cm')) { name = text(name, 0, length(name) - 3) } -var safe = replace(replace(name, '/', '_'), '-', '_') -var symbol = 'js_' + safe + '_use' -var dylib_path = './' + name + '.cm.dylib' -var fd = use('fd') - // --- Test argument for function-returning modules --- var test_arg = 5000000 if (length(args) > 1) { @@ -48,44 +45,35 @@ print('result: ' + text(result_interp)) print('time: ' + text(ms_interp) + ' ms') // --- Native run --- -if (!fd.is_file(dylib_path)) { - print('\nno ' + dylib_path + ' found — run compile.ce first') +// Resolve to .cm path for shop.use_native() +var mod_path = name + '.cm' +if (!fd.is_file(mod_path)) { + print('\nno ' + mod_path + ' found') return } print('\n--- native ---') var t3 = os.now() -var lib = os.dylib_open(dylib_path) +var mod_native = shop.use_native(mod_path) var t4 = os.now() -var mod_native = os.dylib_symbol(lib, symbol) -var t5 = os.now() var result_native = null if (is_function(mod_native)) { print('module returns a function, calling with ' + text(test_arg)) - t4 = os.now() + t3 = os.now() result_native = mod_native(test_arg) - t5 = os.now() + t4 = os.now() } result_native = result_native != null ? result_native : mod_native -var ms_load = (t4 - t3) / 1000000 -var ms_exec = (t5 - t4) / 1000000 -var ms_native = (t5 - t3) / 1000000 +var ms_native = (t4 - t3) / 1000000 print('result: ' + text(result_native)) -print('load: ' + text(ms_load) + ' ms') -print('exec: ' + text(ms_exec) + ' ms') -print('total: ' + text(ms_native) + ' ms') +print('time: ' + text(ms_native) + ' ms') // --- Comparison --- print('\n--- comparison ---') var match = result_interp == result_native var speedup = 0 -var speedup_exec = 0 print('match: ' + text(match)) if (ms_native > 0) { speedup = ms_interp / ms_native - print('speedup: ' + text(speedup) + 'x (total)') -} -if (ms_exec > 0) { - speedup_exec = ms_interp / ms_exec - print('speedup: ' + text(speedup_exec) + 'x (exec only)') + print('speedup: ' + text(speedup) + 'x') } diff --git a/source/cell.c b/source/cell.c index ee47027d..ad91899a 100644 --- a/source/cell.c +++ b/source/cell.c @@ -29,6 +29,7 @@ static int run_test_suite(size_t heap_size); cell_rt *root_cell = NULL; static char *shop_path = NULL; static char *core_path = NULL; +static int native_mode = 0; static JSRuntime *g_runtime = NULL; // Compute blake2b hash of data and return hex string (caller must free) @@ -434,6 +435,7 @@ static void print_usage(const char *prog) printf(" --core Set core path directly (overrides CELL_CORE)\n"); printf(" --shop Set shop path (overrides CELL_SHOP)\n"); printf(" --dev Dev mode (shop=.cell, core=.)\n"); + printf(" --native Use AOT native code instead of bytecode\n"); printf(" --heap Initial heap size (e.g. 256MB, 1GB)\n"); printf(" --test [heap_size] Run C test suite\n"); printf(" -h, --help Show this help message\n"); @@ -510,6 +512,9 @@ int cell_init(int argc, char **argv) if (lstat(".cell/packages/core", &lst) != 0) symlink("../..", ".cell/packages/core"); arg_start++; + } else if (strcmp(argv[arg_start], "--native") == 0) { + native_mode = 1; + arg_start++; } else { break; } @@ -648,7 +653,16 @@ int cell_init(int argc, char **argv) JS_SetPropertyStr(ctx, env_ref.val, "actorsym", JS_DupValue(ctx, cli_rt->actor_sym_ref.val)); tmp = js_core_json_use(ctx); JS_SetPropertyStr(ctx, env_ref.val, "json", tmp); - JS_SetPropertyStr(ctx, env_ref.val, "init", JS_NULL); + if (native_mode) { + JSGCRef init_ref; + JS_AddGCRef(ctx, &init_ref); + init_ref.val = JS_NewObject(ctx); + JS_SetPropertyStr(ctx, init_ref.val, "native_mode", JS_NewBool(ctx, 1)); + JS_SetPropertyStr(ctx, env_ref.val, "init", init_ref.val); + JS_DeleteGCRef(ctx, &init_ref); + } else { + JS_SetPropertyStr(ctx, env_ref.val, "init", JS_NULL); + } JSGCRef args_ref; JS_AddGCRef(ctx, &args_ref); args_ref.val = JS_NewArray(ctx);