diff --git a/internal/engine.cm b/internal/engine.cm index 5883ad62..6a171985 100644 --- a/internal/engine.cm +++ b/internal/engine.cm @@ -34,13 +34,17 @@ var packages_path = shop_path ? shop_path + '/packages' : null // Self-sufficient initialization: content-addressed cache var use_cache = {} +// Save blob intrinsic before var blob = use_core('blob') hoists and shadows it. +// Function declarations see the hoisted null; IIFEs see the intrinsic. +var _make_blob = (function() { return blob })() + function content_hash(content) { var data = content - if (!is_blob(data)) data = stone(blob(text(data))) + if (!is_blob(data)) data = stone(_make_blob(text(data))) return text(crypto.blake2(data), 'h') } -function cache_path(hash) { +function pipeline_cache_path(hash) { if (!shop_path) return null return shop_path + '/build/' + hash } @@ -75,7 +79,7 @@ function detect_cc() { function native_dylib_cache_path(src, target) { var native_key = src + '\n' + target + '\nnative\n' var full_key = native_key + '\nnative' - return cache_path(content_hash(full_key)) + return pipeline_cache_path(content_hash(full_key)) } var _engine_host_target = null @@ -134,9 +138,11 @@ function load_pipeline_module(name, env) { if (fd.is_file(source_path)) { if (!source_blob) source_blob = fd.slurp(source_path) hash = content_hash(source_blob) - cached = cache_path(hash) - if (cached && fd.is_file(cached)) + cached = pipeline_cache_path(hash) + if (cached && fd.is_file(cached)) { + log.system('engine: pipeline ' + name + ' (cached)') return mach_load(fd.slurp(cached), env) + } // Cache miss: compile from source using boot seed pipeline mcode_path = core_path + '/boot/' + name + '.cm.mcode' @@ -158,6 +164,7 @@ function load_pipeline_module(name, env) { compiled = boot_sl(compiled) mcode_json = json.encode(compiled) mach_blob = mach_compile_mcode_bin(name, mcode_json) + log.system('engine: pipeline ' + name + ' (compiled)') if (!native_mode && cached) { ensure_build_dir() fd.slurpwrite(cached, mach_blob) @@ -209,6 +216,31 @@ if (native_mode) { use_cache['core/qbe_emit'] = _qbe_emit_mod } +var compiler_fingerprint = (function() { + var files = [ + "tokenize", "parse", "fold", "mcode", "streamline", + "qbe", "qbe_emit", "ir_stats" + ] + var combined = "" + var i = 0 + var path = null + while (i < length(files)) { + path = core_path + '/' + files[i] + '.cm' + if (fd.is_file(path)) + combined = combined + text(fd.slurp(path)) + i = i + 1 + } + return content_hash(stone(blob(combined))) +})() + +function module_cache_path(content, salt) { + if (!shop_path) return null + var s = salt || 'mach' + return shop_path + '/build/' + content_hash( + stone(_make_blob(text(content) + '\n' + s + '\n' + compiler_fingerprint)) + ) +} + // analyze: tokenize + parse + fold, check for errors function analyze(src, filename) { var tok_result = tokenize_mod(src, filename) @@ -592,7 +624,6 @@ function use_core(path) { arrfor(array(core_extras), function(k) { env[k] = core_extras[k] }) env = stone(env) - var hash = null var cached_path = null var mach_blob = null var source_blob = null @@ -629,14 +660,15 @@ function use_core(path) { // Bytecode path (fallback or non-native mode) _load_mod = function() { if (!source_blob) source_blob = fd.slurp(file_path) - hash = content_hash(source_blob) - cached_path = cache_path(hash) + cached_path = module_cache_path(source_blob, 'mach') if (cached_path && fd.is_file(cached_path)) { + log.system('engine: cache hit for core/' + path) result = mach_load(fd.slurp(cached_path), env) } else { script = text(source_blob) ast = analyze(script, file_path) mach_blob = compile_to_blob('core:' + path, ast) + log.system('engine: compiled core/' + path) if (!native_mode && cached_path) { ensure_build_dir() fd.slurpwrite(cached_path, mach_blob) @@ -733,8 +765,10 @@ function actor_die(err) if (underlings) { unders = array(underlings) arrfor(unders, function(id, index) { - log.console(`calling on ${id} to disrupt too`) - $_.stop(create_actor({id})) + if (!is_null(underlings[id])) { + log.system(`stopping underling ${id}`) + $_.stop(create_actor({id})) + } }) } @@ -786,7 +820,8 @@ core_extras.actor_api = $_ core_extras.log = log core_extras.runtime_env = runtime_env core_extras.content_hash = content_hash -core_extras.cache_path = cache_path +core_extras.cache_path = module_cache_path +core_extras.compiler_fingerprint = compiler_fingerprint core_extras.ensure_build_dir = ensure_build_dir core_extras.compile_to_blob = compile_to_blob core_extras.native_mode = native_mode @@ -1253,6 +1288,7 @@ $_.start = function start(cb, program) { root_id: root ? root[ACTORDATA].id : null, program, native_mode: native_mode, + no_warn: _no_warn, } greeters[id] = cb push(message_queue, { startup }) @@ -1300,7 +1336,7 @@ $_.couple = function couple(actor) { if (actor == $_.self) return // can't couple to self couplings[actor[ACTORDATA].id] = true sys_msg(actor, {kind:'couple', from_id: _cell.id}) - log.system(`coupled to ${actor}`) + log.system(`coupled to ${actor[ACTORDATA].id}`) } function actor_prep(actor, send) { @@ -1363,7 +1399,7 @@ function actor_send(actor, message) { } return } - log.system(`Unable to send message to actor ${actor[ACTORDATA]}`) + log.system(`Unable to send message to actor ${actor[ACTORDATA].id}`) } function send_messages() { @@ -1525,7 +1561,7 @@ function handle_sysym(msg) } greeter(greet_msg) } - if (msg.message.type == 'disrupt') + if (msg.message.type == 'disrupt' || msg.message.type == 'stop') delete underlings[from_id] } else if (msg.kind == 'contact') { if (portal_fn) { @@ -1549,7 +1585,7 @@ function handle_message(msg) { var fn = null if (msg[SYSYM]) { - handle_sysym(msg[SYSYM], msg.from) + handle_sysym(msg[SYSYM]) return } @@ -1734,52 +1770,81 @@ $_.clock(_ => { env.log = log env = stone(env) - var native_build = null - var native_dylib_path = null - var native_handle = null - var native_basename = null - var native_sym = null + // --- run_program: execute the resolved program --- + function run_program() { + var native_build = null + var native_dylib_path = null + var native_handle = null + var native_basename = null + var native_sym = null - // Native execution path: compile to dylib and run - if (native_mode) { - native_build = use_core('build') - native_dylib_path = native_build.compile_native(prog_path, null, null, pkg) - native_handle = os.dylib_open(native_dylib_path) - native_basename = file_info.name ? file_info.name + (file_info.is_actor ? '.ce' : '.cm') : fd.basename(prog_path) - native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null - if (native_sym) - os.native_module_load_named(native_handle, native_sym, env) - else - os.native_module_load(native_handle, env) - return - } - - var source_blob = fd.slurp(prog_path) - var hash = content_hash(source_blob) - var cached_path = cache_path(hash) - var val = null - var script = null - var ast = null - var mach_blob = null - var _compile = function() { - if (cached_path && fd.is_file(cached_path)) { - val = mach_load(fd.slurp(cached_path), env) - } else { - script = text(source_blob) - ast = analyze(script, prog_path) - mach_blob = compile_user_blob(prog, ast, pkg) - if (cached_path) { - ensure_build_dir() - fd.slurpwrite(cached_path, mach_blob) - } - val = mach_load(mach_blob, env) + // Native execution path: compile to dylib and run + if (native_mode) { + native_build = use_core('build') + native_dylib_path = native_build.compile_native(prog_path, null, null, pkg) + native_handle = os.dylib_open(native_dylib_path) + native_basename = file_info.name ? file_info.name + (file_info.is_actor ? '.ce' : '.cm') : fd.basename(prog_path) + native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null + if (native_sym) + os.native_module_load_named(native_handle, native_sym, env) + else + os.native_module_load(native_handle, env) + return + } + + var source_blob = fd.slurp(prog_path) + var _cached_path = module_cache_path(source_blob, 'mach') + var val = null + var script = null + var ast = null + var mach_blob = null + var _compile = function() { + if (_cached_path && fd.is_file(_cached_path)) { + val = mach_load(fd.slurp(_cached_path), env) + } else { + script = text(source_blob) + ast = analyze(script, prog_path) + mach_blob = compile_user_blob(prog, ast, pkg) + if (_cached_path) { + ensure_build_dir() + fd.slurpwrite(_cached_path, mach_blob) + } + val = mach_load(mach_blob, env) + } + } disruption { + os.exit(1) + } + _compile() + if (val) { + log.error('Program must not return anything') + disrupt } - } disruption { - os.exit(1) } - _compile() - if (val) { - log.error('Program must not return anything') - disrupt + + // --- Auto-boot: pre-compile uncached deps before running --- + // Only auto-boot for the root program (not child actors, not boot itself). + // Delegates all discovery + compilation to boot.ce (separate actor/memory). + var _is_root_actor = !_cell.args.overling_id + var _skip_boot = !_is_root_actor || prog == 'boot' || prog == 'compile_worker' + + if (_skip_boot) { + run_program() + } else { + $_.start(function(event) { + if (event.type == 'greet') { + send(event.actor, { + program: prog, + package: pkg, + native: native_mode + }) + } + if (event.type == 'stop') { + run_program() + } + if (event.type == 'disrupt') { + // Boot failed, run program anyway + run_program() + } + }, 'boot') } }) diff --git a/internal/shop.cm b/internal/shop.cm index 2efb2c9f..7bd58feb 100644 --- a/internal/shop.cm +++ b/internal/shop.cm @@ -21,25 +21,6 @@ var my$_ = actor_api var core = "core" -// Compiler fingerprint: hash of all compiler source files so that any compiler -// change invalidates the entire build cache. Folded into hash_path(). -var compiler_fingerprint = (function() { - var files = [ - "tokenize", "parse", "fold", "mcode", "streamline", - "qbe", "qbe_emit", "ir_stats" - ] - var combined = "" - var i = 0 - var path = null - while (i < length(files)) { - path = core_path + '/' + files[i] + '.cm' - if (fd.is_file(path)) - combined = combined + text(fd.slurp(path)) - i = i + 1 - } - return content_hash(stone(blob(combined))) -})() - // Make a package name safe for use in C identifiers. // Replaces /, ., -, @ with _ so the result is a valid C identifier fragment. function safe_c_name(name) { @@ -48,21 +29,18 @@ function safe_c_name(name) { function pull_from_cache(content) { - var path = hash_path(content) - if (fd.is_file(path)) + var path = cache_path(content) + if (fd.is_file(path)) { + log.system('shop: cache hit') return fd.slurp(path) + } } function put_into_cache(content, obj) { - var path = hash_path(content) + var path = cache_path(content) fd.slurpwrite(path, obj) -} - -function hash_path(content, salt) -{ - var s = salt || 'mach' - return global_shop_path + '/build/' + content_hash(stone(blob(text(content) + '\n' + s + '\n' + compiler_fingerprint))) + log.system('shop: cached') } var Shop = {} @@ -818,7 +796,7 @@ function resolve_mod_fn(path, pkg) { // Check for cached mcode in content-addressed store if (policy.allow_compile) { - cached_mcode_path = hash_path(content_key, 'mcode') + cached_mcode_path = cache_path(content_key, 'mcode') if (fd.is_file(cached_mcode_path)) { mcode_json = text(fd.slurp(cached_mcode_path)) compiled = mach_compile_mcode_bin(path, mcode_json) @@ -877,7 +855,7 @@ function resolve_mod_fn_bytecode(path, pkg) { if (cached) return cached // Check for cached mcode - cached_mcode_path = hash_path(content_key, 'mcode') + cached_mcode_path = cache_path(content_key, 'mcode') if (fd.is_file(cached_mcode_path)) { mcode_json = text(fd.slurp(cached_mcode_path)) compiled = mach_compile_mcode_bin(path, mcode_json) @@ -896,7 +874,7 @@ function resolve_mod_fn_bytecode(path, pkg) { mcode_json = shop_json.encode(optimized) fd.ensure_dir(global_shop_path + '/build') - fd.slurpwrite(hash_path(content_key, 'mcode'), stone(blob(mcode_json))) + fd.slurpwrite(cache_path(content_key, 'mcode'), stone(blob(mcode_json))) compiled = mach_compile_mcode_bin(path, mcode_json) put_into_cache(content_key, compiled) @@ -2170,6 +2148,7 @@ Shop.get_lib_dir = function() { Shop.ensure_dir = fd.ensure_dir Shop.install_zip = install_zip Shop.ensure_package_dylibs = ensure_package_dylibs +Shop.resolve_path = resolve_path Shop.get_local_dir = function() { return global_shop_path + "/local" @@ -2236,7 +2215,7 @@ Shop.load_as_mach = function(path, pkg) { // Try cached mcode -> compile to mach if (!compiled) { - cached_mcode_path = hash_path(content_key, 'mcode') + cached_mcode_path = cache_path(content_key, 'mcode') if (fd.is_file(cached_mcode_path)) { mcode_json = text(fd.slurp(cached_mcode_path)) compiled = mach_compile_mcode_bin(file_path, mcode_json) @@ -2256,7 +2235,7 @@ Shop.load_as_mach = function(path, pkg) { ir = _mcode_mod(ast) optimized = _streamline_mod(ir) mcode_json = shop_json.encode(optimized) - cached_mcode_path = hash_path(content_key, 'mcode') + cached_mcode_path = cache_path(content_key, 'mcode') fd.ensure_dir(global_shop_path + '/build') fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json))) compiled = mach_compile_mcode_bin(file_path, mcode_json) @@ -2309,6 +2288,34 @@ Shop.load_as_dylib = function(path, pkg) { return os.native_module_load_named(result._handle, result._sym, env) } +// Check if a .cm file has a cached bytecode artifact (mach or mcode) +Shop.is_cached = function(path) { + if (!fd.is_file(path)) return false + var content_key = stone(blob(text(fd.slurp(path)))) + if (fd.is_file(cache_path(content_key, 'mach'))) return true + if (fd.is_file(cache_path(content_key, 'mcode'))) return true + return false +} + +// Check if a .cm file has a cached native dylib artifact +Shop.is_native_cached = function(path, pkg) { + var build_mod = use_cache['core/build'] + if (!build_mod || !fd.is_file(path)) return false + var src = text(fd.slurp(path)) + var host = detect_host_target() + if (!host) return false + var san_flags = build_mod.native_sanitize_flags ? build_mod.native_sanitize_flags() : '' + var native_key = build_mod.native_cache_content ? + build_mod.native_cache_content(src, host, san_flags) : + (src + '\n' + host) + return fd.is_file(build_mod.cache_path(native_key, build_mod.SALT_NATIVE)) +} + +// Compile + cache a module without executing it +Shop.precompile = function(path, pkg) { + resolve_mod_fn(path, pkg) +} + Shop.audit_packages = function() { var packages = Shop.list_packages() diff --git a/source/cell.c b/source/cell.c index a5283855..e01c2d06 100644 --- a/source/cell.c +++ b/source/cell.c @@ -9,6 +9,7 @@ #include "cell.h" #include "cell_internal.h" +#include "quickjs-internal.h" #include "cJSON.h" #define BOOTSTRAP_MCODE "boot/bootstrap.cm.mcode" @@ -325,7 +326,7 @@ void script_startup(cell_rt *prt) JS_SetGCScanExternal(js, actor_gc_scan); prt->context = js; - /* Set per-actor heap memory limit */ + js->actor_label = prt->name; /* may be NULL; updated when name is set */ JS_SetHeapMemoryLimit(js, ACTOR_MEMORY_LIMIT); /* Register all GCRef fields so the Cheney GC can relocate them. */ diff --git a/source/cell_internal.h b/source/cell_internal.h index 533a9630..a9d5b3a0 100644 --- a/source/cell_internal.h +++ b/source/cell_internal.h @@ -29,7 +29,7 @@ typedef struct letter { #define ACTOR_FAST_TIMER_NS (10ULL * 1000000) // 10ms per turn #define ACTOR_SLOW_TIMER_NS (60000ULL * 1000000) // 60s for slow actors #define ACTOR_SLOW_STRIKES_MAX 3 // consecutive slow turns -> kill -#define ACTOR_MEMORY_LIMIT (16ULL * 1024 * 1024) // 16MB heap cap +#define ACTOR_MEMORY_LIMIT (1024ULL * 1024 * 1024) // 1GB heap cap typedef struct cell_rt { JSContext *context; diff --git a/source/qjs_actor.c b/source/qjs_actor.c index c0f11afe..b50f80ee 100644 --- a/source/qjs_actor.c +++ b/source/qjs_actor.c @@ -91,6 +91,7 @@ JSC_CCALL(actor_disrupt, JSC_SCALL(actor_setname, cell_rt *rt = JS_GetContextOpaque(js); rt->name = strdup(str); + js->actor_label = rt->name; ) JSC_CCALL(actor_on_exception, diff --git a/source/quickjs-internal.h b/source/quickjs-internal.h index e155f1cd..c9ef4afa 100644 --- a/source/quickjs-internal.h +++ b/source/quickjs-internal.h @@ -778,6 +778,7 @@ struct JSContext { uint32_t suspended_pc; /* saved PC for resume */ int vm_call_depth; /* 0 = pure bytecode, >0 = C frames on stack */ size_t heap_memory_limit; /* 0 = no limit, else max heap bytes */ + const char *actor_label; /* human-readable label for OOM diagnostics */ JSValue current_exception; diff --git a/source/runtime.c b/source/runtime.c index 94c146c6..1ea16544 100644 --- a/source/runtime.c +++ b/source/runtime.c @@ -2071,6 +2071,7 @@ JSContext *JS_NewContextRawWithHeapSize (JSRuntime *rt, size_t heap_size) { ctx->suspended_pc = 0; ctx->vm_call_depth = 0; ctx->heap_memory_limit = 0; + ctx->actor_label = NULL; JS_AddGCRef(ctx, &ctx->suspended_frame_ref); ctx->suspended_frame_ref.val = JS_NULL; @@ -3297,7 +3298,20 @@ JS_RaiseDisrupt (JSContext *ctx, const char *fmt, ...) { /* Log to "memory" channel + disrupt. Skips JS callback (can't allocate). */ JSValue JS_RaiseOOM (JSContext *ctx) { - fprintf (stderr, "out of memory\n"); + size_t used = (size_t)((uint8_t *)ctx->heap_free - (uint8_t *)ctx->heap_base); + size_t block = ctx->current_block_size; + size_t limit = ctx->heap_memory_limit; + const char *label = ctx->actor_label; + if (limit > 0) { + fprintf(stderr, "out of memory: heap %zuKB / %zuKB block, limit %zuMB", + used / 1024, block / 1024, limit / (1024 * 1024)); + } else { + fprintf(stderr, "out of memory: heap %zuKB / %zuKB block, no limit", + used / 1024, block / 1024); + } + if (label) + fprintf(stderr, " [%s]", label); + fprintf(stderr, "\n"); ctx->current_exception = JS_TRUE; return JS_EXCEPTION; }