john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

correct caching

Browse Source
This commit is contained in:
John Alanbrook
2026-02-23 13:37:11 -06:00
parent 99fb575c9c
commit d066ab03cd
7 changed files with 184 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
internal/engine.cm
View File

@@ -34,13 +34,17 @@ var packages_path = shop_path ? shop_path + '/packages' : null
// Self-sufficient initialization: content-addressed cache
var use_cache = {}
// Save blob intrinsic before var blob = use_core('blob') hoists and shadows it.
// Function declarations see the hoisted null; IIFEs see the intrinsic.
var _make_blob = (function() { return blob })()
function content_hash(content) {
var data = content
if (!is_blob(data)) data = stone(blob(text(data)))
if (!is_blob(data)) data = stone(_make_blob(text(data)))
return text(crypto.blake2(data), 'h')
}
function cache_path(hash) {
function pipeline_cache_path(hash) {
if (!shop_path) return null
return shop_path + '/build/' + hash
}
@@ -75,7 +79,7 @@ function detect_cc() {
function native_dylib_cache_path(src, target) {
var native_key = src + '\n' + target + '\nnative\n'
var full_key = native_key + '\nnative'
return cache_path(content_hash(full_key))
return pipeline_cache_path(content_hash(full_key))
}
var _engine_host_target = null
@@ -134,9 +138,11 @@ function load_pipeline_module(name, env) {
if (fd.is_file(source_path)) {
if (!source_blob) source_blob = fd.slurp(source_path)
hash = content_hash(source_blob)
cached = cache_path(hash)
if (cached && fd.is_file(cached))
cached = pipeline_cache_path(hash)
if (cached && fd.is_file(cached)) {
log.system('engine: pipeline ' + name + ' (cached)')
return mach_load(fd.slurp(cached), env)
}
// Cache miss: compile from source using boot seed pipeline
mcode_path = core_path + '/boot/' + name + '.cm.mcode'
@@ -158,6 +164,7 @@ function load_pipeline_module(name, env) {
compiled = boot_sl(compiled)
mcode_json = json.encode(compiled)
mach_blob = mach_compile_mcode_bin(name, mcode_json)
log.system('engine: pipeline ' + name + ' (compiled)')
if (!native_mode && cached) {
ensure_build_dir()
fd.slurpwrite(cached, mach_blob)
@@ -209,6 +216,31 @@ if (native_mode) {
use_cache['core/qbe_emit'] = _qbe_emit_mod
}
var compiler_fingerprint = (function() {
var files = [
"tokenize", "parse", "fold", "mcode", "streamline",
"qbe", "qbe_emit", "ir_stats"
]
var combined = ""
var i = 0
var path = null
while (i < length(files)) {
path = core_path + '/' + files[i] + '.cm'
if (fd.is_file(path))
combined = combined + text(fd.slurp(path))
i = i + 1
}
return content_hash(stone(blob(combined)))
})()
function module_cache_path(content, salt) {
if (!shop_path) return null
var s = salt || 'mach'
return shop_path + '/build/' + content_hash(
stone(_make_blob(text(content) + '\n' + s + '\n' + compiler_fingerprint))
)
}
// analyze: tokenize + parse + fold, check for errors
function analyze(src, filename) {
var tok_result = tokenize_mod(src, filename)
@@ -592,7 +624,6 @@ function use_core(path) {
arrfor(array(core_extras), function(k) { env[k] = core_extras[k] })
env = stone(env)
var hash = null
var cached_path = null
var mach_blob = null
var source_blob = null
@@ -629,14 +660,15 @@ function use_core(path) {
// Bytecode path (fallback or non-native mode)
_load_mod = function() {
if (!source_blob) source_blob = fd.slurp(file_path)
hash = content_hash(source_blob)
cached_path = cache_path(hash)
cached_path = module_cache_path(source_blob, 'mach')
if (cached_path && fd.is_file(cached_path)) {
log.system('engine: cache hit for core/' + path)
result = mach_load(fd.slurp(cached_path), env)
} else {
script = text(source_blob)
ast = analyze(script, file_path)
mach_blob = compile_to_blob('core:' + path, ast)
log.system('engine: compiled core/' + path)
if (!native_mode && cached_path) {
ensure_build_dir()
fd.slurpwrite(cached_path, mach_blob)
@@ -733,8 +765,10 @@ function actor_die(err)
if (underlings) {
unders = array(underlings)
arrfor(unders, function(id, index) {
log.console(`calling on ${id} to disrupt too`)
$_.stop(create_actor({id}))
if (!is_null(underlings[id])) {
log.system(`stopping underling ${id}`)
$_.stop(create_actor({id}))
}
})
}
@@ -786,7 +820,8 @@ core_extras.actor_api = $_
core_extras.log = log
core_extras.runtime_env = runtime_env
core_extras.content_hash = content_hash
core_extras.cache_path = cache_path
core_extras.cache_path = module_cache_path
core_extras.compiler_fingerprint = compiler_fingerprint
core_extras.ensure_build_dir = ensure_build_dir
core_extras.compile_to_blob = compile_to_blob
core_extras.native_mode = native_mode
@@ -1253,6 +1288,7 @@ $_.start = function start(cb, program) {
root_id: root ? root[ACTORDATA].id : null,
program,
native_mode: native_mode,
no_warn: _no_warn,
}
greeters[id] = cb
push(message_queue, { startup })
@@ -1300,7 +1336,7 @@ $_.couple = function couple(actor) {
if (actor == $_.self) return // can't couple to self
couplings[actor[ACTORDATA].id] = true
sys_msg(actor, {kind:'couple', from_id: _cell.id})
log.system(`coupled to ${actor}`)
log.system(`coupled to ${actor[ACTORDATA].id}`)
}
function actor_prep(actor, send) {
@@ -1363,7 +1399,7 @@ function actor_send(actor, message) {
}
return
}
log.system(`Unable to send message to actor ${actor[ACTORDATA]}`)
log.system(`Unable to send message to actor ${actor[ACTORDATA].id}`)
}
function send_messages() {
@@ -1525,7 +1561,7 @@ function handle_sysym(msg)
}
greeter(greet_msg)
}
if (msg.message.type == 'disrupt')
if (msg.message.type == 'disrupt' || msg.message.type == 'stop')
delete underlings[from_id]
} else if (msg.kind == 'contact') {
if (portal_fn) {
@@ -1549,7 +1585,7 @@ function handle_message(msg) {
var fn = null
if (msg[SYSYM]) {
handle_sysym(msg[SYSYM], msg.from)
handle_sysym(msg[SYSYM])
return
}
@@ -1734,52 +1770,81 @@ $_.clock(_ => {
env.log = log
env = stone(env)
var native_build = null
var native_dylib_path = null
var native_handle = null
var native_basename = null
var native_sym = null
// --- run_program: execute the resolved program ---
function run_program() {
var native_build = null
var native_dylib_path = null
var native_handle = null
var native_basename = null
var native_sym = null
// Native execution path: compile to dylib and run
if (native_mode) {
native_build = use_core('build')
native_dylib_path = native_build.compile_native(prog_path, null, null, pkg)
native_handle = os.dylib_open(native_dylib_path)
native_basename = file_info.name ? file_info.name + (file_info.is_actor ? '.ce' : '.cm') : fd.basename(prog_path)
native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null
if (native_sym)
os.native_module_load_named(native_handle, native_sym, env)
else
os.native_module_load(native_handle, env)
return
}
var source_blob = fd.slurp(prog_path)
var hash = content_hash(source_blob)
var cached_path = cache_path(hash)
var val = null
var script = null
var ast = null
var mach_blob = null
var _compile = function() {
if (cached_path && fd.is_file(cached_path)) {
val = mach_load(fd.slurp(cached_path), env)
} else {
script = text(source_blob)
ast = analyze(script, prog_path)
mach_blob = compile_user_blob(prog, ast, pkg)
if (cached_path) {
ensure_build_dir()
fd.slurpwrite(cached_path, mach_blob)
}
val = mach_load(mach_blob, env)
// Native execution path: compile to dylib and run
if (native_mode) {
native_build = use_core('build')
native_dylib_path = native_build.compile_native(prog_path, null, null, pkg)
native_handle = os.dylib_open(native_dylib_path)
native_basename = file_info.name ? file_info.name + (file_info.is_actor ? '.ce' : '.cm') : fd.basename(prog_path)
native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null
if (native_sym)
os.native_module_load_named(native_handle, native_sym, env)
else
os.native_module_load(native_handle, env)
return
}
var source_blob = fd.slurp(prog_path)
var _cached_path = module_cache_path(source_blob, 'mach')
var val = null
var script = null
var ast = null
var mach_blob = null
var _compile = function() {
if (_cached_path && fd.is_file(_cached_path)) {
val = mach_load(fd.slurp(_cached_path), env)
} else {
script = text(source_blob)
ast = analyze(script, prog_path)
mach_blob = compile_user_blob(prog, ast, pkg)
if (_cached_path) {
ensure_build_dir()
fd.slurpwrite(_cached_path, mach_blob)
}
val = mach_load(mach_blob, env)
}
} disruption {
os.exit(1)
}
_compile()
if (val) {
log.error('Program must not return anything')
disrupt
}
} disruption {
os.exit(1)
}
_compile()
if (val) {
log.error('Program must not return anything')
disrupt
// --- Auto-boot: pre-compile uncached deps before running ---
// Only auto-boot for the root program (not child actors, not boot itself).
// Delegates all discovery + compilation to boot.ce (separate actor/memory).
var _is_root_actor = !_cell.args.overling_id
var _skip_boot = !_is_root_actor || prog == 'boot' || prog == 'compile_worker'
if (_skip_boot) {
run_program()
} else {
$_.start(function(event) {
if (event.type == 'greet') {
send(event.actor, {
program: prog,
package: pkg,
native: native_mode
})
}
if (event.type == 'stop') {
run_program()
}
if (event.type == 'disrupt') {
// Boot failed, run program anyway
run_program()
}
}, 'boot')
}
})

73
internal/shop.cm
View File

@@ -21,25 +21,6 @@ var my$_ = actor_api
var core = "core"
// Compiler fingerprint: hash of all compiler source files so that any compiler
// change invalidates the entire build cache. Folded into hash_path().
var compiler_fingerprint = (function() {
var files = [
"tokenize", "parse", "fold", "mcode", "streamline",
"qbe", "qbe_emit", "ir_stats"
]
var combined = ""
var i = 0
var path = null
while (i < length(files)) {
path = core_path + '/' + files[i] + '.cm'
if (fd.is_file(path))
combined = combined + text(fd.slurp(path))
i = i + 1
}
return content_hash(stone(blob(combined)))
})()
// Make a package name safe for use in C identifiers.
// Replaces /, ., -, @ with _ so the result is a valid C identifier fragment.
function safe_c_name(name) {
@@ -48,21 +29,18 @@ function safe_c_name(name) {
function pull_from_cache(content)
{
var path = hash_path(content)
if (fd.is_file(path))
var path = cache_path(content)
if (fd.is_file(path)) {
log.system('shop: cache hit')
return fd.slurp(path)
}
}
function put_into_cache(content, obj)
{
var path = hash_path(content)
var path = cache_path(content)
fd.slurpwrite(path, obj)
}
function hash_path(content, salt)
{
var s = salt || 'mach'
return global_shop_path + '/build/' + content_hash(stone(blob(text(content) + '\n' + s + '\n' + compiler_fingerprint)))
log.system('shop: cached')
}
var Shop = {}
@@ -818,7 +796,7 @@ function resolve_mod_fn(path, pkg) {
// Check for cached mcode in content-addressed store
if (policy.allow_compile) {
cached_mcode_path = hash_path(content_key, 'mcode')
cached_mcode_path = cache_path(content_key, 'mcode')
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
@@ -877,7 +855,7 @@ function resolve_mod_fn_bytecode(path, pkg) {
if (cached) return cached
// Check for cached mcode
cached_mcode_path = hash_path(content_key, 'mcode')
cached_mcode_path = cache_path(content_key, 'mcode')
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
@@ -896,7 +874,7 @@ function resolve_mod_fn_bytecode(path, pkg) {
mcode_json = shop_json.encode(optimized)
fd.ensure_dir(global_shop_path + '/build')
fd.slurpwrite(hash_path(content_key, 'mcode'), stone(blob(mcode_json)))
fd.slurpwrite(cache_path(content_key, 'mcode'), stone(blob(mcode_json)))
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
@@ -2170,6 +2148,7 @@ Shop.get_lib_dir = function() {
Shop.ensure_dir = fd.ensure_dir
Shop.install_zip = install_zip
Shop.ensure_package_dylibs = ensure_package_dylibs
Shop.resolve_path = resolve_path
Shop.get_local_dir = function() {
return global_shop_path + "/local"
@@ -2236,7 +2215,7 @@ Shop.load_as_mach = function(path, pkg) {
// Try cached mcode -> compile to mach
if (!compiled) {
cached_mcode_path = hash_path(content_key, 'mcode')
cached_mcode_path = cache_path(content_key, 'mcode')
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
@@ -2256,7 +2235,7 @@ Shop.load_as_mach = function(path, pkg) {
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
cached_mcode_path = hash_path(content_key, 'mcode')
cached_mcode_path = cache_path(content_key, 'mcode')
fd.ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
@@ -2309,6 +2288,34 @@ Shop.load_as_dylib = function(path, pkg) {
return os.native_module_load_named(result._handle, result._sym, env)
}
// Check if a .cm file has a cached bytecode artifact (mach or mcode)
Shop.is_cached = function(path) {
if (!fd.is_file(path)) return false
var content_key = stone(blob(text(fd.slurp(path))))
if (fd.is_file(cache_path(content_key, 'mach'))) return true
if (fd.is_file(cache_path(content_key, 'mcode'))) return true
return false
}
// Check if a .cm file has a cached native dylib artifact
Shop.is_native_cached = function(path, pkg) {
var build_mod = use_cache['core/build']
if (!build_mod || !fd.is_file(path)) return false
var src = text(fd.slurp(path))
var host = detect_host_target()
if (!host) return false
var san_flags = build_mod.native_sanitize_flags ? build_mod.native_sanitize_flags() : ''
var native_key = build_mod.native_cache_content ?
build_mod.native_cache_content(src, host, san_flags) :
(src + '\n' + host)
return fd.is_file(build_mod.cache_path(native_key, build_mod.SALT_NATIVE))
}
// Compile + cache a module without executing it
Shop.precompile = function(path, pkg) {
resolve_mod_fn(path, pkg)
}
Shop.audit_packages = function() {
var packages = Shop.list_packages()

3
source/cell.c
View File

@@ -9,6 +9,7 @@
#include "cell.h"
#include "cell_internal.h"
#include "quickjs-internal.h"
#include "cJSON.h"
#define BOOTSTRAP_MCODE "boot/bootstrap.cm.mcode"
@@ -325,7 +326,7 @@ void script_startup(cell_rt *prt)
JS_SetGCScanExternal(js, actor_gc_scan);
prt->context = js;
/* Set per-actor heap memory limit */
js->actor_label = prt->name; /* may be NULL; updated when name is set */
JS_SetHeapMemoryLimit(js, ACTOR_MEMORY_LIMIT);
/* Register all GCRef fields so the Cheney GC can relocate them. */

2
source/cell_internal.h
View File

@@ -29,7 +29,7 @@ typedef struct letter {
#define ACTOR_FAST_TIMER_NS (10ULL * 1000000) // 10ms per turn
#define ACTOR_SLOW_TIMER_NS (60000ULL * 1000000) // 60s for slow actors
#define ACTOR_SLOW_STRIKES_MAX 3 // consecutive slow turns -> kill
#define ACTOR_MEMORY_LIMIT (16ULL * 1024 * 1024) // 16MB heap cap
#define ACTOR_MEMORY_LIMIT (1024ULL * 1024 * 1024) // 1GB heap cap
typedef struct cell_rt {
JSContext *context;

1
source/qjs_actor.c
View File

@@ -91,6 +91,7 @@ JSC_CCALL(actor_disrupt,
JSC_SCALL(actor_setname,
cell_rt *rt = JS_GetContextOpaque(js);
rt->name = strdup(str);
js->actor_label = rt->name;
)
JSC_CCALL(actor_on_exception,

1
source/quickjs-internal.h
View File

@@ -778,6 +778,7 @@ struct JSContext {
uint32_t suspended_pc; /* saved PC for resume */
int vm_call_depth; /* 0 = pure bytecode, >0 = C frames on stack */
size_t heap_memory_limit; /* 0 = no limit, else max heap bytes */
const char *actor_label; /* human-readable label for OOM diagnostics */
JSValue current_exception;

16
source/runtime.c
View File

@@ -2071,6 +2071,7 @@ JSContext *JS_NewContextRawWithHeapSize (JSRuntime *rt, size_t heap_size) {
ctx->suspended_pc = 0;
ctx->vm_call_depth = 0;
ctx->heap_memory_limit = 0;
ctx->actor_label = NULL;
JS_AddGCRef(ctx, &ctx->suspended_frame_ref);
ctx->suspended_frame_ref.val = JS_NULL;
@@ -3297,7 +3298,20 @@ JS_RaiseDisrupt (JSContext *ctx, const char *fmt, ...) {
/* Log to "memory" channel + disrupt. Skips JS callback (can't allocate). */
JSValue JS_RaiseOOM (JSContext *ctx) {
fprintf (stderr, "out of memory\n");
size_t used = (size_t)((uint8_t *)ctx->heap_free - (uint8_t *)ctx->heap_base);
size_t block = ctx->current_block_size;
size_t limit = ctx->heap_memory_limit;
const char *label = ctx->actor_label;
if (limit > 0) {
fprintf(stderr, "out of memory: heap %zuKB / %zuKB block, limit %zuMB",
used / 1024, block / 1024, limit / (1024 * 1024));
} else {
fprintf(stderr, "out of memory: heap %zuKB / %zuKB block, no limit",
used / 1024, block / 1024);
}
if (label)
fprintf(stderr, " [%s]", label);
fprintf(stderr, "\n");
ctx->current_exception = JS_TRUE;
return JS_EXCEPTION;
}