john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

correct caching

Browse Source
This commit is contained in:
John Alanbrook
2026-02-23 13:37:11 -06:00
parent 99fb575c9c
commit d066ab03cd
7 changed files with 184 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
internal/engine.cm
View File

@@ -34,13 +34,17 @@ var packages_path = shop_path ? shop_path + '/packages' : null
// Self-sufficient initialization: content-addressed cache
var use_cache = {}
// Save blob intrinsic before var blob = use_core('blob') hoists and shadows it.
// Function declarations see the hoisted null; IIFEs see the intrinsic.
var _make_blob = (function() { return blob })()
function content_hash(content) {
var data = content
if (!is_blob(data)) data = stone(blob(text(data)))
if (!is_blob(data)) data = stone(_make_blob(text(data)))
return text(crypto.blake2(data), 'h')
}
function cache_path(hash) {
function pipeline_cache_path(hash) {
if (!shop_path) return null
return shop_path + '/build/' + hash
}
@@ -75,7 +79,7 @@ function detect_cc() {
function native_dylib_cache_path(src, target) {
var native_key = src + '\n' + target + '\nnative\n'
var full_key = native_key + '\nnative'
return cache_path(content_hash(full_key))
return pipeline_cache_path(content_hash(full_key))
}
var _engine_host_target = null
@@ -134,9 +138,11 @@ function load_pipeline_module(name, env) {
if (fd.is_file(source_path)) {
if (!source_blob) source_blob = fd.slurp(source_path)
hash = content_hash(source_blob)
cached = cache_path(hash)
if (cached && fd.is_file(cached))
cached = pipeline_cache_path(hash)
if (cached && fd.is_file(cached)) {
log.system('engine: pipeline ' + name + ' (cached)')
return mach_load(fd.slurp(cached), env)
}
// Cache miss: compile from source using boot seed pipeline
mcode_path = core_path + '/boot/' + name + '.cm.mcode'
@@ -158,6 +164,7 @@ function load_pipeline_module(name, env) {
compiled = boot_sl(compiled)
mcode_json = json.encode(compiled)
mach_blob = mach_compile_mcode_bin(name, mcode_json)
log.system('engine: pipeline ' + name + ' (compiled)')
if (!native_mode && cached) {
ensure_build_dir()
fd.slurpwrite(cached, mach_blob)
@@ -209,6 +216,31 @@ if (native_mode) {
use_cache['core/qbe_emit'] = _qbe_emit_mod
}
var compiler_fingerprint = (function() {
var files = [
"tokenize", "parse", "fold", "mcode", "streamline",
"qbe", "qbe_emit", "ir_stats"
]
var combined = ""
var i = 0
var path = null
while (i < length(files)) {
path = core_path + '/' + files[i] + '.cm'
if (fd.is_file(path))
combined = combined + text(fd.slurp(path))
i = i + 1
}
return content_hash(stone(blob(combined)))
})()
function module_cache_path(content, salt) {
if (!shop_path) return null
var s = salt || 'mach'
return shop_path + '/build/' + content_hash(
stone(_make_blob(text(content) + '\n' + s + '\n' + compiler_fingerprint))
)
}
// analyze: tokenize + parse + fold, check for errors
function analyze(src, filename) {
var tok_result = tokenize_mod(src, filename)
@@ -592,7 +624,6 @@ function use_core(path) {
arrfor(array(core_extras), function(k) { env[k] = core_extras[k] })
env = stone(env)
var hash = null
var cached_path = null
var mach_blob = null
var source_blob = null
@@ -629,14 +660,15 @@ function use_core(path) {
// Bytecode path (fallback or non-native mode)
_load_mod = function() {
if (!source_blob) source_blob = fd.slurp(file_path)
hash = content_hash(source_blob)
cached_path = cache_path(hash)
cached_path = module_cache_path(source_blob, 'mach')
if (cached_path && fd.is_file(cached_path)) {
log.system('engine: cache hit for core/' + path)
result = mach_load(fd.slurp(cached_path), env)
} else {
script = text(source_blob)
ast = analyze(script, file_path)
mach_blob = compile_to_blob('core:' + path, ast)
log.system('engine: compiled core/' + path)
if (!native_mode && cached_path) {
ensure_build_dir()
fd.slurpwrite(cached_path, mach_blob)
@@ -733,8 +765,10 @@ function actor_die(err)
if (underlings) {
unders = array(underlings)
arrfor(unders, function(id, index) {
log.console(`calling on ${id} to disrupt too`)
$_.stop(create_actor({id}))
if (!is_null(underlings[id])) {
log.system(`stopping underling ${id}`)
$_.stop(create_actor({id}))
}
})
}
@@ -786,7 +820,8 @@ core_extras.actor_api = $_
core_extras.log = log
core_extras.runtime_env = runtime_env
core_extras.content_hash = content_hash
core_extras.cache_path = cache_path
core_extras.cache_path = module_cache_path
core_extras.compiler_fingerprint = compiler_fingerprint
core_extras.ensure_build_dir = ensure_build_dir
core_extras.compile_to_blob = compile_to_blob
core_extras.native_mode = native_mode
@@ -1253,6 +1288,7 @@ $_.start = function start(cb, program) {
root_id: root ? root[ACTORDATA].id : null,
program,
native_mode: native_mode,
no_warn: _no_warn,
}
greeters[id] = cb
push(message_queue, { startup })
@@ -1300,7 +1336,7 @@ $_.couple = function couple(actor) {
if (actor == $_.self) return // can't couple to self
couplings[actor[ACTORDATA].id] = true
sys_msg(actor, {kind:'couple', from_id: _cell.id})
log.system(`coupled to ${actor}`)
log.system(`coupled to ${actor[ACTORDATA].id}`)
}
function actor_prep(actor, send) {
@@ -1363,7 +1399,7 @@ function actor_send(actor, message) {
}
return
}
log.system(`Unable to send message to actor ${actor[ACTORDATA]}`)
log.system(`Unable to send message to actor ${actor[ACTORDATA].id}`)
}
function send_messages() {
@@ -1525,7 +1561,7 @@ function handle_sysym(msg)
}
greeter(greet_msg)
}
if (msg.message.type == 'disrupt')
if (msg.message.type == 'disrupt' || msg.message.type == 'stop')
delete underlings[from_id]
} else if (msg.kind == 'contact') {
if (portal_fn) {
@@ -1549,7 +1585,7 @@ function handle_message(msg) {
var fn = null
if (msg[SYSYM]) {
handle_sysym(msg[SYSYM], msg.from)
handle_sysym(msg[SYSYM])
return
}
@@ -1734,52 +1770,81 @@ $_.clock(_ => {
env.log = log
env = stone(env)
var native_build = null
var native_dylib_path = null
var native_handle = null
var native_basename = null
var native_sym = null
// --- run_program: execute the resolved program ---
function run_program() {
var native_build = null
var native_dylib_path = null
var native_handle = null
var native_basename = null
var native_sym = null
// Native execution path: compile to dylib and run
if (native_mode) {
native_build = use_core('build')
native_dylib_path = native_build.compile_native(prog_path, null, null, pkg)
native_handle = os.dylib_open(native_dylib_path)
native_basename = file_info.name ? file_info.name + (file_info.is_actor ? '.ce' : '.cm') : fd.basename(prog_path)
native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null
if (native_sym)
os.native_module_load_named(native_handle, native_sym, env)
else
os.native_module_load(native_handle, env)
return
}
var source_blob = fd.slurp(prog_path)
var hash = content_hash(source_blob)
var cached_path = cache_path(hash)
var val = null
var script = null
var ast = null
var mach_blob = null
var _compile = function() {
if (cached_path && fd.is_file(cached_path)) {
val = mach_load(fd.slurp(cached_path), env)
} else {
script = text(source_blob)
ast = analyze(script, prog_path)
mach_blob = compile_user_blob(prog, ast, pkg)
if (cached_path) {
ensure_build_dir()
fd.slurpwrite(cached_path, mach_blob)
}
val = mach_load(mach_blob, env)
// Native execution path: compile to dylib and run
if (native_mode) {
native_build = use_core('build')
native_dylib_path = native_build.compile_native(prog_path, null, null, pkg)
native_handle = os.dylib_open(native_dylib_path)
native_basename = file_info.name ? file_info.name + (file_info.is_actor ? '.ce' : '.cm') : fd.basename(prog_path)
native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null
if (native_sym)
os.native_module_load_named(native_handle, native_sym, env)
else
os.native_module_load(native_handle, env)
return
}
var source_blob = fd.slurp(prog_path)
var _cached_path = module_cache_path(source_blob, 'mach')
var val = null
var script = null
var ast = null
var mach_blob = null
var _compile = function() {
if (_cached_path && fd.is_file(_cached_path)) {
val = mach_load(fd.slurp(_cached_path), env)
} else {
script = text(source_blob)
ast = analyze(script, prog_path)
mach_blob = compile_user_blob(prog, ast, pkg)
if (_cached_path) {
ensure_build_dir()
fd.slurpwrite(_cached_path, mach_blob)
}
val = mach_load(mach_blob, env)
}
} disruption {
os.exit(1)
}
_compile()
if (val) {
log.error('Program must not return anything')
disrupt
}
} disruption {
os.exit(1)
}
_compile()
if (val) {
log.error('Program must not return anything')
disrupt
// --- Auto-boot: pre-compile uncached deps before running ---
// Only auto-boot for the root program (not child actors, not boot itself).
// Delegates all discovery + compilation to boot.ce (separate actor/memory).
var _is_root_actor = !_cell.args.overling_id
var _skip_boot = !_is_root_actor || prog == 'boot' || prog == 'compile_worker'
if (_skip_boot) {
run_program()
} else {
$_.start(function(event) {
if (event.type == 'greet') {
send(event.actor, {
program: prog,
package: pkg,
native: native_mode
})
}
if (event.type == 'stop') {
run_program()
}
if (event.type == 'disrupt') {
// Boot failed, run program anyway
run_program()
}
}, 'boot')
}
})