john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge branch 'native_boot'

Browse Source
This commit is contained in:
John Alanbrook
2026-02-23 19:07:07 -06:00
parent 5ac1620b48 940807c37a
commit 7bd17c6476
20 changed files with 38708 additions and 38130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

183
internal/engine.cm
View File

@@ -35,13 +35,16 @@ var packages_path = shop_path ? shop_path + '/packages' : null
// Self-sufficient initialization: content-addressed cache
var use_cache = {}
// Save blob intrinsic before var blob = use_core('blob') shadows it.
var _make_blob = (function() { return blob })()
function content_hash(content) {
var data = content
if (!is_blob(data)) data = stone(blob(text(data)))
if (!is_blob(data)) data = stone(_make_blob(text(data)))
return text(crypto.blake2(data), 'h')
}
function cache_path(hash) {
function pipeline_cache_path(hash) {
if (!shop_path) return null
return shop_path + '/build/' + hash
}
@@ -81,7 +84,7 @@ function detect_cc() {
function native_dylib_cache_path(src, target) {
var native_key = src + '\n' + target + '\nnative\n'
var full_key = native_key + '\nnative'
return cache_path(content_hash(full_key))
return pipeline_cache_path(content_hash(full_key))
}
var _engine_host_target = null
@@ -140,9 +143,11 @@ function load_pipeline_module(name, env) {
if (fd.is_file(source_path)) {
if (!source_blob) source_blob = fd.slurp(source_path)
hash = content_hash(source_blob)
cached = cache_path(hash)
if (cached && fd.is_file(cached))
cached = pipeline_cache_path(hash)
if (cached && fd.is_file(cached)) {
log.system('engine: pipeline ' + name + ' (cached)')
return mach_load(fd.slurp(cached), env)
}
// Cache miss: compile from source using boot seed pipeline
mcode_path = core_path + '/boot/' + name + '.cm.mcode'
@@ -164,7 +169,8 @@ function load_pipeline_module(name, env) {
compiled = boot_sl(compiled)
mcode_json = json.encode(compiled)
mach_blob = mach_compile_mcode_bin(name, mcode_json)
if (cached) {
log.system('engine: pipeline ' + name + ' (compiled)')
if (!native_mode && cached) {
ensure_build_dir()
fd.slurpwrite(cached, mach_blob)
}
@@ -215,6 +221,31 @@ if (native_mode) {
use_cache['core/qbe_emit'] = _qbe_emit_mod
}
var compiler_fingerprint = (function() {
var files = [
"tokenize", "parse", "fold", "mcode", "streamline",
"qbe", "qbe_emit", "ir_stats"
]
var combined = ""
var i = 0
var path = null
while (i < length(files)) {
path = core_path + '/' + files[i] + '.cm'
if (fd.is_file(path))
combined = combined + text(fd.slurp(path))
i = i + 1
}
return content_hash(stone(blob(combined)))
})()
function module_cache_path(content, salt) {
if (!shop_path) return null
var s = salt || 'mach'
return shop_path + '/build/' + content_hash(
stone(_make_blob(text(content) + '\n' + s + '\n' + compiler_fingerprint))
)
}
// analyze: tokenize + parse + fold, check for errors
function analyze(src, filename) {
var tok_result = tokenize_mod(src, filename)
@@ -605,7 +636,6 @@ function use_core(path) {
arrfor(array(core_extras), function(k) { env[k] = core_extras[k] })
env = stone(env)
var hash = null
var cached_path = null
var mach_blob = null
var source_blob = null
@@ -644,14 +674,15 @@ function use_core(path) {
// Bytecode path (fallback or non-native mode)
_load_mod = function() {
if (!source_blob) source_blob = fd.slurp(file_path)
hash = content_hash(source_blob)
cached_path = cache_path(hash)
cached_path = module_cache_path(source_blob, 'mach')
if (cached_path && fd.is_file(cached_path)) {
log.system('engine: cache hit for core/' + path)
result = mach_load(fd.slurp(cached_path), env)
} else {
script = text(source_blob)
ast = analyze(script, file_path)
mach_blob = compile_to_blob('core:' + path, ast)
log.system('engine: compiled core/' + path)
if (!native_mode && cached_path) {
ensure_build_dir()
fd.slurpwrite(cached_path, mach_blob)
@@ -748,8 +779,10 @@ function actor_die(err)
if (underlings) {
unders = array(underlings)
arrfor(unders, function(id, index) {
log.console(`calling on ${id} to disrupt too`)
$_.stop(create_actor({id}))
if (!is_null(underlings[id])) {
log.system(`stopping underling ${id}`)
$_.stop(create_actor({id}))
}
})
}
@@ -801,7 +834,8 @@ core_extras.actor_api = $_
core_extras.log = log
core_extras.runtime_env = runtime_env
core_extras.content_hash = content_hash
core_extras.cache_path = cache_path
core_extras.cache_path = module_cache_path
core_extras.compiler_fingerprint = compiler_fingerprint
core_extras.ensure_build_dir = ensure_build_dir
core_extras.compile_to_blob = compile_to_blob
core_extras.native_mode = native_mode
@@ -1150,6 +1184,7 @@ var id_address = {}
var peer_queue = {}
var portal = null
var portal_fn = null
var enet = use_core('enet')
function peer_connection(peer) {
return {
@@ -1276,6 +1311,7 @@ $_.start = function start(cb, program) {
root_id: root ? root[ACTORDATA].id : null,
program,
native_mode: native_mode,
no_warn: _no_warn,
}
greeters[id] = cb
push(message_queue, { startup })
@@ -1321,7 +1357,7 @@ $_.couple = function couple(actor) {
if (actor == $_.self) return // can't couple to self
couplings[actor[ACTORDATA].id] = true
sys_msg(actor, {kind:'couple', from_id: _cell.id})
log.system(`coupled to ${actor}`)
log.system(`coupled to ${actor[ACTORDATA].id}`)
}
function actor_prep(actor, send) {
@@ -1385,7 +1421,7 @@ function actor_send(actor, message) {
}
return
}
log.system(`Unable to send message to actor ${actor[ACTORDATA]}`)
log.system(`Unable to send message to actor ${actor[ACTORDATA].id}`)
}
function send_messages() {
@@ -1547,7 +1583,7 @@ function handle_sysym(msg)
}
greeter(greet_msg)
}
if (msg.message.type == 'disrupt')
if (msg.message.type == 'disrupt' || msg.message.type == 'stop')
delete underlings[from_id]
} else if (msg.kind == 'contact') {
if (portal_fn) {
@@ -1756,52 +1792,81 @@ $_.clock(_ => {
env.log = log
env = stone(env)
var native_build = null
var native_dylib_path = null
var native_handle = null
var native_basename = null
var native_sym = null
// --- run_program: execute the resolved program ---
function run_program() {
var native_build = null
var native_dylib_path = null
var native_handle = null
var native_basename = null
var native_sym = null
// Native execution path: compile to dylib and run
if (native_mode) {
native_build = use_core('build')
native_dylib_path = native_build.compile_native(prog_path, null, null, pkg)
native_handle = os.dylib_open(native_dylib_path)
native_basename = file_info.name ? file_info.name + (file_info.is_actor ? '.ce' : '.cm') : fd.basename(prog_path)
native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null
if (native_sym)
os.native_module_load_named(native_handle, native_sym, env)
else
os.native_module_load(native_handle, env)
return
}
var source_blob = fd.slurp(prog_path)
var hash = content_hash(source_blob)
var cached_path = cache_path(hash)
var val = null
var script = null
var ast = null
var mach_blob = null
var _compile = function() {
if (cached_path && fd.is_file(cached_path)) {
val = mach_load(fd.slurp(cached_path), env)
} else {
script = text(source_blob)
ast = analyze(script, prog_path)
mach_blob = compile_user_blob(prog, ast, pkg)
if (cached_path) {
ensure_build_dir()
fd.slurpwrite(cached_path, mach_blob)
}
val = mach_load(mach_blob, env)
// Native execution path: compile to dylib and run
if (native_mode) {
native_build = use_core('build')
native_dylib_path = native_build.compile_native(prog_path, null, null, pkg)
native_handle = os.dylib_open(native_dylib_path)
native_basename = file_info.name ? file_info.name + (file_info.is_actor ? '.ce' : '.cm') : fd.basename(prog_path)
native_sym = pkg ? shop.c_symbol_for_file(pkg, native_basename) : null
if (native_sym)
os.native_module_load_named(native_handle, native_sym, env)
else
os.native_module_load(native_handle, env)
return
}
var source_blob = fd.slurp(prog_path)
var _cached_path = module_cache_path(source_blob, 'mach')
var val = null
var script = null
var ast = null
var mach_blob = null
var _compile = function() {
if (_cached_path && fd.is_file(_cached_path)) {
val = mach_load(fd.slurp(_cached_path), env)
} else {
script = text(source_blob)
ast = analyze(script, prog_path)
mach_blob = compile_user_blob(prog, ast, pkg)
if (_cached_path) {
ensure_build_dir()
fd.slurpwrite(_cached_path, mach_blob)
}
val = mach_load(mach_blob, env)
}
} disruption {
os.exit(1)
}
_compile()
if (val) {
log.error('Program must not return anything')
disrupt
}
} disruption {
os.exit(1)
}
_compile()
if (val) {
log.error('Program must not return anything')
disrupt
// --- Auto-boot: pre-compile uncached deps before running ---
// Only auto-boot for the root program (not child actors, not boot itself).
// Delegates all discovery + compilation to boot.ce (separate actor/memory).
var _is_root_actor = !_cell.args.overling_id
var _skip_boot = !_is_root_actor || prog == 'boot' || prog == 'compile_worker'
if (_skip_boot) {
run_program()
} else {
$_.start(function(event) {
if (event.type == 'greet') {
send(event.actor, {
program: prog,
package: pkg,
native: native_mode
})
}
if (event.type == 'stop') {
run_program()
}
if (event.type == 'disrupt') {
// Boot failed, run program anyway
run_program()
}
}, 'boot')
}
})

73
internal/shop.cm
View File

@@ -21,25 +21,6 @@ var my$_ = actor_api
var core = "core"
// Compiler fingerprint: hash of all compiler source files so that any compiler
// change invalidates the entire build cache. Folded into hash_path().
var compiler_fingerprint = (function() {
var files = [
"tokenize", "parse", "fold", "mcode", "streamline",
"qbe", "qbe_emit", "ir_stats"
]
var combined = ""
var i = 0
var path = null
while (i < length(files)) {
path = core_path + '/' + files[i] + '.cm'
if (fd.is_file(path))
combined = combined + text(fd.slurp(path))
i = i + 1
}
return content_hash(stone(blob(combined)))
})()
// Make a package name safe for use in C identifiers.
// Replaces /, ., -, @ with _ so the result is a valid C identifier fragment.
function safe_c_name(name) {
@@ -48,21 +29,18 @@ function safe_c_name(name) {
function pull_from_cache(content)
{
var path = hash_path(content)
if (fd.is_file(path))
var path = cache_path(content)
if (fd.is_file(path)) {
log.system('shop: cache hit')
return fd.slurp(path)
}
}
function put_into_cache(content, obj)
{
var path = hash_path(content)
var path = cache_path(content)
fd.slurpwrite(path, obj)
}
function hash_path(content, salt)
{
var s = salt || 'mach'
return global_shop_path + '/build/' + content_hash(stone(blob(text(content) + '\n' + s + '\n' + compiler_fingerprint)))
log.system('shop: cached')
}
var Shop = {}
@@ -818,7 +796,7 @@ function resolve_mod_fn(path, pkg) {
// Check for cached mcode in content-addressed store
if (policy.allow_compile) {
cached_mcode_path = hash_path(content_key, 'mcode')
cached_mcode_path = cache_path(content_key, 'mcode')
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
@@ -877,7 +855,7 @@ function resolve_mod_fn_bytecode(path, pkg) {
if (cached) return cached
// Check for cached mcode
cached_mcode_path = hash_path(content_key, 'mcode')
cached_mcode_path = cache_path(content_key, 'mcode')
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
@@ -896,7 +874,7 @@ function resolve_mod_fn_bytecode(path, pkg) {
mcode_json = shop_json.encode(optimized)
fd.ensure_dir(global_shop_path + '/build')
fd.slurpwrite(hash_path(content_key, 'mcode'), stone(blob(mcode_json)))
fd.slurpwrite(cache_path(content_key, 'mcode'), stone(blob(mcode_json)))
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
@@ -2170,6 +2148,7 @@ Shop.get_lib_dir = function() {
Shop.ensure_dir = fd.ensure_dir
Shop.install_zip = install_zip
Shop.ensure_package_dylibs = ensure_package_dylibs
Shop.resolve_path = resolve_path
Shop.get_local_dir = function() {
return global_shop_path + "/local"
@@ -2236,7 +2215,7 @@ Shop.load_as_mach = function(path, pkg) {
// Try cached mcode -> compile to mach
if (!compiled) {
cached_mcode_path = hash_path(content_key, 'mcode')
cached_mcode_path = cache_path(content_key, 'mcode')
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
@@ -2256,7 +2235,7 @@ Shop.load_as_mach = function(path, pkg) {
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
cached_mcode_path = hash_path(content_key, 'mcode')
cached_mcode_path = cache_path(content_key, 'mcode')
fd.ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
@@ -2309,6 +2288,34 @@ Shop.load_as_dylib = function(path, pkg) {
return os.native_module_load_named(result._handle, result._sym, env)
}
// Check if a .cm file has a cached bytecode artifact (mach or mcode)
Shop.is_cached = function(path) {
if (!fd.is_file(path)) return false
var content_key = stone(blob(text(fd.slurp(path))))
if (fd.is_file(cache_path(content_key, 'mach'))) return true
if (fd.is_file(cache_path(content_key, 'mcode'))) return true
return false
}
// Check if a .cm file has a cached native dylib artifact
Shop.is_native_cached = function(path, pkg) {
var build_mod = use_cache['core/build']
if (!build_mod || !fd.is_file(path)) return false
var src = text(fd.slurp(path))
var host = detect_host_target()
if (!host) return false
var san_flags = build_mod.native_sanitize_flags ? build_mod.native_sanitize_flags() : ''
var native_key = build_mod.native_cache_content ?
build_mod.native_cache_content(src, host, san_flags) :
(src + '\n' + host)
return fd.is_file(build_mod.cache_path(native_key, build_mod.SALT_NATIVE))
}
// Compile + cache a module without executing it
Shop.precompile = function(path, pkg) {
resolve_mod_fn(path, pkg)
}
Shop.audit_packages = function() {
var packages = Shop.list_packages()