john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

mach loading

Browse Source
This commit is contained in:
John Alanbrook
2026-02-13 07:26:49 -06:00
parent 4f18a0b524
commit 77fa058135
11 changed files with 7940 additions and 8637 deletions
Download Patch File Download Diff File Expand all files Collapse all files

365
build.cm
View File

@@ -214,36 +214,22 @@ function compute_link_key(objects, ldflags, target_ldflags, target, cc) {
return content_hash(text(parts, '\n'))
}
// Build a dynamic library for a package
// Output goes to .cell/lib/<package_name>.<ext>
// Dynamic libraries do NOT link against core; undefined symbols are resolved at dlopen time
// Uses content-addressed store + symlink for caching
Build.build_dynamic = function(pkg, target = Build.detect_host_target(), buildtype = 'release') {
var objects = Build.build_package(pkg, target, true, buildtype) // exclude main.c
// Build a per-module dynamic library for a single C file
// Returns the content-addressed dylib path in .cell/build/<hash>.<target>.dylib
Build.build_module_dylib = function(pkg, file, target, buildtype) {
var _target = target || Build.detect_host_target()
var _buildtype = buildtype || 'release'
var obj = Build.compile_file(pkg, file, _target, _buildtype)
if (length(objects) == 0) {
log.console('No C files in ' + pkg)
return null
}
var lib_dir = shop.get_lib_dir()
var store_dir = lib_dir + '/store'
ensure_dir(lib_dir)
ensure_dir(store_dir)
var lib_name = shop.lib_name_for_package(pkg)
var dylib_ext = toolchains[target].system == 'windows' ? '.dll' : (toolchains[target].system == 'darwin' ? '.dylib' : '.so')
var stable_path = lib_dir + '/' + lib_name + dylib_ext
// Get link flags (with sigil replacement)
var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', target))
var target_ldflags = toolchains[target].c_link_args || []
var cc = toolchains[target].cpp || toolchains[target].c
var pkg_dir = shop.get_package_dir(pkg)
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.cpp || tc.c
var local_dir = get_local_dir()
var tc = toolchains[target]
var pkg_dir = shop.get_package_dir(pkg)
// Resolve relative -L paths in ldflags for hash computation
// Get link flags
var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target))
var target_ldflags = tc.c_link_args || []
var resolved_ldflags = []
arrfor(ldflags, function(flag) {
if (starts_with(flag, '-L') && !starts_with(flag, '-L/')) {
@@ -252,36 +238,21 @@ Build.build_dynamic = function(pkg, target = Build.detect_host_target(), buildty
push(resolved_ldflags, flag)
})
// Compute link key
var link_key = compute_link_key(objects, resolved_ldflags, target_ldflags, target, cc)
var store_path = store_dir + '/' + lib_name + '-' + link_key + dylib_ext
// Content-addressed output: hash of (object + link flags + target)
var link_key = compute_link_key([obj], resolved_ldflags, target_ldflags, _target, cc)
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + link_key + '.' + _target + dylib_ext
// Check if already linked in store
if (fd.is_file(store_path)) {
// Ensure symlink points to the store file
if (fd.is_link(stable_path)) {
var current_target = fd.readlink(stable_path)
if (current_target == store_path) {
// Already up to date
return stable_path
}
fd.unlink(stable_path)
} else if (fd.is_file(stable_path)) {
fd.unlink(stable_path)
}
fd.symlink(store_path, stable_path)
return stable_path
}
if (fd.is_file(dylib_path))
return dylib_path
// Build link command
var cmd_parts = [cc, '-shared', '-fPIC']
// Platform-specific flags for undefined symbols (resolved at dlopen) and size optimization
if (tc.system == 'darwin') {
cmd_parts = array(cmd_parts, [
'-undefined', 'dynamic_lookup',
'-Wl,-dead_strip',
'-Wl,-install_name,' + stable_path,
'-Wl,-rpath,@loader_path/../local',
'-Wl,-rpath,' + local_dir
])
@@ -293,41 +264,54 @@ Build.build_dynamic = function(pkg, target = Build.detect_host_target(), buildty
'-Wl,-rpath,' + local_dir
])
} else if (tc.system == 'windows') {
// Windows DLLs: use --allow-shlib-undefined for mingw
push(cmd_parts, '-Wl,--allow-shlib-undefined')
}
// Add .cell/local to library search path
push(cmd_parts, '-L"' + local_dir + '"')
arrfor(objects, function(obj) {
push(cmd_parts, '"' + obj + '"')
})
// Do NOT link against core library - symbols resolved at dlopen time
push(cmd_parts, '"' + obj + '"')
cmd_parts = array(cmd_parts, resolved_ldflags)
cmd_parts = array(cmd_parts, target_ldflags)
push(cmd_parts, '-o')
push(cmd_parts, '"' + store_path + '"')
push(cmd_parts, '"' + dylib_path + '"')
var cmd_str = text(cmd_parts, ' ')
log.console('Linking ' + lib_name + dylib_ext)
log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path))
var ret = os.system(cmd_str)
if (ret != 0) {
throw Error('Linking failed: ' + pkg)
print('Linking failed: ' + file); disrupt
}
// Update symlink to point to the new store file
if (fd.is_link(stable_path)) {
fd.unlink(stable_path)
} else if (fd.is_file(stable_path)) {
fd.unlink(stable_path)
}
fd.symlink(store_path, stable_path)
return dylib_path
}
return stable_path
// Build a dynamic library for a package (one dylib per C file)
// Returns array of {file, symbol, dylib} for each module
// Also writes a manifest mapping symbols to dylib paths
Build.build_dynamic = function(pkg, target, buildtype) {
var _target = target || Build.detect_host_target()
var _buildtype = buildtype || 'release'
var c_files = pkg_tools.get_c_files(pkg, _target, true)
var results = []
var manifest = {}
arrfor(c_files, function(file) {
var sym_name = shop.c_symbol_for_file(pkg, file)
var dylib = Build.build_module_dylib(pkg, file, _target, _buildtype)
push(results, {file: file, symbol: sym_name, dylib: dylib})
manifest[sym_name] = dylib
})
// Write manifest so the loader can find per-module dylibs by symbol
if (length(results) > 0) {
var lib_dir = shop.get_lib_dir()
ensure_dir(lib_dir)
var lib_name = shop.lib_name_for_package(pkg)
var manifest_path = lib_dir + '/' + lib_name + '.manifest.json'
var json = use('json')
fd.slurpwrite(manifest_path, stone(blob(json.encode(manifest))))
}
return results
}
// ============================================================================
@@ -412,43 +396,236 @@ Build.build_static = function(packages, target = Build.detect_host_target(), out
return output
}
// ============================================================================
// Native .cm compilation (source → mcode → QBE IL → .o → .dylib)
// ============================================================================
// Post-process QBE IL: insert dead labels after ret/jmp (QBE requirement)
function qbe_insert_dead_labels(il_text) {
var lines = array(il_text, "\n")
var result = []
var dead_id = 0
var need_label = false
var i = 0
var line = null
var trimmed = null
while (i < length(lines)) {
line = lines[i]
trimmed = trim(line)
if (need_label && !starts_with(trimmed, '@') && !starts_with(trimmed, '}') && length(trimmed) > 0) {
push(result, "@_dead_" + text(dead_id))
dead_id = dead_id + 1
need_label = false
}
if (starts_with(trimmed, '@') || starts_with(trimmed, '}') || length(trimmed) == 0) {
need_label = false
}
if (starts_with(trimmed, 'ret ') || starts_with(trimmed, 'jmp ')) {
need_label = true
}
push(result, line)
i = i + 1
}
return text(result, "\n")
}
// Compile a .cm source file to a native .dylib via QBE
// Returns the content-addressed dylib path
Build.compile_native = function(src_path, target, buildtype) {
var _target = target || Build.detect_host_target()
var _buildtype = buildtype || 'release'
if (!fd.is_file(src_path)) {
print('Source file not found: ' + src_path); disrupt
}
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.c
// Step 1: Read source and compile through pipeline
var content = fd.slurp(src_path)
var src = text(content)
var tokenize = use('tokenize')
var parse = use('parse')
var fold = use('fold')
var mcode_mod = use('mcode')
var streamline_mod = use('streamline')
var qbe_macros = use('qbe')
var qbe_emit = use('qbe_emit')
var tok_result = tokenize(src, src_path)
var ast = parse(tok_result.tokens, src, src_path, tokenize)
var folded = fold(ast)
var compiled = mcode_mod(folded)
var optimized = streamline_mod(compiled)
// Step 2: Generate QBE IL
var il = qbe_emit(optimized, qbe_macros)
// Step 3: Post-process (insert dead labels)
il = qbe_insert_dead_labels(il)
// Content hash for cache key
var hash = content_hash(src + '\n' + _target + '\nnative')
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
if (fd.is_file(dylib_path))
return dylib_path
// Step 4: Write QBE IL to temp file
var tmp = '/tmp/cell_native_' + hash
var ssa_path = tmp + '.ssa'
var s_path = tmp + '.s'
var o_path = tmp + '.o'
var rt_o_path = '/tmp/cell_qbe_rt.o'
fd.slurpwrite(ssa_path, stone(blob(il)))
// Step 5: QBE compile to assembly
var rc = os.system('qbe -o ' + s_path + ' ' + ssa_path)
if (rc != 0) {
print('QBE compilation failed for: ' + src_path); disrupt
}
// Step 6: Assemble
rc = os.system(cc + ' -c ' + s_path + ' -o ' + o_path)
if (rc != 0) {
print('Assembly failed for: ' + src_path); disrupt
}
// Step 7: Compile QBE runtime stubs if needed
if (!fd.is_file(rt_o_path)) {
var qbe_rt_path = shop.get_package_dir('core') + '/qbe_rt.c'
rc = os.system(cc + ' -c ' + qbe_rt_path + ' -o ' + rt_o_path + ' -fPIC')
if (rc != 0) {
print('QBE runtime stubs compilation failed'); disrupt
}
}
// Step 8: Link dylib
var link_cmd = cc + ' -shared -fPIC'
if (tc.system == 'darwin') {
link_cmd = link_cmd + ' -undefined dynamic_lookup'
} else if (tc.system == 'linux') {
link_cmd = link_cmd + ' -Wl,--allow-shlib-undefined'
}
link_cmd = link_cmd + ' ' + o_path + ' ' + rt_o_path + ' -o ' + dylib_path
rc = os.system(link_cmd)
if (rc != 0) {
print('Linking native dylib failed for: ' + src_path); disrupt
}
log.console('Built native: ' + fd.basename(dylib_path))
return dylib_path
}
// ============================================================================
// Module table generation (for static builds)
// ============================================================================
// Compile a .cm module to mach bytecode blob
// Returns the raw mach bytes as a blob
Build.compile_cm_to_mach = function(src_path) {
if (!fd.is_file(src_path)) {
print('Source file not found: ' + src_path); disrupt
}
var src = text(fd.slurp(src_path))
var tokenize = use('tokenize')
var parse = use('parse')
var fold = use('fold')
var mcode_mod = use('mcode')
var streamline_mod = use('streamline')
var json = use('json')
var tok_result = tokenize(src, src_path)
var ast = parse(tok_result.tokens, src, src_path, tokenize)
var folded = fold(ast)
var compiled = mcode_mod(folded)
var optimized = streamline_mod(compiled)
return mach_compile_mcode_bin(src_path, json.encode(optimized))
}
// Generate a module_table.c file that embeds mach bytecode for .cm modules
// modules: array of {name, src_path} — name is the module name, src_path is the .cm file
// output: path to write the generated .c file
Build.generate_module_table = function(modules, output) {
var lines = []
var json = use('json')
push(lines, '/* Generated module table — do not edit */')
push(lines, '#include <stddef.h>')
push(lines, '#include <string.h>')
push(lines, '')
push(lines, 'struct cell_embedded_entry {')
push(lines, ' const char *name;')
push(lines, ' const unsigned char *data;')
push(lines, ' size_t size;')
push(lines, '};')
push(lines, '')
var entries = []
arrfor(modules, function(mod) {
var safe = replace(replace(replace(mod.name, '/', '_'), '.', '_'), '-', '_')
var mach = Build.compile_cm_to_mach(mod.src_path)
var bytes = array(mach)
var hex = []
arrfor(bytes, function(b) {
push(hex, '0x' + text(b, 'h2'))
})
push(lines, 'static const unsigned char mod_' + safe + '_data[] = {')
push(lines, ' ' + text(hex, ', '))
push(lines, '};')
push(lines, '')
push(entries, safe)
log.console('Embedded: ' + mod.name + ' (' + text(length(bytes)) + ' bytes)')
})
// Lookup function
push(lines, 'const struct cell_embedded_entry *cell_embedded_module_lookup(const char *name) {')
arrfor(modules, function(mod, i) {
var safe = entries[i]
push(lines, ' if (strcmp(name, "' + mod.name + '") == 0) {')
push(lines, ' static const struct cell_embedded_entry e = {"' + mod.name + '", mod_' + safe + '_data, sizeof(mod_' + safe + '_data)};')
push(lines, ' return &e;')
push(lines, ' }')
})
push(lines, ' return (void *)0;')
push(lines, '}')
var c_text = text(lines, '\n')
fd.slurpwrite(output, stone(blob(c_text)))
log.console('Generated ' + output)
return output
}
// ============================================================================
// Convenience functions
// ============================================================================
// Build dynamic libraries for all installed packages
Build.build_all_dynamic = function(target, buildtype = 'release') {
target = target || Build.detect_host_target()
Build.build_all_dynamic = function(target, buildtype) {
var _target = target || Build.detect_host_target()
var _buildtype = buildtype || 'release'
var packages = shop.list_packages()
var results = []
// Build core first
if (find(packages, 'core') != null) {
try {
var lib = Build.build_dynamic('core', target, buildtype)
push(results, { package: 'core', library: lib })
} catch (e) {
log.error('Failed to build core: ' + text(e))
push(results, { package: 'core', error: e })
}
if (find(packages, function(p) { return p == 'core' }) != null) {
var core_mods = Build.build_dynamic('core', _target, _buildtype)
push(results, {package: 'core', modules: core_mods})
}
// Build other packages
arrfor(packages, function(pkg) {
if (pkg == 'core') return
try {
var lib = Build.build_dynamic(pkg, target, buildtype)
push(results, { package: pkg, library: lib })
} catch (e) {
log.error('Failed to build ' + pkg + ': ')
log.console(e.message)
log.console(e.stack)
push(results, { package: pkg, error: e })
}
var pkg_mods = Build.build_dynamic(pkg, _target, _buildtype)
push(results, {package: pkg, modules: pkg_mods})
})
return results
}

2
fd.cm
View File

@@ -1,4 +1,4 @@
var fd = native
var fd = use('internal/fd_c')
var wildstar = use('wildstar')
function last_pos(str, sep) {

94
internal/bootstrap.cm
View File

@@ -65,6 +65,7 @@ function check_mach_stale() {
"streamline.cm",
"qbe.cm",
"qbe_emit.cm",
"verify_ir.cm",
"internal/bootstrap.cm",
"internal/engine.cm"
]
@@ -140,38 +141,11 @@ function analyze(src, filename) {
return ast
}
// Load a module from .mach/.mcode bytecode, falling back to source compilation
function load_module(name, env) {
var mach_path = core_path + '/' + name + ".cm.mach"
var mcode_path = core_path + '/' + name + ".cm.mcode"
var data = null
var mcode_json = null
var src_path = null
var src = null
var ast = null
var compiled = null
var optimized = null
if (fd.is_file(mach_path)) {
data = fd.slurp(mach_path)
return mach_load(data, env)
}
if (fd.is_file(mcode_path)) {
mcode_json = text(fd.slurp(mcode_path))
return mach_eval_mcode(name, mcode_json, env)
}
src_path = core_path + '/' + name + ".cm"
src = text(fd.slurp(src_path))
ast = analyze(src, src_path)
compiled = mcode_mod(ast)
optimized = streamline_mod(compiled)
return mach_eval_mcode(name, json.encode(optimized), env)
}
// Load optimization pipeline modules (needs analyze to be defined)
streamline_mod = load_module("streamline", boot_env)
streamline_mod = boot_load("streamline", boot_env)
use_cache['streamline'] = streamline_mod
// Lazy-loaded verify_ir module (loaded on first use via use_fn)
// Lazy-loaded verify_ir module (loaded on first use)
var _verify_ir_mod = null
// Run AST through mcode pipeline → register VM
@@ -179,7 +153,7 @@ function run_ast(name, ast, env) {
var compiled = mcode_mod(ast)
if (os._verify_ir) {
if (_verify_ir_mod == null) {
_verify_ir_mod = use_fn('verify_ir')
_verify_ir_mod = boot_load('verify_ir', boot_env)
}
compiled._verify = true
compiled._verify_mod = _verify_ir_mod
@@ -199,60 +173,6 @@ function run_ast_noopt(name, ast, env) {
return mach_eval_mcode(name, json.encode(compiled), env)
}
// use() with ƿit pipeline for .cm modules
function use_fn(path) {
var file_path = null
var mach_path = null
var mcode_path = null
var mcode_json = null
var data = null
var script = null
var ast = null
var result = null
if (use_cache[path])
return use_cache[path]
// Try .cm.mach bytecode first (CWD then core_path)
mach_path = path + '.cm.mach'
if (!fd.is_file(mach_path))
mach_path = core_path + '/' + path + '.cm.mach'
if (fd.is_file(mach_path)) {
data = fd.slurp(mach_path)
result = mach_load(data, {use: use_fn})
use_cache[path] = result
return result
}
// Try .cm.mcode JSON IR (CWD then core_path)
mcode_path = path + '.cm.mcode'
if (!fd.is_file(mcode_path))
mcode_path = core_path + '/' + path + '.cm.mcode'
if (fd.is_file(mcode_path)) {
mcode_json = text(fd.slurp(mcode_path))
result = mach_eval_mcode(path, mcode_json, {use: use_fn})
use_cache[path] = result
return result
}
// Try .cm source (CWD then core_path)
file_path = path + '.cm'
if (!fd.is_file(file_path))
file_path = core_path + '/' + path + '.cm'
if (fd.is_file(file_path)) {
script = text(fd.slurp(file_path))
ast = analyze(script, file_path)
result = run_ast(path, ast, {use: use_fn})
use_cache[path] = result
return result
}
// Fallback to embedded C module
result = use_embed(replace(path, '/', '_'))
use_cache[path] = result
return result
}
// Helper to load engine.cm and run it with given env
function load_engine(env) {
var engine_path = core_path + '/internal/engine.cm.mach'
@@ -298,13 +218,15 @@ if (args != null) {
os: os, actorsym: actorsym,
init: {program: program, arg: user_args},
core_path: core_path, shop_path: shop_path, json: json,
analyze: analyze, run_ast_fn: run_ast, run_ast_noopt_fn: run_ast_noopt
analyze: analyze, run_ast_fn: run_ast, run_ast_noopt_fn: run_ast_noopt,
use_cache: use_cache
})
} else {
// Actor spawn mode — load engine.cm with full actor env
load_engine({
os: os, actorsym: actorsym, init: init,
core_path: core_path, shop_path: shop_path, json: json, nota: nota, wota: wota,
analyze: analyze, run_ast_fn: run_ast, run_ast_noopt_fn: run_ast_noopt
analyze: analyze, run_ast_fn: run_ast, run_ast_noopt_fn: run_ast_noopt,
use_cache: use_cache
})
}

4166
internal/bootstrap.cm.mcode
View File

File diff suppressed because it is too large Load Diff

14
internal/engine.cm
View File

@@ -1,4 +1,4 @@
// Hidden vars (os, actorsym, init, core_path, shop_path, analyze, run_ast_fn, run_ast_noopt_fn, json) come from env
// Hidden vars (os, actorsym, init, core_path, shop_path, analyze, run_ast_fn, run_ast_noopt_fn, json, use_cache) come from env
// In actor spawn mode, also: nota, wota
var ACTORDATA = actorsym
var SYSYM = '__SYSTEM__'
@@ -53,7 +53,6 @@ var js = use_embed('js')
// shop_path may be null if --core was used without --shop
var packages_path = shop_path ? shop_path + '/packages' : null
var use_cache = {}
use_cache['core/os'] = os
// Extra env properties added as engine initializes (log, runtime fns, etc.)
@@ -70,11 +69,18 @@ function use_core(path) {
var result = null
var script = null
var ast = null
var c_cache_key = null
// Build env: merge core_extras, include C embed as 'native' if available
// If C embed exists, register it so .cm modules can use('internal/<name>_c')
if (sym) {
c_cache_key = 'core/internal/' + path + '_c'
if (!use_cache[c_cache_key])
use_cache[c_cache_key] = sym
}
// Build env: merge core_extras
env = {use: use_core}
arrfor(array(core_extras), function(k) { env[k] = core_extras[k] })
if (sym) env.native = sym
// Check for pre-compiled .cm.mach file first
var mach_path = core_path + '/' + path + '.cm.mach'

11627
internal/engine.cm.mcode
View File

File diff suppressed because it is too large Load Diff

49
internal/os.c
View File

@@ -425,6 +425,22 @@ static JSValue js_os_dylib_has_symbol(JSContext *js, JSValue self, int argc, JSV
return JS_NewBool(js, symbol != NULL);
}
/* Load a native .cm module from a dylib handle.
Uses cell_rt_native_module_load from qbe_helpers.c */
extern JSValue cell_rt_native_module_load(JSContext *ctx, void *dl_handle);
static JSValue js_os_native_module_load(JSContext *js, JSValue self, int argc, JSValue *argv)
{
if (argc < 1)
return JS_ThrowTypeError(js, "native_module_load requires a dylib object");
void *handle = JS_GetOpaque(argv[0], js_dylib_class_id);
if (!handle)
return JS_ThrowTypeError(js, "First argument must be a dylib object");
return cell_rt_native_module_load(js, handle);
}
JSC_CCALL(os_print,
size_t len;
const char *str = JS_ToCStringLen(js, &len, argv[0]);
@@ -552,6 +568,37 @@ JSC_CCALL(os_getenv,
ret = JS_NULL;
)
/* --- Embedded module table (generated for static builds) ---
Uses dlsym to check if cell_embedded_module_lookup exists at runtime.
When linking a static build with a generated module_table.c, the symbol
will be found; in dynamic builds it returns NULL gracefully. */
struct cell_embedded_entry {
const char *name;
const unsigned char *data;
size_t size;
};
typedef const struct cell_embedded_entry *(*cell_embed_lookup_fn)(const char *);
static JSValue js_os_embedded_module(JSContext *js, JSValue self, int argc, JSValue *argv)
{
cell_embed_lookup_fn lookup = (cell_embed_lookup_fn)dlsym(RTLD_DEFAULT, "cell_embedded_module_lookup");
if (!lookup)
return JS_NULL;
const char *name = JS_ToCString(js, argv[0]);
if (!name) return JS_NULL;
const struct cell_embedded_entry *entry = lookup(name);
JS_FreeCString(js, name);
if (!entry) return JS_NULL;
/* Return the mach blob as a stoned blob */
return js_new_blob_stoned_copy(js, (void *)entry->data, entry->size);
}
static const JSCFunctionListEntry js_os_funcs[] = {
MIST_FUNC_DEF(os, platform, 0),
MIST_FUNC_DEF(os, arch, 0),
@@ -568,6 +615,8 @@ static const JSCFunctionListEntry js_os_funcs[] = {
MIST_FUNC_DEF(os, dylib_open, 1),
MIST_FUNC_DEF(os, dylib_symbol, 2),
MIST_FUNC_DEF(os, dylib_has_symbol, 2),
MIST_FUNC_DEF(os, native_module_load, 1),
MIST_FUNC_DEF(os, embedded_module, 1),
MIST_FUNC_DEF(os, load_internal, 1),
MIST_FUNC_DEF(os, internal_exists, 1),
MIST_FUNC_DEF(os, print, 1),

177
internal/shop.cm
View File

@@ -379,6 +379,32 @@ Shop.extract_commit_hash = function(pkg, response) {
var dylib_visited = {}
var open_dls = {}
var loaded_manifests = {}
// Host target detection for native dylib resolution
function detect_host_target() {
var platform = os.platform()
var arch = os.arch ? os.arch() : 'arm64'
if (platform == 'macOS' || platform == 'darwin')
return arch == 'x86_64' ? 'macos_x86_64' : 'macos_arm64'
if (platform == 'Linux' || platform == 'linux')
return arch == 'x86_64' ? 'linux' : 'linux_arm64'
if (platform == 'Windows' || platform == 'windows')
return 'windows'
return null
}
var host_target = detect_host_target()
// Check for a native .cm dylib in the content-addressed cache
// Returns the loaded module value, or null if no native dylib exists
function try_native_dylib(content_key) {
var native_path = hash_path(content_key) + '.' + host_target + dylib_ext
if (!fd.is_file(native_path)) return null
var handle = os.dylib_open(native_path)
if (!handle) return null
return os.native_module_load(handle)
}
// Default capabilities injected into scripts
// These map to $_ properties in engine.cm
@@ -433,7 +459,9 @@ function resolve_mod_fn(path, pkg) {
if (!fd.is_file(path)) { print(`path ${path} is not a file`); disrupt }
var content = text(fd.slurp(path))
var cached = pull_from_cache(stone(blob(content)))
var content_key = stone(blob(content))
var native_result = null
var cached = null
var ast = null
var compiled = null
var mach_path = null
@@ -441,24 +469,42 @@ function resolve_mod_fn(path, pkg) {
var mcode_path = null
var ir = null
var optimized = null
var mcode_json = null
var cached_mcode_path = null
// Check for native .cm dylib first (highest performance)
native_result = try_native_dylib(content_key)
if (native_result != null) {
return {_native: true, value: native_result}
}
// Check cache for pre-compiled .mach blob
cached = pull_from_cache(content_key)
if (cached) {
return cached
}
// Check for cached mcode in content-addressed store
cached_mcode_path = hash_path(content_key) + '.mcode'
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
}
// Check for pre-compiled .mach or .mcode file alongside .cm source
if (ends_with(path, '.cm')) {
mach_path = text(path, 0, length(path) - 3) + '.mach'
if (fd.is_file(mach_path)) {
mach_blob = fd.slurp(mach_path)
put_into_cache(stone(blob(content)), mach_blob)
put_into_cache(content_key, mach_blob)
return mach_blob
}
mcode_path = path + '.mcode'
if (fd.is_file(mcode_path)) {
compiled = mach_compile_mcode_bin(path, text(fd.slurp(mcode_path)))
put_into_cache(stone(blob(content)), compiled)
put_into_cache(content_key, compiled)
return compiled
}
}
@@ -469,8 +515,15 @@ function resolve_mod_fn(path, pkg) {
ast = analyze(content, path)
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
compiled = mach_compile_mcode_bin(path, shop_json.encode(optimized))
put_into_cache(stone(blob(content)), compiled)
mcode_json = shop_json.encode(optimized)
// Cache mcode (architecture-independent) in content-addressed store
ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
// Cache mach blob
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
}
@@ -576,7 +629,45 @@ function get_lib_path(pkg) {
return global_shop_path + '/lib/' + lib_name + dylib_ext
}
// Open a package's dynamic library and all its dependencies
// Load the manifest for a package's per-module dylibs
// Returns a map of symbol_name -> dylib_path, or null if no manifest
function load_package_manifest(pkg) {
if (loaded_manifests[pkg] != null) return loaded_manifests[pkg]
var lib_name = replace(replace(replace(pkg, '/', '_'), '.', '_'), '-', '_')
var manifest_path = global_shop_path + '/lib/' + lib_name + '.manifest.json'
if (!fd.is_file(manifest_path)) {
loaded_manifests[pkg] = false
return null
}
var content = text(fd.slurp(manifest_path))
var manifest = json.decode(content)
loaded_manifests[pkg] = manifest
return manifest
}
// Open a per-module dylib from a manifest and return the dlopen handle
function open_module_dylib(dylib_path) {
if (open_dls[dylib_path]) return open_dls[dylib_path]
if (!fd.is_file(dylib_path)) return null
open_dls[dylib_path] = os.dylib_open(dylib_path)
return open_dls[dylib_path]
}
// Resolve a C symbol from per-module dylibs for a package
// Returns a loader function or null
function resolve_dylib_symbol(sym, pkg) {
var manifest = load_package_manifest(pkg)
if (!manifest) return null
var dylib_path = manifest[sym]
if (!dylib_path) return null
var handle = open_module_dylib(dylib_path)
if (!handle) return null
if (!os.dylib_has_symbol(handle, sym)) return null
return function() { return os.dylib_symbol(handle, sym) }
}
// Open a package's dynamic libraries (loads manifest + dependency manifests)
Shop.open_package_dylib = function(pkg) {
if (pkg == 'core' || !pkg) return
if (dylib_visited[pkg]) return
@@ -606,22 +697,18 @@ Shop.open_package_dylib = function(pkg) {
}
}
var dl_path = get_lib_path(pkg)
if (fd.is_file(dl_path)) {
if (!open_dls[dl_path]) {
open_dls[dl_path] = os.dylib_open(dl_path)
}
}
// Pre-load the manifest
load_package_manifest(pkg)
}
// Resolve a C symbol by searching:
// 1. If package_context is null, only check core internal symbols
// 2. Otherwise: own package (internal then dylib) -> other packages (internal then dylib) -> core (internal only)
// 2. Otherwise: own package (internal then per-module dylib) -> aliased packages -> core (internal only)
// Core is never loaded as a dynamic library via dlopen
function resolve_c_symbol(path, package_context) {
var explicit = split_explicit_package_import(path)
var sym = null
var dl_path = null
var loader = null
var _path = null
var core_sym = null
var canon_pkg = null
@@ -643,10 +730,10 @@ function resolve_c_symbol(path, package_context) {
}
Shop.open_package_dylib(explicit.package)
dl_path = get_lib_path(explicit.package)
if (open_dls[dl_path] && os.dylib_has_symbol(open_dls[dl_path], sym)) {
loader = resolve_dylib_symbol(sym, explicit.package)
if (loader) {
return {
symbol: function() { return os.dylib_symbol(open_dls[dl_path], sym) },
symbol: loader,
scope: SCOPE_PACKAGE,
package: explicit.package,
path: sym
@@ -668,7 +755,7 @@ function resolve_c_symbol(path, package_context) {
return null
}
// 1. Check own package first (internal, then dylib)
// 1. Check own package first (internal, then per-module dylib)
sym = make_c_symbol(package_context, path)
if (os.internal_exists(sym)) {
return {
@@ -679,11 +766,10 @@ function resolve_c_symbol(path, package_context) {
}
Shop.open_package_dylib(package_context)
dl_path = get_lib_path(package_context)
if (open_dls[dl_path] && os.dylib_has_symbol(open_dls[dl_path], sym)) {
loader = resolve_dylib_symbol(sym, package_context)
if (loader) {
return {
symbol: function() { return os.dylib_symbol(open_dls[dl_path], sym) },
symbol: loader,
scope: SCOPE_LOCAL,
path: sym
}
@@ -700,7 +786,6 @@ function resolve_c_symbol(path, package_context) {
mod_name = get_import_name(path)
sym = make_c_symbol(canon_pkg, mod_name)
// Check internal first
if (os.internal_exists(sym)) {
return {
symbol: function() { return os.load_internal(sym) },
@@ -710,12 +795,11 @@ function resolve_c_symbol(path, package_context) {
}
}
// Then check dylib
Shop.open_package_dylib(canon_pkg)
dl_path = get_lib_path(canon_pkg)
if (open_dls[dl_path] && os.dylib_has_symbol(open_dls[dl_path], sym)) {
loader = resolve_dylib_symbol(sym, canon_pkg)
if (loader) {
return {
symbol: function() { return os.dylib_symbol(open_dls[dl_path], sym) },
symbol: loader,
scope: SCOPE_PACKAGE,
package: canon_pkg,
path: sym
@@ -841,20 +925,20 @@ function execute_module(info)
var pkg = null
if (mod_resolve.scope < 900) {
// Build env with runtime fns, capabilities, and use function
file_info = Shop.file_info(mod_resolve.path)
inject = Shop.script_inject_for(file_info)
env = inject_env(inject)
pkg = file_info.package
env.use = make_use_fn(pkg)
// Check if native dylib was resolved
if (is_object(mod_resolve.symbol) && mod_resolve.symbol._native) {
used = mod_resolve.symbol.value
} else {
// Build env with runtime fns, capabilities, and use function
file_info = Shop.file_info(mod_resolve.path)
inject = Shop.script_inject_for(file_info)
env = inject_env(inject)
pkg = file_info.package
env.use = make_use_fn(pkg)
// Add C module as native context if available
if (c_resolve.scope < 900) {
env.native = call_c_module(c_resolve)
// Load compiled bytecode with env
used = mach_load(mod_resolve.symbol, env)
}
// Load compiled bytecode with env
used = mach_load(mod_resolve.symbol, env)
} else if (c_resolve.scope < 900) {
// C only
used = call_c_module(c_resolve)
@@ -876,6 +960,21 @@ function get_module(path, package_context) {
}
Shop.use = function use(path, package_context) {
// Check for embedded module (static builds)
var embed_key = 'embedded:' + path
var embedded = null
var embed_env = null
if (use_cache[embed_key]) return use_cache[embed_key]
if (os.embedded_module) {
embedded = os.embedded_module(path)
if (embedded) {
embed_env = inject_env(SHOP_DEFAULT_INJECT)
embed_env.use = make_use_fn(package_context)
use_cache[embed_key] = mach_load(embedded, embed_env)
return use_cache[embed_key]
}
}
var info = resolve_module_info(path, package_context)
if (!info) { print(`Module ${path} could not be found in ${package_context}`); disrupt }

1
regen.ce
View File

@@ -17,6 +17,7 @@ var files = [
{src: "streamline.cm", name: "streamline", out: "streamline.cm.mcode"},
{src: "qbe.cm", name: "qbe", out: "qbe.cm.mcode"},
{src: "qbe_emit.cm", name: "qbe_emit", out: "qbe_emit.cm.mcode"},
{src: "verify_ir.cm", name: "verify_ir", out: "verify_ir.cm.mcode"},
{src: "internal/bootstrap.cm", name: "bootstrap", out: "internal/bootstrap.cm.mcode"},
{src: "internal/engine.cm", name: "engine", out: "internal/engine.cm.mcode"}
]

80
source/qbe_helpers.c
View File

@@ -277,19 +277,37 @@ void cell_rt_put_closure(JSContext *ctx, void *fp, JSValue val, int64_t depth,
typedef JSValue (*cell_compiled_fn)(JSContext *ctx, void *fp);
/* Table mapping fn_idx → outer_fp at creation time.
Valid for single-threaded, non-recursive closure patterns. */
#define MAX_QBE_FUNCTIONS 256
static void *g_outer_fp[MAX_QBE_FUNCTIONS];
/* Per-module function registry.
Each native .cm module gets its own dylib. When a module creates closures
via cell_rt_make_function, we record the dylib handle so the trampoline
can look up the correct cell_fn_N in the right dylib. */
#define MAX_NATIVE_FN 4096
static struct {
void *dl_handle;
int fn_idx;
void *outer_fp;
} g_native_fn_registry[MAX_NATIVE_FN];
static int g_native_fn_count = 0;
/* Set before executing a native module's cell_main */
static void *g_current_dl_handle = NULL;
static JSValue cell_fn_trampoline(JSContext *ctx, JSValue this_val,
int argc, JSValue *argv, int magic) {
char name[64];
snprintf(name, sizeof(name), "cell_fn_%d", magic);
if (magic < 0 || magic >= g_native_fn_count)
return JS_ThrowTypeError(ctx, "invalid native function id %d", magic);
cell_compiled_fn fn = (cell_compiled_fn)dlsym(RTLD_DEFAULT, name);
void *handle = g_native_fn_registry[magic].dl_handle;
int fn_idx = g_native_fn_registry[magic].fn_idx;
char name[64];
snprintf(name, sizeof(name), "cell_fn_%d", fn_idx);
cell_compiled_fn fn = (cell_compiled_fn)dlsym(handle, name);
if (!fn)
return JS_ThrowTypeError(ctx, "native function %s not found", name);
return JS_ThrowTypeError(ctx, "native function %s not found in dylib", name);
/* Allocate frame: slot 0 = this, slots 1..argc = args */
JSValue frame[512];
@@ -299,17 +317,22 @@ static JSValue cell_fn_trampoline(JSContext *ctx, JSValue this_val,
frame[1 + i] = argv[i];
/* Link to outer frame for closure access */
if (magic >= 0 && magic < MAX_QBE_FUNCTIONS)
frame[QBE_FRAME_OUTER_SLOT] = (JSValue)(uintptr_t)g_outer_fp[magic];
frame[QBE_FRAME_OUTER_SLOT] = (JSValue)(uintptr_t)g_native_fn_registry[magic].outer_fp;
return fn(ctx, frame);
}
JSValue cell_rt_make_function(JSContext *ctx, int64_t fn_idx, void *outer_fp) {
if (fn_idx >= 0 && fn_idx < MAX_QBE_FUNCTIONS)
g_outer_fp[fn_idx] = outer_fp;
if (g_native_fn_count >= MAX_NATIVE_FN)
return JS_ThrowTypeError(ctx, "too many native functions (max %d)", MAX_NATIVE_FN);
int global_id = g_native_fn_count++;
g_native_fn_registry[global_id].dl_handle = g_current_dl_handle;
g_native_fn_registry[global_id].fn_idx = (int)fn_idx;
g_native_fn_registry[global_id].outer_fp = outer_fp;
return JS_NewCFunction2(ctx, (JSCFunction *)cell_fn_trampoline, "native_fn",
255, JS_CFUNC_generic_magic, (int)fn_idx);
255, JS_CFUNC_generic_magic, global_id);
}
/* --- Frame-based function calling --- */
@@ -430,20 +453,35 @@ void cell_rt_disrupt(JSContext *ctx) {
}
/* --- Module entry point ---
Called as symbol(ctx) by os.dylib_symbol. Looks up cell_main
in the loaded dylib, builds a heap-allocated frame (so closures
can reference it after the module returns), and runs the module body. */
Loads a native .cm module from a dylib handle.
Looks up cell_main, builds a heap-allocated frame, sets
g_current_dl_handle so closures register in the right module. */
JSValue cell_rt_module_entry(JSContext *ctx) {
cell_compiled_fn fn = (cell_compiled_fn)dlsym(RTLD_DEFAULT, "cell_main");
JSValue cell_rt_native_module_load(JSContext *ctx, void *dl_handle) {
cell_compiled_fn fn = (cell_compiled_fn)dlsym(dl_handle, "cell_main");
if (!fn)
return JS_ThrowTypeError(ctx, "cell_main not found in loaded dylib");
return JS_ThrowTypeError(ctx, "cell_main not found in native module dylib");
/* Set current handle so cell_rt_make_function registers closures
against this module's dylib */
void *prev_handle = g_current_dl_handle;
g_current_dl_handle = dl_handle;
/* Heap-allocate so closures created in cell_main can reference
this frame after the module entry returns. */
JSValue *frame = calloc(512, sizeof(JSValue));
if (!frame)
if (!frame) {
g_current_dl_handle = prev_handle;
return JS_ThrowTypeError(ctx, "frame allocation failed");
}
return fn(ctx, frame);
JSValue result = fn(ctx, frame);
g_current_dl_handle = prev_handle;
return result;
}
/* Backward-compat: uses RTLD_DEFAULT (works when dylib opened with RTLD_GLOBAL) */
JSValue cell_rt_module_entry(JSContext *ctx) {
void *handle = dlopen(NULL, RTLD_LAZY);
return cell_rt_native_module_load(ctx, handle);
}

2
time.cm
View File

@@ -1,5 +1,5 @@
// epoch = 0000-01-01 00:00:00 +0000
var time = native
var time = use('internal/time_c')
var now = time.now
var computer_zone = time.computer_zone