john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

mach loading

Browse Source
This commit is contained in:
John Alanbrook
2026-02-13 07:26:49 -06:00
parent 4f18a0b524
commit 77fa058135
11 changed files with 7940 additions and 8637 deletions
Download Patch File Download Diff File Expand all files Collapse all files

177
internal/shop.cm
View File

@@ -379,6 +379,32 @@ Shop.extract_commit_hash = function(pkg, response) {
var dylib_visited = {}
var open_dls = {}
var loaded_manifests = {}
// Host target detection for native dylib resolution
function detect_host_target() {
var platform = os.platform()
var arch = os.arch ? os.arch() : 'arm64'
if (platform == 'macOS' || platform == 'darwin')
return arch == 'x86_64' ? 'macos_x86_64' : 'macos_arm64'
if (platform == 'Linux' || platform == 'linux')
return arch == 'x86_64' ? 'linux' : 'linux_arm64'
if (platform == 'Windows' || platform == 'windows')
return 'windows'
return null
}
var host_target = detect_host_target()
// Check for a native .cm dylib in the content-addressed cache
// Returns the loaded module value, or null if no native dylib exists
function try_native_dylib(content_key) {
var native_path = hash_path(content_key) + '.' + host_target + dylib_ext
if (!fd.is_file(native_path)) return null
var handle = os.dylib_open(native_path)
if (!handle) return null
return os.native_module_load(handle)
}
// Default capabilities injected into scripts
// These map to $_ properties in engine.cm
@@ -433,7 +459,9 @@ function resolve_mod_fn(path, pkg) {
if (!fd.is_file(path)) { print(`path ${path} is not a file`); disrupt }
var content = text(fd.slurp(path))
var cached = pull_from_cache(stone(blob(content)))
var content_key = stone(blob(content))
var native_result = null
var cached = null
var ast = null
var compiled = null
var mach_path = null
@@ -441,24 +469,42 @@ function resolve_mod_fn(path, pkg) {
var mcode_path = null
var ir = null
var optimized = null
var mcode_json = null
var cached_mcode_path = null
// Check for native .cm dylib first (highest performance)
native_result = try_native_dylib(content_key)
if (native_result != null) {
return {_native: true, value: native_result}
}
// Check cache for pre-compiled .mach blob
cached = pull_from_cache(content_key)
if (cached) {
return cached
}
// Check for cached mcode in content-addressed store
cached_mcode_path = hash_path(content_key) + '.mcode'
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
}
// Check for pre-compiled .mach or .mcode file alongside .cm source
if (ends_with(path, '.cm')) {
mach_path = text(path, 0, length(path) - 3) + '.mach'
if (fd.is_file(mach_path)) {
mach_blob = fd.slurp(mach_path)
put_into_cache(stone(blob(content)), mach_blob)
put_into_cache(content_key, mach_blob)
return mach_blob
}
mcode_path = path + '.mcode'
if (fd.is_file(mcode_path)) {
compiled = mach_compile_mcode_bin(path, text(fd.slurp(mcode_path)))
put_into_cache(stone(blob(content)), compiled)
put_into_cache(content_key, compiled)
return compiled
}
}
@@ -469,8 +515,15 @@ function resolve_mod_fn(path, pkg) {
ast = analyze(content, path)
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
compiled = mach_compile_mcode_bin(path, shop_json.encode(optimized))
put_into_cache(stone(blob(content)), compiled)
mcode_json = shop_json.encode(optimized)
// Cache mcode (architecture-independent) in content-addressed store
ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
// Cache mach blob
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
}
@@ -576,7 +629,45 @@ function get_lib_path(pkg) {
return global_shop_path + '/lib/' + lib_name + dylib_ext
}
// Open a package's dynamic library and all its dependencies
// Load the manifest for a package's per-module dylibs
// Returns a map of symbol_name -> dylib_path, or null if no manifest
function load_package_manifest(pkg) {
if (loaded_manifests[pkg] != null) return loaded_manifests[pkg]
var lib_name = replace(replace(replace(pkg, '/', '_'), '.', '_'), '-', '_')
var manifest_path = global_shop_path + '/lib/' + lib_name + '.manifest.json'
if (!fd.is_file(manifest_path)) {
loaded_manifests[pkg] = false
return null
}
var content = text(fd.slurp(manifest_path))
var manifest = json.decode(content)
loaded_manifests[pkg] = manifest
return manifest
}
// Open a per-module dylib from a manifest and return the dlopen handle
function open_module_dylib(dylib_path) {
if (open_dls[dylib_path]) return open_dls[dylib_path]
if (!fd.is_file(dylib_path)) return null
open_dls[dylib_path] = os.dylib_open(dylib_path)
return open_dls[dylib_path]
}
// Resolve a C symbol from per-module dylibs for a package
// Returns a loader function or null
function resolve_dylib_symbol(sym, pkg) {
var manifest = load_package_manifest(pkg)
if (!manifest) return null
var dylib_path = manifest[sym]
if (!dylib_path) return null
var handle = open_module_dylib(dylib_path)
if (!handle) return null
if (!os.dylib_has_symbol(handle, sym)) return null
return function() { return os.dylib_symbol(handle, sym) }
}
// Open a package's dynamic libraries (loads manifest + dependency manifests)
Shop.open_package_dylib = function(pkg) {
if (pkg == 'core' || !pkg) return
if (dylib_visited[pkg]) return
@@ -606,22 +697,18 @@ Shop.open_package_dylib = function(pkg) {
}
}
var dl_path = get_lib_path(pkg)
if (fd.is_file(dl_path)) {
if (!open_dls[dl_path]) {
open_dls[dl_path] = os.dylib_open(dl_path)
}
}
// Pre-load the manifest
load_package_manifest(pkg)
}
// Resolve a C symbol by searching:
// 1. If package_context is null, only check core internal symbols
// 2. Otherwise: own package (internal then dylib) -> other packages (internal then dylib) -> core (internal only)
// 2. Otherwise: own package (internal then per-module dylib) -> aliased packages -> core (internal only)
// Core is never loaded as a dynamic library via dlopen
function resolve_c_symbol(path, package_context) {
var explicit = split_explicit_package_import(path)
var sym = null
var dl_path = null
var loader = null
var _path = null
var core_sym = null
var canon_pkg = null
@@ -643,10 +730,10 @@ function resolve_c_symbol(path, package_context) {
}
Shop.open_package_dylib(explicit.package)
dl_path = get_lib_path(explicit.package)
if (open_dls[dl_path] && os.dylib_has_symbol(open_dls[dl_path], sym)) {
loader = resolve_dylib_symbol(sym, explicit.package)
if (loader) {
return {
symbol: function() { return os.dylib_symbol(open_dls[dl_path], sym) },
symbol: loader,
scope: SCOPE_PACKAGE,
package: explicit.package,
path: sym
@@ -668,7 +755,7 @@ function resolve_c_symbol(path, package_context) {
return null
}
// 1. Check own package first (internal, then dylib)
// 1. Check own package first (internal, then per-module dylib)
sym = make_c_symbol(package_context, path)
if (os.internal_exists(sym)) {
return {
@@ -679,11 +766,10 @@ function resolve_c_symbol(path, package_context) {
}
Shop.open_package_dylib(package_context)
dl_path = get_lib_path(package_context)
if (open_dls[dl_path] && os.dylib_has_symbol(open_dls[dl_path], sym)) {
loader = resolve_dylib_symbol(sym, package_context)
if (loader) {
return {
symbol: function() { return os.dylib_symbol(open_dls[dl_path], sym) },
symbol: loader,
scope: SCOPE_LOCAL,
path: sym
}
@@ -700,7 +786,6 @@ function resolve_c_symbol(path, package_context) {
mod_name = get_import_name(path)
sym = make_c_symbol(canon_pkg, mod_name)
// Check internal first
if (os.internal_exists(sym)) {
return {
symbol: function() { return os.load_internal(sym) },
@@ -710,12 +795,11 @@ function resolve_c_symbol(path, package_context) {
}
}
// Then check dylib
Shop.open_package_dylib(canon_pkg)
dl_path = get_lib_path(canon_pkg)
if (open_dls[dl_path] && os.dylib_has_symbol(open_dls[dl_path], sym)) {
loader = resolve_dylib_symbol(sym, canon_pkg)
if (loader) {
return {
symbol: function() { return os.dylib_symbol(open_dls[dl_path], sym) },
symbol: loader,
scope: SCOPE_PACKAGE,
package: canon_pkg,
path: sym
@@ -841,20 +925,20 @@ function execute_module(info)
var pkg = null
if (mod_resolve.scope < 900) {
// Build env with runtime fns, capabilities, and use function
file_info = Shop.file_info(mod_resolve.path)
inject = Shop.script_inject_for(file_info)
env = inject_env(inject)
pkg = file_info.package
env.use = make_use_fn(pkg)
// Check if native dylib was resolved
if (is_object(mod_resolve.symbol) && mod_resolve.symbol._native) {
used = mod_resolve.symbol.value
} else {
// Build env with runtime fns, capabilities, and use function
file_info = Shop.file_info(mod_resolve.path)
inject = Shop.script_inject_for(file_info)
env = inject_env(inject)
pkg = file_info.package
env.use = make_use_fn(pkg)
// Add C module as native context if available
if (c_resolve.scope < 900) {
env.native = call_c_module(c_resolve)
// Load compiled bytecode with env
used = mach_load(mod_resolve.symbol, env)
}
// Load compiled bytecode with env
used = mach_load(mod_resolve.symbol, env)
} else if (c_resolve.scope < 900) {
// C only
used = call_c_module(c_resolve)
@@ -876,6 +960,21 @@ function get_module(path, package_context) {
}
Shop.use = function use(path, package_context) {
// Check for embedded module (static builds)
var embed_key = 'embedded:' + path
var embedded = null
var embed_env = null
if (use_cache[embed_key]) return use_cache[embed_key]
if (os.embedded_module) {
embedded = os.embedded_module(path)
if (embedded) {
embed_env = inject_env(SHOP_DEFAULT_INJECT)
embed_env.use = make_use_fn(package_context)
use_cache[embed_key] = mach_load(embedded, embed_env)
return use_cache[embed_key]
}
}
var info = resolve_module_info(path, package_context)
if (!info) { print(`Module ${path} could not be found in ${package_context}`); disrupt }