mach loading
This commit is contained in:
365
build.cm
365
build.cm
@@ -214,36 +214,22 @@ function compute_link_key(objects, ldflags, target_ldflags, target, cc) {
|
||||
return content_hash(text(parts, '\n'))
|
||||
}
|
||||
|
||||
// Build a dynamic library for a package
|
||||
// Output goes to .cell/lib/<package_name>.<ext>
|
||||
// Dynamic libraries do NOT link against core; undefined symbols are resolved at dlopen time
|
||||
// Uses content-addressed store + symlink for caching
|
||||
Build.build_dynamic = function(pkg, target = Build.detect_host_target(), buildtype = 'release') {
|
||||
var objects = Build.build_package(pkg, target, true, buildtype) // exclude main.c
|
||||
// Build a per-module dynamic library for a single C file
|
||||
// Returns the content-addressed dylib path in .cell/build/<hash>.<target>.dylib
|
||||
Build.build_module_dylib = function(pkg, file, target, buildtype) {
|
||||
var _target = target || Build.detect_host_target()
|
||||
var _buildtype = buildtype || 'release'
|
||||
var obj = Build.compile_file(pkg, file, _target, _buildtype)
|
||||
|
||||
if (length(objects) == 0) {
|
||||
log.console('No C files in ' + pkg)
|
||||
return null
|
||||
}
|
||||
|
||||
var lib_dir = shop.get_lib_dir()
|
||||
var store_dir = lib_dir + '/store'
|
||||
ensure_dir(lib_dir)
|
||||
ensure_dir(store_dir)
|
||||
|
||||
var lib_name = shop.lib_name_for_package(pkg)
|
||||
var dylib_ext = toolchains[target].system == 'windows' ? '.dll' : (toolchains[target].system == 'darwin' ? '.dylib' : '.so')
|
||||
var stable_path = lib_dir + '/' + lib_name + dylib_ext
|
||||
|
||||
// Get link flags (with sigil replacement)
|
||||
var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', target))
|
||||
var target_ldflags = toolchains[target].c_link_args || []
|
||||
var cc = toolchains[target].cpp || toolchains[target].c
|
||||
var pkg_dir = shop.get_package_dir(pkg)
|
||||
var tc = toolchains[_target]
|
||||
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
|
||||
var cc = tc.cpp || tc.c
|
||||
var local_dir = get_local_dir()
|
||||
var tc = toolchains[target]
|
||||
var pkg_dir = shop.get_package_dir(pkg)
|
||||
|
||||
// Resolve relative -L paths in ldflags for hash computation
|
||||
// Get link flags
|
||||
var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target))
|
||||
var target_ldflags = tc.c_link_args || []
|
||||
var resolved_ldflags = []
|
||||
arrfor(ldflags, function(flag) {
|
||||
if (starts_with(flag, '-L') && !starts_with(flag, '-L/')) {
|
||||
@@ -252,36 +238,21 @@ Build.build_dynamic = function(pkg, target = Build.detect_host_target(), buildty
|
||||
push(resolved_ldflags, flag)
|
||||
})
|
||||
|
||||
// Compute link key
|
||||
var link_key = compute_link_key(objects, resolved_ldflags, target_ldflags, target, cc)
|
||||
var store_path = store_dir + '/' + lib_name + '-' + link_key + dylib_ext
|
||||
// Content-addressed output: hash of (object + link flags + target)
|
||||
var link_key = compute_link_key([obj], resolved_ldflags, target_ldflags, _target, cc)
|
||||
var build_dir = get_build_dir()
|
||||
ensure_dir(build_dir)
|
||||
var dylib_path = build_dir + '/' + link_key + '.' + _target + dylib_ext
|
||||
|
||||
// Check if already linked in store
|
||||
if (fd.is_file(store_path)) {
|
||||
// Ensure symlink points to the store file
|
||||
if (fd.is_link(stable_path)) {
|
||||
var current_target = fd.readlink(stable_path)
|
||||
if (current_target == store_path) {
|
||||
// Already up to date
|
||||
return stable_path
|
||||
}
|
||||
fd.unlink(stable_path)
|
||||
} else if (fd.is_file(stable_path)) {
|
||||
fd.unlink(stable_path)
|
||||
}
|
||||
fd.symlink(store_path, stable_path)
|
||||
return stable_path
|
||||
}
|
||||
if (fd.is_file(dylib_path))
|
||||
return dylib_path
|
||||
|
||||
// Build link command
|
||||
var cmd_parts = [cc, '-shared', '-fPIC']
|
||||
|
||||
// Platform-specific flags for undefined symbols (resolved at dlopen) and size optimization
|
||||
if (tc.system == 'darwin') {
|
||||
cmd_parts = array(cmd_parts, [
|
||||
'-undefined', 'dynamic_lookup',
|
||||
'-Wl,-dead_strip',
|
||||
'-Wl,-install_name,' + stable_path,
|
||||
'-Wl,-rpath,@loader_path/../local',
|
||||
'-Wl,-rpath,' + local_dir
|
||||
])
|
||||
@@ -293,41 +264,54 @@ Build.build_dynamic = function(pkg, target = Build.detect_host_target(), buildty
|
||||
'-Wl,-rpath,' + local_dir
|
||||
])
|
||||
} else if (tc.system == 'windows') {
|
||||
// Windows DLLs: use --allow-shlib-undefined for mingw
|
||||
push(cmd_parts, '-Wl,--allow-shlib-undefined')
|
||||
}
|
||||
|
||||
// Add .cell/local to library search path
|
||||
push(cmd_parts, '-L"' + local_dir + '"')
|
||||
|
||||
arrfor(objects, function(obj) {
|
||||
push(cmd_parts, '"' + obj + '"')
|
||||
})
|
||||
|
||||
// Do NOT link against core library - symbols resolved at dlopen time
|
||||
push(cmd_parts, '"' + obj + '"')
|
||||
cmd_parts = array(cmd_parts, resolved_ldflags)
|
||||
cmd_parts = array(cmd_parts, target_ldflags)
|
||||
|
||||
push(cmd_parts, '-o')
|
||||
push(cmd_parts, '"' + store_path + '"')
|
||||
push(cmd_parts, '"' + dylib_path + '"')
|
||||
|
||||
var cmd_str = text(cmd_parts, ' ')
|
||||
|
||||
log.console('Linking ' + lib_name + dylib_ext)
|
||||
log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path))
|
||||
var ret = os.system(cmd_str)
|
||||
if (ret != 0) {
|
||||
throw Error('Linking failed: ' + pkg)
|
||||
print('Linking failed: ' + file); disrupt
|
||||
}
|
||||
|
||||
// Update symlink to point to the new store file
|
||||
if (fd.is_link(stable_path)) {
|
||||
fd.unlink(stable_path)
|
||||
} else if (fd.is_file(stable_path)) {
|
||||
fd.unlink(stable_path)
|
||||
}
|
||||
fd.symlink(store_path, stable_path)
|
||||
return dylib_path
|
||||
}
|
||||
|
||||
return stable_path
|
||||
// Build a dynamic library for a package (one dylib per C file)
|
||||
// Returns array of {file, symbol, dylib} for each module
|
||||
// Also writes a manifest mapping symbols to dylib paths
|
||||
Build.build_dynamic = function(pkg, target, buildtype) {
|
||||
var _target = target || Build.detect_host_target()
|
||||
var _buildtype = buildtype || 'release'
|
||||
var c_files = pkg_tools.get_c_files(pkg, _target, true)
|
||||
var results = []
|
||||
var manifest = {}
|
||||
|
||||
arrfor(c_files, function(file) {
|
||||
var sym_name = shop.c_symbol_for_file(pkg, file)
|
||||
var dylib = Build.build_module_dylib(pkg, file, _target, _buildtype)
|
||||
push(results, {file: file, symbol: sym_name, dylib: dylib})
|
||||
manifest[sym_name] = dylib
|
||||
})
|
||||
|
||||
// Write manifest so the loader can find per-module dylibs by symbol
|
||||
if (length(results) > 0) {
|
||||
var lib_dir = shop.get_lib_dir()
|
||||
ensure_dir(lib_dir)
|
||||
var lib_name = shop.lib_name_for_package(pkg)
|
||||
var manifest_path = lib_dir + '/' + lib_name + '.manifest.json'
|
||||
var json = use('json')
|
||||
fd.slurpwrite(manifest_path, stone(blob(json.encode(manifest))))
|
||||
}
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
@@ -412,43 +396,236 @@ Build.build_static = function(packages, target = Build.detect_host_target(), out
|
||||
return output
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Native .cm compilation (source → mcode → QBE IL → .o → .dylib)
|
||||
// ============================================================================
|
||||
|
||||
// Post-process QBE IL: insert dead labels after ret/jmp (QBE requirement)
|
||||
function qbe_insert_dead_labels(il_text) {
|
||||
var lines = array(il_text, "\n")
|
||||
var result = []
|
||||
var dead_id = 0
|
||||
var need_label = false
|
||||
var i = 0
|
||||
var line = null
|
||||
var trimmed = null
|
||||
while (i < length(lines)) {
|
||||
line = lines[i]
|
||||
trimmed = trim(line)
|
||||
if (need_label && !starts_with(trimmed, '@') && !starts_with(trimmed, '}') && length(trimmed) > 0) {
|
||||
push(result, "@_dead_" + text(dead_id))
|
||||
dead_id = dead_id + 1
|
||||
need_label = false
|
||||
}
|
||||
if (starts_with(trimmed, '@') || starts_with(trimmed, '}') || length(trimmed) == 0) {
|
||||
need_label = false
|
||||
}
|
||||
if (starts_with(trimmed, 'ret ') || starts_with(trimmed, 'jmp ')) {
|
||||
need_label = true
|
||||
}
|
||||
push(result, line)
|
||||
i = i + 1
|
||||
}
|
||||
return text(result, "\n")
|
||||
}
|
||||
|
||||
// Compile a .cm source file to a native .dylib via QBE
|
||||
// Returns the content-addressed dylib path
|
||||
Build.compile_native = function(src_path, target, buildtype) {
|
||||
var _target = target || Build.detect_host_target()
|
||||
var _buildtype = buildtype || 'release'
|
||||
|
||||
if (!fd.is_file(src_path)) {
|
||||
print('Source file not found: ' + src_path); disrupt
|
||||
}
|
||||
|
||||
var tc = toolchains[_target]
|
||||
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
|
||||
var cc = tc.c
|
||||
|
||||
// Step 1: Read source and compile through pipeline
|
||||
var content = fd.slurp(src_path)
|
||||
var src = text(content)
|
||||
var tokenize = use('tokenize')
|
||||
var parse = use('parse')
|
||||
var fold = use('fold')
|
||||
var mcode_mod = use('mcode')
|
||||
var streamline_mod = use('streamline')
|
||||
var qbe_macros = use('qbe')
|
||||
var qbe_emit = use('qbe_emit')
|
||||
|
||||
var tok_result = tokenize(src, src_path)
|
||||
var ast = parse(tok_result.tokens, src, src_path, tokenize)
|
||||
var folded = fold(ast)
|
||||
var compiled = mcode_mod(folded)
|
||||
var optimized = streamline_mod(compiled)
|
||||
|
||||
// Step 2: Generate QBE IL
|
||||
var il = qbe_emit(optimized, qbe_macros)
|
||||
|
||||
// Step 3: Post-process (insert dead labels)
|
||||
il = qbe_insert_dead_labels(il)
|
||||
|
||||
// Content hash for cache key
|
||||
var hash = content_hash(src + '\n' + _target + '\nnative')
|
||||
var build_dir = get_build_dir()
|
||||
ensure_dir(build_dir)
|
||||
|
||||
var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
|
||||
if (fd.is_file(dylib_path))
|
||||
return dylib_path
|
||||
|
||||
// Step 4: Write QBE IL to temp file
|
||||
var tmp = '/tmp/cell_native_' + hash
|
||||
var ssa_path = tmp + '.ssa'
|
||||
var s_path = tmp + '.s'
|
||||
var o_path = tmp + '.o'
|
||||
var rt_o_path = '/tmp/cell_qbe_rt.o'
|
||||
|
||||
fd.slurpwrite(ssa_path, stone(blob(il)))
|
||||
|
||||
// Step 5: QBE compile to assembly
|
||||
var rc = os.system('qbe -o ' + s_path + ' ' + ssa_path)
|
||||
if (rc != 0) {
|
||||
print('QBE compilation failed for: ' + src_path); disrupt
|
||||
}
|
||||
|
||||
// Step 6: Assemble
|
||||
rc = os.system(cc + ' -c ' + s_path + ' -o ' + o_path)
|
||||
if (rc != 0) {
|
||||
print('Assembly failed for: ' + src_path); disrupt
|
||||
}
|
||||
|
||||
// Step 7: Compile QBE runtime stubs if needed
|
||||
if (!fd.is_file(rt_o_path)) {
|
||||
var qbe_rt_path = shop.get_package_dir('core') + '/qbe_rt.c'
|
||||
rc = os.system(cc + ' -c ' + qbe_rt_path + ' -o ' + rt_o_path + ' -fPIC')
|
||||
if (rc != 0) {
|
||||
print('QBE runtime stubs compilation failed'); disrupt
|
||||
}
|
||||
}
|
||||
|
||||
// Step 8: Link dylib
|
||||
var link_cmd = cc + ' -shared -fPIC'
|
||||
if (tc.system == 'darwin') {
|
||||
link_cmd = link_cmd + ' -undefined dynamic_lookup'
|
||||
} else if (tc.system == 'linux') {
|
||||
link_cmd = link_cmd + ' -Wl,--allow-shlib-undefined'
|
||||
}
|
||||
link_cmd = link_cmd + ' ' + o_path + ' ' + rt_o_path + ' -o ' + dylib_path
|
||||
|
||||
rc = os.system(link_cmd)
|
||||
if (rc != 0) {
|
||||
print('Linking native dylib failed for: ' + src_path); disrupt
|
||||
}
|
||||
|
||||
log.console('Built native: ' + fd.basename(dylib_path))
|
||||
return dylib_path
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Module table generation (for static builds)
|
||||
// ============================================================================
|
||||
|
||||
// Compile a .cm module to mach bytecode blob
|
||||
// Returns the raw mach bytes as a blob
|
||||
Build.compile_cm_to_mach = function(src_path) {
|
||||
if (!fd.is_file(src_path)) {
|
||||
print('Source file not found: ' + src_path); disrupt
|
||||
}
|
||||
var src = text(fd.slurp(src_path))
|
||||
var tokenize = use('tokenize')
|
||||
var parse = use('parse')
|
||||
var fold = use('fold')
|
||||
var mcode_mod = use('mcode')
|
||||
var streamline_mod = use('streamline')
|
||||
var json = use('json')
|
||||
|
||||
var tok_result = tokenize(src, src_path)
|
||||
var ast = parse(tok_result.tokens, src, src_path, tokenize)
|
||||
var folded = fold(ast)
|
||||
var compiled = mcode_mod(folded)
|
||||
var optimized = streamline_mod(compiled)
|
||||
return mach_compile_mcode_bin(src_path, json.encode(optimized))
|
||||
}
|
||||
|
||||
// Generate a module_table.c file that embeds mach bytecode for .cm modules
|
||||
// modules: array of {name, src_path} — name is the module name, src_path is the .cm file
|
||||
// output: path to write the generated .c file
|
||||
Build.generate_module_table = function(modules, output) {
|
||||
var lines = []
|
||||
var json = use('json')
|
||||
push(lines, '/* Generated module table — do not edit */')
|
||||
push(lines, '#include <stddef.h>')
|
||||
push(lines, '#include <string.h>')
|
||||
push(lines, '')
|
||||
push(lines, 'struct cell_embedded_entry {')
|
||||
push(lines, ' const char *name;')
|
||||
push(lines, ' const unsigned char *data;')
|
||||
push(lines, ' size_t size;')
|
||||
push(lines, '};')
|
||||
push(lines, '')
|
||||
|
||||
var entries = []
|
||||
arrfor(modules, function(mod) {
|
||||
var safe = replace(replace(replace(mod.name, '/', '_'), '.', '_'), '-', '_')
|
||||
var mach = Build.compile_cm_to_mach(mod.src_path)
|
||||
var bytes = array(mach)
|
||||
var hex = []
|
||||
arrfor(bytes, function(b) {
|
||||
push(hex, '0x' + text(b, 'h2'))
|
||||
})
|
||||
push(lines, 'static const unsigned char mod_' + safe + '_data[] = {')
|
||||
push(lines, ' ' + text(hex, ', '))
|
||||
push(lines, '};')
|
||||
push(lines, '')
|
||||
push(entries, safe)
|
||||
log.console('Embedded: ' + mod.name + ' (' + text(length(bytes)) + ' bytes)')
|
||||
})
|
||||
|
||||
// Lookup function
|
||||
push(lines, 'const struct cell_embedded_entry *cell_embedded_module_lookup(const char *name) {')
|
||||
arrfor(modules, function(mod, i) {
|
||||
var safe = entries[i]
|
||||
push(lines, ' if (strcmp(name, "' + mod.name + '") == 0) {')
|
||||
push(lines, ' static const struct cell_embedded_entry e = {"' + mod.name + '", mod_' + safe + '_data, sizeof(mod_' + safe + '_data)};')
|
||||
push(lines, ' return &e;')
|
||||
push(lines, ' }')
|
||||
})
|
||||
push(lines, ' return (void *)0;')
|
||||
push(lines, '}')
|
||||
|
||||
var c_text = text(lines, '\n')
|
||||
fd.slurpwrite(output, stone(blob(c_text)))
|
||||
log.console('Generated ' + output)
|
||||
return output
|
||||
}
|
||||
|
||||
// ============================================================================
|
||||
// Convenience functions
|
||||
// ============================================================================
|
||||
|
||||
// Build dynamic libraries for all installed packages
|
||||
Build.build_all_dynamic = function(target, buildtype = 'release') {
|
||||
target = target || Build.detect_host_target()
|
||||
|
||||
Build.build_all_dynamic = function(target, buildtype) {
|
||||
var _target = target || Build.detect_host_target()
|
||||
var _buildtype = buildtype || 'release'
|
||||
|
||||
var packages = shop.list_packages()
|
||||
var results = []
|
||||
|
||||
|
||||
// Build core first
|
||||
if (find(packages, 'core') != null) {
|
||||
try {
|
||||
var lib = Build.build_dynamic('core', target, buildtype)
|
||||
push(results, { package: 'core', library: lib })
|
||||
} catch (e) {
|
||||
log.error('Failed to build core: ' + text(e))
|
||||
push(results, { package: 'core', error: e })
|
||||
}
|
||||
if (find(packages, function(p) { return p == 'core' }) != null) {
|
||||
var core_mods = Build.build_dynamic('core', _target, _buildtype)
|
||||
push(results, {package: 'core', modules: core_mods})
|
||||
}
|
||||
|
||||
|
||||
// Build other packages
|
||||
arrfor(packages, function(pkg) {
|
||||
if (pkg == 'core') return
|
||||
|
||||
try {
|
||||
var lib = Build.build_dynamic(pkg, target, buildtype)
|
||||
push(results, { package: pkg, library: lib })
|
||||
} catch (e) {
|
||||
log.error('Failed to build ' + pkg + ': ')
|
||||
log.console(e.message)
|
||||
log.console(e.stack)
|
||||
push(results, { package: pkg, error: e })
|
||||
}
|
||||
var pkg_mods = Build.build_dynamic(pkg, _target, _buildtype)
|
||||
push(results, {package: pkg, modules: pkg_mods})
|
||||
})
|
||||
|
||||
|
||||
return results
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user