john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge branch 'master' into fix_aot

Browse Source
This commit is contained in:
John Alanbrook
2026-02-18 23:55:17 -06:00
parent e004b2c472 777474ab4f
commit e59bfe19f7
23 changed files with 682 additions and 649 deletions
Download Patch File Download Diff File Expand all files Collapse all files

279
build.cm
View File

@@ -80,9 +80,6 @@ function content_hash(str) {
return text(crypto.blake2(bb, 32), 'h')
}
// Bump when native codegen/runtime ABI changes so stale dylibs are not reused.
def NATIVE_CACHE_VERSION = "native-v23"
// Enable AOT ASan by creating .cell/asan_aot in the package root.
function native_sanitize_flags() {
if (fd.is_file('.cell/asan_aot')) {
@@ -91,6 +88,32 @@ function native_sanitize_flags() {
return ''
}
// ============================================================================
// Cache key salts — canonical registry
// Every artifact type has a unique salt so hash collisions between types
// are impossible, and no file extensions are needed in build/.
// ============================================================================
var SALT_OBJ = 'obj' // compiled C object file
var SALT_DYLIB = 'dylib' // linked dynamic library
var SALT_NATIVE = 'native' // native-compiled .cm dylib
var SALT_MACH = 'mach' // mach bytecode blob
var SALT_MCODE = 'mcode' // mcode IR (JSON)
var SALT_DEPS = 'deps' // cached cc -MM dependency list
var SALT_FAIL = 'fail' // cached compilation failure
function cache_path(content, salt) {
return get_build_dir() + '/' + content_hash(content + '\n' + salt)
}
// Deterministic manifest path for a package's built dylibs
function manifest_path(pkg) {
return get_build_dir() + '/' + content_hash(pkg + '\n' + 'manifest')
}
function native_cache_content(src, target, san_flags) {
return src + '\n' + target + '\nnative\n' + (san_flags || '')
}
function get_build_dir() {
return shop.get_build_dir()
}
@@ -109,6 +132,52 @@ function ensure_dir(path) {
Build.ensure_dir = ensure_dir
// ============================================================================
// Dependency scanning helpers
// ============================================================================
// Parse make-style dependency output:
// foo.o: foo.c header1.h \
// header2.h
// Returns array of dependency file paths (skips the target)
function parse_makefile_deps(dep_text) {
var joined = replace(dep_text, /\\\n\s*/, ' ')
var colon_pos = search(joined, ':')
if (colon_pos == null) return []
var rest = trim(text(joined, colon_pos + 1))
var parts = filter(array(rest, /\s+/), function(p) {
return length(p) > 0
})
return parts
}
// Run cc -MM to get the preprocessor dependency list.
// Returns array of dependency file paths.
function get_c_deps(cc, flags, src_path) {
var dep_file = '/tmp/cell_deps_' + content_hash(src_path) + '.d'
var dep_cmd = [cc, '-MM', '-MG', '-MF', '"' + dep_file + '"']
dep_cmd = array(dep_cmd, flags)
push(dep_cmd, '"' + src_path + '"')
var ret = os.system(text(dep_cmd, ' ') + ' 2>/dev/null')
if (ret != 0) return [src_path]
if (!fd.is_file(dep_file)) return [src_path]
var dep_text = text(fd.slurp(dep_file))
return parse_makefile_deps(dep_text)
}
// Build a full hash string from the compilation command and all dependency
// file contents. This is the content key for the object file.
function hash_all_deps(cmd_str, deps) {
var parts = [cmd_str]
arrfor(deps, function(dep_path) {
if (fd.is_file(dep_path))
push(parts, dep_path + '\n' + text(fd.slurp(dep_path)))
else
push(parts, dep_path + '\n<missing>')
})
return text(parts, '\n')
}
// ============================================================================
// Compilation
// ============================================================================
@@ -135,30 +204,30 @@ Build.compile_file = function(pkg, file, target, opts) {
// Symbol name for this file
var sym_name = shop.c_symbol_for_file(pkg, file)
// Build command
var cmd_parts = [cc, '-c', '-fPIC']
// Build common flags (shared between dep scan and compilation)
var common_flags = []
// Add buildtype-specific flags
if (_buildtype == 'release') {
cmd_parts = array(cmd_parts, ['-O3', '-DNDEBUG'])
common_flags = array(common_flags, ['-O3', '-DNDEBUG'])
} else if (_buildtype == 'debug') {
cmd_parts = array(cmd_parts, ['-O2', '-g'])
common_flags = array(common_flags, ['-O2', '-g'])
} else if (_buildtype == 'minsize') {
cmd_parts = array(cmd_parts, ['-Os', '-DNDEBUG'])
common_flags = array(common_flags, ['-Os', '-DNDEBUG'])
}
push(cmd_parts, '-DCELL_USE_NAME=' + sym_name)
push(cmd_parts, '-I"' + pkg_dir + '"')
push(common_flags, '-DCELL_USE_NAME=' + sym_name)
push(common_flags, '-I"' + pkg_dir + '"')
// Auto-discover include/ directory
if (fd.is_dir(pkg_dir + '/include')) {
push(cmd_parts, '-I"' + pkg_dir + '/include"')
push(common_flags, '-I"' + pkg_dir + '/include"')
}
// External packages need core's source dir for cell.h, quickjs.h, blob.h
if (pkg != 'core') {
core_dir = shop.get_package_dir('core')
push(cmd_parts, '-I"' + core_dir + '/source"')
push(common_flags, '-I"' + core_dir + '/source"')
}
// Add package CFLAGS (resolve relative -I paths)
@@ -171,16 +240,19 @@ Build.compile_file = function(pkg, file, target, opts) {
f = '-I"' + pkg_dir + '/' + ipath + '"'
}
}
push(cmd_parts, f)
push(common_flags, f)
})
// Add target CFLAGS
arrfor(target_cflags, function(flag) {
push(cmd_parts, flag)
push(common_flags, flag)
})
// Build full compilation command
var cmd_parts = [cc, '-c', '-fPIC']
cmd_parts = array(cmd_parts, common_flags)
push(cmd_parts, '"' + src_path + '"')
var cmd_str = text(cmd_parts, ' ')
if (_opts.verbose) {
@@ -188,30 +260,64 @@ Build.compile_file = function(pkg, file, target, opts) {
print('[verbose] compile: ' + cmd_str)
}
// Content hash: command + file content
// Two-level cache: quick hash for deps file, full hash for object
var file_content = fd.slurp(src_path)
var hash_input = cmd_str + '\n' + text(file_content)
var hash = content_hash(hash_input)
var quick_content = cmd_str + '\n' + text(file_content)
var deps_path = cache_path(quick_content, SALT_DEPS)
var fail_path = cache_path(quick_content, SALT_FAIL)
var build_dir = get_build_dir()
ensure_dir(build_dir)
var obj_path = build_dir + '/' + hash
// Check if already compiled
// Check for cached failure (skip files that previously failed to compile)
if (fd.is_file(fail_path)) {
if (_opts.verbose) print('[verbose] skipping ' + file + ' (cached failure)')
log.shop('skip ' + file + ' (cached failure)')
return null
}
var deps = null
var full_content = null
var obj_path = null
// Warm path: read cached dep list, verify by hashing all deps
if (fd.is_file(deps_path)) {
deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) {
return length(p) > 0
})
full_content = hash_all_deps(cmd_str, deps)
obj_path = cache_path(full_content, SALT_OBJ)
if (fd.is_file(obj_path)) {
if (_opts.verbose) print('[verbose] cache hit: ' + file)
log.shop('cache hit ' + file)
return obj_path
}
log.shop('cache stale ' + file + ' (header changed)')
}
// Cold path: run cc -MM to discover deps
log.shop('dep scan ' + file)
deps = get_c_deps(cc, common_flags, src_path)
full_content = hash_all_deps(cmd_str, deps)
obj_path = cache_path(full_content, SALT_OBJ)
// Check if object exists (might exist from previous build with same deps)
if (fd.is_file(obj_path)) {
if (_opts.verbose) print('[verbose] cache hit: ' + file)
fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
if (_opts.verbose) print('[verbose] cache hit: ' + file + ' (after dep scan)')
log.shop('cache hit ' + file + ' (after dep scan)')
return obj_path
}
if (_opts.verbose) print('[verbose] cache miss: ' + file)
// Compile — capture stderr to detect missing-header vs real errors
var err_path = '/tmp/cell_build_err_' + hash + '.log'
// Compile
log.shop('compiling ' + file)
log.console('Compiling ' + file)
var err_path = '/tmp/cell_build_err_' + content_hash(src_path) + '.log'
var full_cmd = cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"'
var err_text = null
var missing = null
var err_lines = null
var first_err = null
log.console('Compiling ' + file)
var ret = os.system(full_cmd)
if (ret != 0) {
if (fd.is_file(err_path)) {
@@ -230,9 +336,13 @@ Build.compile_file = function(pkg, file, target, opts) {
if (err_text) print(err_text)
else print('Command: ' + full_cmd)
}
// Cache the failure so we don't retry on every build
fd.slurpwrite(fail_path, stone(blob(err_text || 'compilation failed')))
return null
}
// Save deps for future warm-path lookups
fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
return obj_path
}
@@ -260,27 +370,27 @@ Build.build_package = function(pkg, target, exclude_main, buildtype) {
// Dynamic library building
// ============================================================================
// Compute link key from all inputs that affect the dylib output
function compute_link_key(objects, ldflags, target_ldflags, opts) {
// Compute link content string from all inputs that affect the dylib output
function compute_link_content(objects, ldflags, target_ldflags, opts) {
// Sort objects for deterministic hash
var sorted_objects = sort(objects)
// Build a string representing all link inputs
var parts = []
push(parts, 'target:' + opts.target)
push(parts, 'cc:' + opts.cc)
push(parts, 'target:' + text(opts.target))
push(parts, 'cc:' + text(opts.cc))
arrfor(sorted_objects, function(obj) {
// Object paths are content-addressed, so the path itself is the hash
push(parts, 'obj:' + obj)
push(parts, 'obj:' + text(obj))
})
arrfor(ldflags, function(flag) {
push(parts, 'ldflag:' + flag)
push(parts, 'ldflag:' + text(flag))
})
arrfor(target_ldflags, function(flag) {
push(parts, 'target_ldflag:' + flag)
push(parts, 'target_ldflag:' + text(flag))
})
return content_hash(text(parts, '\n'))
return text(parts, '\n')
}
// Build a per-module dynamic library for a single C file
@@ -294,7 +404,6 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
if (!obj) return null
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.cpp || tc.c
var local_dir = get_local_dir()
var pkg_dir = shop.get_package_dir(pkg)
@@ -318,10 +427,10 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
// Content-addressed output: hash of (all objects + link flags + target)
var all_objects = [obj]
all_objects = array(all_objects, _extra)
var link_key = compute_link_key(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
var link_content = compute_link_content(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + link_key + '.' + _target + dylib_ext
var dylib_path = cache_path(link_content, SALT_DYLIB)
var cmd_parts = null
var cmd_str = null
var ret = null
@@ -352,9 +461,9 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
}
push(cmd_parts, '-L"' + local_dir + '"')
push(cmd_parts, '"' + obj + '"')
push(cmd_parts, '"' + text(obj) + '"')
arrfor(_extra, function(extra_obj) {
push(cmd_parts, '"' + extra_obj + '"')
if (extra_obj != null) push(cmd_parts, '"' + text(extra_obj) + '"')
})
cmd_parts = array(cmd_parts, resolved_ldflags)
cmd_parts = array(cmd_parts, target_ldflags)
@@ -363,29 +472,17 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
cmd_str = text(cmd_parts, ' ')
if (_opts.verbose) print('[verbose] link: ' + cmd_str)
log.shop('linking ' + file)
log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path))
ret = os.system(cmd_str)
if (ret != 0) {
print('Linking failed: ' + file)
return null
}
} else {
log.shop('link cache hit ' + file)
}
// Always install to deterministic lib/<pkg>/<stem>.dylib
// Strip .c/.cpp extension so the loader can find it by module name
var file_stem = file
if (ends_with(file_stem, '.cpp')) file_stem = text(file_stem, 0, -4)
else if (ends_with(file_stem, '.c')) file_stem = text(file_stem, 0, -2)
var install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
var stem_dir = fd.dirname(file_stem)
if (stem_dir && stem_dir != '.') {
install_dir = install_dir + '/' + stem_dir
}
ensure_dir(install_dir)
var install_path = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) + '/' + file_stem + dylib_ext
fd.slurpwrite(install_path, fd.slurp(dylib_path))
if (_opts.verbose) print('[verbose] install: ' + install_path)
return dylib_path
}
@@ -406,10 +503,12 @@ Build.build_dynamic = function(pkg, target, buildtype, opts) {
// Compile support sources to cached objects
var sources = pkg_tools.get_sources(pkg)
var support_objects = []
arrfor(sources, function(src_file) {
var obj = Build.compile_file(pkg, src_file, _target, {buildtype: _buildtype, cflags: cached_cflags, verbose: _opts.verbose})
push(support_objects, obj)
})
if (pkg != 'core') {
arrfor(sources, function(src_file) {
var obj = Build.compile_file(pkg, src_file, _target, {buildtype: _buildtype, cflags: cached_cflags, verbose: _opts.verbose})
if (obj != null) push(support_objects, obj)
})
}
arrfor(c_files, function(file) {
var sym_name = shop.c_symbol_for_file(pkg, file)
@@ -419,6 +518,11 @@ Build.build_dynamic = function(pkg, target, buildtype, opts) {
}
})
// Write manifest so runtime can find dylibs without the build module
var json = use('json')
var mpath = manifest_path(pkg)
fd.slurpwrite(mpath, stone(blob(json.encode(results))))
return results
}
@@ -573,16 +677,12 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
var _target = target || Build.detect_host_target()
var _buildtype = buildtype || 'release'
var qbe_rt_path = null
var native_stem = null
var native_install_dir = null
var native_install_path = null
if (!fd.is_file(src_path)) {
print('Source file not found: ' + src_path); disrupt
}
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.c
var san_flags = native_sanitize_flags()
var san_suffix = length(san_flags) > 0 ? '_asan' : ''
@@ -600,16 +700,17 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
var il_parts = qbe_emit(optimized, qbe_macros, sym_name)
// Content hash for cache key
var hash = content_hash(text(fd.slurp(src_path)) + '\n' + _target + '\nnative\n' + NATIVE_CACHE_VERSION + '\n' + san_flags)
var src = text(fd.slurp(src_path))
var native_key = native_cache_content(src, _target, san_flags)
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
var dylib_path = cache_path(native_key, SALT_NATIVE)
if (fd.is_file(dylib_path))
return dylib_path
// Compile and assemble via batched parallel pipeline
var tmp = '/tmp/cell_native_' + hash
var tmp = '/tmp/cell_native_' + content_hash(native_key)
var rt_o_path = '/tmp/cell_qbe_rt' + san_suffix + '.o'
var o_paths = compile_native_single(il_parts, cc, tmp, san_flags)
@@ -645,15 +746,6 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
log.console('Built native: ' + fd.basename(dylib_path))
// Install to deterministic lib/<pkg>/<stem>.dylib
if (pkg) {
native_stem = fd.basename(src_path)
native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
ensure_dir(native_install_dir)
native_install_path = native_install_dir + '/' + native_stem + dylib_ext
fd.slurpwrite(native_install_path, fd.slurp(dylib_path))
}
return dylib_path
}
@@ -665,12 +757,8 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
var _buildtype = (opts && opts.buildtype) || 'release'
var pkg = opts && opts.pkg
var qbe_rt_path = null
var native_stem = null
var native_install_dir = null
var native_install_path = null
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.c
var san_flags = native_sanitize_flags()
var san_suffix = length(san_flags) > 0 ? '_asan' : ''
@@ -685,16 +773,16 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
var il_parts = qbe_emit(optimized, qbe_macros, sym_name)
var src = text(fd.slurp(src_path))
var hash = content_hash(src + '\n' + _target + '\nnative\n' + NATIVE_CACHE_VERSION + '\n' + san_flags)
var native_key = native_cache_content(src, _target, san_flags)
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
var dylib_path = cache_path(native_key, SALT_NATIVE)
if (fd.is_file(dylib_path))
return dylib_path
// Compile and assemble via batched parallel pipeline
var tmp = '/tmp/cell_native_' + hash
var tmp = '/tmp/cell_native_' + content_hash(native_key)
var rt_o_path = '/tmp/cell_qbe_rt' + san_suffix + '.o'
var o_paths = compile_native_single(il_parts, cc, tmp, san_flags)
@@ -730,14 +818,6 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
log.console('Built native: ' + fd.basename(dylib_path))
if (pkg) {
native_stem = fd.basename(src_path)
native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
ensure_dir(native_install_dir)
native_install_path = native_install_dir + '/' + native_stem + dylib_ext
fd.slurpwrite(native_install_path, fd.slurp(dylib_path))
}
return dylib_path
}
@@ -859,4 +939,17 @@ Build.build_all_dynamic = function(target, buildtype, opts) {
return results
}
// Export salt constants and cache_path for shop.cm and others
Build.SALT_OBJ = SALT_OBJ
Build.SALT_DYLIB = SALT_DYLIB
Build.SALT_NATIVE = SALT_NATIVE
Build.SALT_MACH = SALT_MACH
Build.SALT_MCODE = SALT_MCODE
Build.SALT_DEPS = SALT_DEPS
Build.SALT_FAIL = SALT_FAIL
Build.cache_path = cache_path
Build.manifest_path = manifest_path
Build.native_sanitize_flags = native_sanitize_flags
Build.native_cache_content = native_cache_content
return Build