john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

better cache handling

Browse Source
This commit is contained in:
John Alanbrook
2026-02-18 19:27:28 -06:00
parent dc70a15981
commit 6bc9dd53a7
6 changed files with 238 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
build.cm
View File

@@ -80,6 +80,22 @@ function content_hash(str) {
return text(crypto.blake2(bb, 32), 'h')
}
// ============================================================================
// Cache key salts — canonical registry
// Every artifact type has a unique salt so hash collisions between types
// are impossible, and no file extensions are needed in build/.
// ============================================================================
var SALT_OBJ = 'obj' // compiled C object file
var SALT_DYLIB = 'dylib' // linked dynamic library
var SALT_NATIVE = 'native' // native-compiled .cm dylib
var SALT_MACH = 'mach' // mach bytecode blob
var SALT_MCODE = 'mcode' // mcode IR (JSON)
var SALT_DEPS = 'deps' // cached cc -MM dependency list
function cache_path(content, salt) {
return get_build_dir() + '/' + content_hash(content + '\n' + salt)
}
function get_build_dir() {
return shop.get_build_dir()
}
@@ -98,6 +114,52 @@ function ensure_dir(path) {
Build.ensure_dir = ensure_dir
// ============================================================================
// Dependency scanning helpers
// ============================================================================
// Parse make-style dependency output:
// foo.o: foo.c header1.h \
// header2.h
// Returns array of dependency file paths (skips the target)
function parse_makefile_deps(dep_text) {
var joined = replace(dep_text, /\\\n\s*/, ' ')
var colon_pos = search(joined, ':')
if (colon_pos == null) return []
var rest = trim(text(joined, colon_pos + 1))
var parts = filter(array(rest, /\s+/), function(p) {
return length(p) > 0
})
return parts
}
// Run cc -MM to get the preprocessor dependency list.
// Returns array of dependency file paths.
function get_c_deps(cc, flags, src_path) {
var dep_file = '/tmp/cell_deps_' + content_hash(src_path) + '.d'
var dep_cmd = [cc, '-MM', '-MG', '-MF', '"' + dep_file + '"']
dep_cmd = array(dep_cmd, flags)
push(dep_cmd, '"' + src_path + '"')
var ret = os.system(text(dep_cmd, ' ') + ' 2>/dev/null')
if (ret != 0) return [src_path]
if (!fd.is_file(dep_file)) return [src_path]
var dep_text = text(fd.slurp(dep_file))
return parse_makefile_deps(dep_text)
}
// Build a full hash string from the compilation command and all dependency
// file contents. This is the content key for the object file.
function hash_all_deps(cmd_str, deps) {
var parts = [cmd_str]
arrfor(deps, function(dep_path) {
if (fd.is_file(dep_path))
push(parts, dep_path + '\n' + text(fd.slurp(dep_path)))
else
push(parts, dep_path + '\n<missing>')
})
return text(parts, '\n')
}
// ============================================================================
// Compilation
// ============================================================================
@@ -124,30 +186,30 @@ Build.compile_file = function(pkg, file, target, opts) {
// Symbol name for this file
var sym_name = shop.c_symbol_for_file(pkg, file)
// Build command
var cmd_parts = [cc, '-c', '-fPIC']
// Build common flags (shared between dep scan and compilation)
var common_flags = []
// Add buildtype-specific flags
if (_buildtype == 'release') {
cmd_parts = array(cmd_parts, ['-O3', '-DNDEBUG'])
common_flags = array(common_flags, ['-O3', '-DNDEBUG'])
} else if (_buildtype == 'debug') {
cmd_parts = array(cmd_parts, ['-O2', '-g'])
common_flags = array(common_flags, ['-O2', '-g'])
} else if (_buildtype == 'minsize') {
cmd_parts = array(cmd_parts, ['-Os', '-DNDEBUG'])
common_flags = array(common_flags, ['-Os', '-DNDEBUG'])
}
push(cmd_parts, '-DCELL_USE_NAME=' + sym_name)
push(cmd_parts, '-I"' + pkg_dir + '"')
push(common_flags, '-DCELL_USE_NAME=' + sym_name)
push(common_flags, '-I"' + pkg_dir + '"')
// Auto-discover include/ directory
if (fd.is_dir(pkg_dir + '/include')) {
push(cmd_parts, '-I"' + pkg_dir + '/include"')
push(common_flags, '-I"' + pkg_dir + '/include"')
}
// External packages need core's source dir for cell.h, quickjs.h, blob.h
if (pkg != 'core') {
core_dir = shop.get_package_dir('core')
push(cmd_parts, '-I"' + core_dir + '/source"')
push(common_flags, '-I"' + core_dir + '/source"')
}
// Add package CFLAGS (resolve relative -I paths)
@@ -160,16 +222,19 @@ Build.compile_file = function(pkg, file, target, opts) {
f = '-I"' + pkg_dir + '/' + ipath + '"'
}
}
push(cmd_parts, f)
push(common_flags, f)
})
// Add target CFLAGS
arrfor(target_cflags, function(flag) {
push(cmd_parts, flag)
push(common_flags, flag)
})
// Build full compilation command
var cmd_parts = [cc, '-c', '-fPIC']
cmd_parts = array(cmd_parts, common_flags)
push(cmd_parts, '"' + src_path + '"')
var cmd_str = text(cmd_parts, ' ')
if (_opts.verbose) {
@@ -177,30 +242,56 @@ Build.compile_file = function(pkg, file, target, opts) {
print('[verbose] compile: ' + cmd_str)
}
// Content hash: command + file content
// Two-level cache: quick hash for deps file, full hash for object
var file_content = fd.slurp(src_path)
var hash_input = cmd_str + '\n' + text(file_content)
var hash = content_hash(hash_input)
var quick_content = cmd_str + '\n' + text(file_content)
var deps_path = cache_path(quick_content, SALT_DEPS)
var build_dir = get_build_dir()
ensure_dir(build_dir)
var obj_path = build_dir + '/' + hash
// Check if already compiled
var deps = null
var full_content = null
var obj_path = null
// Warm path: read cached dep list, verify by hashing all deps
if (fd.is_file(deps_path)) {
deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) {
return length(p) > 0
})
full_content = hash_all_deps(cmd_str, deps)
obj_path = cache_path(full_content, SALT_OBJ)
if (fd.is_file(obj_path)) {
if (_opts.verbose) print('[verbose] cache hit: ' + file)
log.shop('cache hit ' + file)
return obj_path
}
log.shop('cache stale ' + file + ' (header changed)')
}
// Cold path: run cc -MM to discover deps
log.shop('dep scan ' + file)
deps = get_c_deps(cc, common_flags, src_path)
full_content = hash_all_deps(cmd_str, deps)
obj_path = cache_path(full_content, SALT_OBJ)
// Check if object exists (might exist from previous build with same deps)
if (fd.is_file(obj_path)) {
if (_opts.verbose) print('[verbose] cache hit: ' + file)
fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
if (_opts.verbose) print('[verbose] cache hit: ' + file + ' (after dep scan)')
log.shop('cache hit ' + file + ' (after dep scan)')
return obj_path
}
if (_opts.verbose) print('[verbose] cache miss: ' + file)
// Compile — capture stderr to detect missing-header vs real errors
var err_path = '/tmp/cell_build_err_' + hash + '.log'
// Compile
log.shop('compiling ' + file)
log.console('Compiling ' + file)
var err_path = '/tmp/cell_build_err_' + content_hash(src_path) + '.log'
var full_cmd = cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"'
var err_text = null
var missing = null
var err_lines = null
var first_err = null
log.console('Compiling ' + file)
var ret = os.system(full_cmd)
if (ret != 0) {
if (fd.is_file(err_path)) {
@@ -222,6 +313,8 @@ Build.compile_file = function(pkg, file, target, opts) {
return null
}
// Save deps for future warm-path lookups
fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
return obj_path
}
@@ -249,8 +342,8 @@ Build.build_package = function(pkg, target, exclude_main, buildtype) {
// Dynamic library building
// ============================================================================
// Compute link key from all inputs that affect the dylib output
function compute_link_key(objects, ldflags, target_ldflags, opts) {
// Compute link content string from all inputs that affect the dylib output
function compute_link_content(objects, ldflags, target_ldflags, opts) {
// Sort objects for deterministic hash
var sorted_objects = sort(objects)
@@ -269,7 +362,7 @@ function compute_link_key(objects, ldflags, target_ldflags, opts) {
push(parts, 'target_ldflag:' + flag)
})
return content_hash(text(parts, '\n'))
return text(parts, '\n')
}
// Build a per-module dynamic library for a single C file
@@ -283,7 +376,6 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
if (!obj) return null
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.cpp || tc.c
var local_dir = get_local_dir()
var pkg_dir = shop.get_package_dir(pkg)
@@ -307,10 +399,10 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
// Content-addressed output: hash of (all objects + link flags + target)
var all_objects = [obj]
all_objects = array(all_objects, _extra)
var link_key = compute_link_key(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
var link_content = compute_link_content(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + link_key + '.' + _target + dylib_ext
var dylib_path = cache_path(link_content, SALT_DYLIB)
var cmd_parts = null
var cmd_str = null
var ret = null
@@ -352,29 +444,17 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
cmd_str = text(cmd_parts, ' ')
if (_opts.verbose) print('[verbose] link: ' + cmd_str)
log.shop('linking ' + file)
log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path))
ret = os.system(cmd_str)
if (ret != 0) {
print('Linking failed: ' + file)
return null
}
} else {
log.shop('link cache hit ' + file)
}
// Always install to deterministic lib/<pkg>/<stem>.dylib
// Strip .c/.cpp extension so the loader can find it by module name
var file_stem = file
if (ends_with(file_stem, '.cpp')) file_stem = text(file_stem, 0, -4)
else if (ends_with(file_stem, '.c')) file_stem = text(file_stem, 0, -2)
var install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
var stem_dir = fd.dirname(file_stem)
if (stem_dir && stem_dir != '.') {
install_dir = install_dir + '/' + stem_dir
}
ensure_dir(install_dir)
var install_path = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) + '/' + file_stem + dylib_ext
fd.slurpwrite(install_path, fd.slurp(dylib_path))
if (_opts.verbose) print('[verbose] install: ' + install_path)
return dylib_path
}
@@ -602,16 +682,12 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
var _target = target || Build.detect_host_target()
var _buildtype = buildtype || 'release'
var qbe_rt_path = null
var native_stem = null
var native_install_dir = null
var native_install_path = null
if (!fd.is_file(src_path)) {
print('Source file not found: ' + src_path); disrupt
}
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.c
// Step 1: Compile through pipeline
@@ -627,16 +703,15 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
var il_parts = qbe_emit(optimized, qbe_macros, sym_name)
// Content hash for cache key
var hash = content_hash(text(fd.slurp(src_path)) + '\n' + _target + '\nnative')
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
var dylib_path = cache_path(text(fd.slurp(src_path)) + '\n' + _target, SALT_NATIVE)
if (fd.is_file(dylib_path))
return dylib_path
// Compile and assemble via batched parallel pipeline
var tmp = '/tmp/cell_native_' + hash
var tmp = '/tmp/cell_native_' + content_hash(src_path)
var rt_o_path = '/tmp/cell_qbe_rt.o'
var o_paths = compile_native_batched(il_parts, cc, tmp)
@@ -672,15 +747,6 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
log.console('Built native: ' + fd.basename(dylib_path))
// Install to deterministic lib/<pkg>/<stem>.dylib
if (pkg) {
native_stem = fd.basename(src_path)
native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
ensure_dir(native_install_dir)
native_install_path = native_install_dir + '/' + native_stem + dylib_ext
fd.slurpwrite(native_install_path, fd.slurp(dylib_path))
}
return dylib_path
}
@@ -692,12 +758,8 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
var _buildtype = (opts && opts.buildtype) || 'release'
var pkg = opts && opts.pkg
var qbe_rt_path = null
var native_stem = null
var native_install_dir = null
var native_install_path = null
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.c
var qbe_macros = use('qbe')
@@ -710,16 +772,15 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
var il_parts = qbe_emit(optimized, qbe_macros, sym_name)
var src = text(fd.slurp(src_path))
var hash = content_hash(src + '\n' + _target + '\nnative')
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
var dylib_path = cache_path(src + '\n' + _target, SALT_NATIVE)
if (fd.is_file(dylib_path))
return dylib_path
// Compile and assemble via batched parallel pipeline
var tmp = '/tmp/cell_native_' + hash
var tmp = '/tmp/cell_native_' + content_hash(src_path)
var rt_o_path = '/tmp/cell_qbe_rt.o'
var o_paths = compile_native_batched(il_parts, cc, tmp)
@@ -755,14 +816,6 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
log.console('Built native: ' + fd.basename(dylib_path))
if (pkg) {
native_stem = fd.basename(src_path)
native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
ensure_dir(native_install_dir)
native_install_path = native_install_dir + '/' + native_stem + dylib_ext
fd.slurpwrite(native_install_path, fd.slurp(dylib_path))
}
return dylib_path
}
@@ -884,4 +937,13 @@ Build.build_all_dynamic = function(target, buildtype, opts) {
return results
}
// Export salt constants and cache_path for shop.cm and others
Build.SALT_OBJ = SALT_OBJ
Build.SALT_DYLIB = SALT_DYLIB
Build.SALT_NATIVE = SALT_NATIVE
Build.SALT_MACH = SALT_MACH
Build.SALT_MCODE = SALT_MCODE
Build.SALT_DEPS = SALT_DEPS
Build.cache_path = cache_path
return Build