john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge branch 'dylib_cache'

Browse Source
This commit is contained in:
John Alanbrook
2026-02-18 19:27:33 -06:00
parent f7499c4f60 6bc9dd53a7
commit 14a94aff12
6 changed files with 238 additions and 271 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
build.cm
View File

@@ -80,6 +80,22 @@ function content_hash(str) {
return text(crypto.blake2(bb, 32), 'h')
}
// ============================================================================
// Cache key salts — canonical registry
// Every artifact type has a unique salt so hash collisions between types
// are impossible, and no file extensions are needed in build/.
// ============================================================================
var SALT_OBJ = 'obj' // compiled C object file
var SALT_DYLIB = 'dylib' // linked dynamic library
var SALT_NATIVE = 'native' // native-compiled .cm dylib
var SALT_MACH = 'mach' // mach bytecode blob
var SALT_MCODE = 'mcode' // mcode IR (JSON)
var SALT_DEPS = 'deps' // cached cc -MM dependency list
function cache_path(content, salt) {
return get_build_dir() + '/' + content_hash(content + '\n' + salt)
}
function get_build_dir() {
return shop.get_build_dir()
}
@@ -98,6 +114,52 @@ function ensure_dir(path) {
Build.ensure_dir = ensure_dir
// ============================================================================
// Dependency scanning helpers
// ============================================================================
// Parse make-style dependency output:
// foo.o: foo.c header1.h \
// header2.h
// Returns array of dependency file paths (skips the target)
function parse_makefile_deps(dep_text) {
var joined = replace(dep_text, /\\\n\s*/, ' ')
var colon_pos = search(joined, ':')
if (colon_pos == null) return []
var rest = trim(text(joined, colon_pos + 1))
var parts = filter(array(rest, /\s+/), function(p) {
return length(p) > 0
})
return parts
}
// Run cc -MM to get the preprocessor dependency list.
// Returns array of dependency file paths.
function get_c_deps(cc, flags, src_path) {
var dep_file = '/tmp/cell_deps_' + content_hash(src_path) + '.d'
var dep_cmd = [cc, '-MM', '-MG', '-MF', '"' + dep_file + '"']
dep_cmd = array(dep_cmd, flags)
push(dep_cmd, '"' + src_path + '"')
var ret = os.system(text(dep_cmd, ' ') + ' 2>/dev/null')
if (ret != 0) return [src_path]
if (!fd.is_file(dep_file)) return [src_path]
var dep_text = text(fd.slurp(dep_file))
return parse_makefile_deps(dep_text)
}
// Build a full hash string from the compilation command and all dependency
// file contents. This is the content key for the object file.
function hash_all_deps(cmd_str, deps) {
var parts = [cmd_str]
arrfor(deps, function(dep_path) {
if (fd.is_file(dep_path))
push(parts, dep_path + '\n' + text(fd.slurp(dep_path)))
else
push(parts, dep_path + '\n<missing>')
})
return text(parts, '\n')
}
// ============================================================================
// Compilation
// ============================================================================
@@ -124,30 +186,30 @@ Build.compile_file = function(pkg, file, target, opts) {
// Symbol name for this file
var sym_name = shop.c_symbol_for_file(pkg, file)
// Build command
var cmd_parts = [cc, '-c', '-fPIC']
// Build common flags (shared between dep scan and compilation)
var common_flags = []
// Add buildtype-specific flags
if (_buildtype == 'release') {
cmd_parts = array(cmd_parts, ['-O3', '-DNDEBUG'])
common_flags = array(common_flags, ['-O3', '-DNDEBUG'])
} else if (_buildtype == 'debug') {
cmd_parts = array(cmd_parts, ['-O2', '-g'])
common_flags = array(common_flags, ['-O2', '-g'])
} else if (_buildtype == 'minsize') {
cmd_parts = array(cmd_parts, ['-Os', '-DNDEBUG'])
common_flags = array(common_flags, ['-Os', '-DNDEBUG'])
}
push(cmd_parts, '-DCELL_USE_NAME=' + sym_name)
push(cmd_parts, '-I"' + pkg_dir + '"')
push(common_flags, '-DCELL_USE_NAME=' + sym_name)
push(common_flags, '-I"' + pkg_dir + '"')
// Auto-discover include/ directory
if (fd.is_dir(pkg_dir + '/include')) {
push(cmd_parts, '-I"' + pkg_dir + '/include"')
push(common_flags, '-I"' + pkg_dir + '/include"')
}
// External packages need core's source dir for cell.h, quickjs.h, blob.h
if (pkg != 'core') {
core_dir = shop.get_package_dir('core')
push(cmd_parts, '-I"' + core_dir + '/source"')
push(common_flags, '-I"' + core_dir + '/source"')
}
// Add package CFLAGS (resolve relative -I paths)
@@ -160,16 +222,19 @@ Build.compile_file = function(pkg, file, target, opts) {
f = '-I"' + pkg_dir + '/' + ipath + '"'
}
}
push(cmd_parts, f)
push(common_flags, f)
})
// Add target CFLAGS
arrfor(target_cflags, function(flag) {
push(cmd_parts, flag)
push(common_flags, flag)
})
// Build full compilation command
var cmd_parts = [cc, '-c', '-fPIC']
cmd_parts = array(cmd_parts, common_flags)
push(cmd_parts, '"' + src_path + '"')
var cmd_str = text(cmd_parts, ' ')
if (_opts.verbose) {
@@ -177,30 +242,56 @@ Build.compile_file = function(pkg, file, target, opts) {
print('[verbose] compile: ' + cmd_str)
}
// Content hash: command + file content
// Two-level cache: quick hash for deps file, full hash for object
var file_content = fd.slurp(src_path)
var hash_input = cmd_str + '\n' + text(file_content)
var hash = content_hash(hash_input)
var quick_content = cmd_str + '\n' + text(file_content)
var deps_path = cache_path(quick_content, SALT_DEPS)
var build_dir = get_build_dir()
ensure_dir(build_dir)
var obj_path = build_dir + '/' + hash
// Check if already compiled
var deps = null
var full_content = null
var obj_path = null
// Warm path: read cached dep list, verify by hashing all deps
if (fd.is_file(deps_path)) {
deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) {
return length(p) > 0
})
full_content = hash_all_deps(cmd_str, deps)
obj_path = cache_path(full_content, SALT_OBJ)
if (fd.is_file(obj_path)) {
if (_opts.verbose) print('[verbose] cache hit: ' + file)
log.shop('cache hit ' + file)
return obj_path
}
log.shop('cache stale ' + file + ' (header changed)')
}
// Cold path: run cc -MM to discover deps
log.shop('dep scan ' + file)
deps = get_c_deps(cc, common_flags, src_path)
full_content = hash_all_deps(cmd_str, deps)
obj_path = cache_path(full_content, SALT_OBJ)
// Check if object exists (might exist from previous build with same deps)
if (fd.is_file(obj_path)) {
if (_opts.verbose) print('[verbose] cache hit: ' + file)
fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
if (_opts.verbose) print('[verbose] cache hit: ' + file + ' (after dep scan)')
log.shop('cache hit ' + file + ' (after dep scan)')
return obj_path
}
if (_opts.verbose) print('[verbose] cache miss: ' + file)
// Compile — capture stderr to detect missing-header vs real errors
var err_path = '/tmp/cell_build_err_' + hash + '.log'
// Compile
log.shop('compiling ' + file)
log.console('Compiling ' + file)
var err_path = '/tmp/cell_build_err_' + content_hash(src_path) + '.log'
var full_cmd = cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"'
var err_text = null
var missing = null
var err_lines = null
var first_err = null
log.console('Compiling ' + file)
var ret = os.system(full_cmd)
if (ret != 0) {
if (fd.is_file(err_path)) {
@@ -222,6 +313,8 @@ Build.compile_file = function(pkg, file, target, opts) {
return null
}
// Save deps for future warm-path lookups
fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
return obj_path
}
@@ -249,8 +342,8 @@ Build.build_package = function(pkg, target, exclude_main, buildtype) {
// Dynamic library building
// ============================================================================
// Compute link key from all inputs that affect the dylib output
function compute_link_key(objects, ldflags, target_ldflags, opts) {
// Compute link content string from all inputs that affect the dylib output
function compute_link_content(objects, ldflags, target_ldflags, opts) {
// Sort objects for deterministic hash
var sorted_objects = sort(objects)
@@ -269,7 +362,7 @@ function compute_link_key(objects, ldflags, target_ldflags, opts) {
push(parts, 'target_ldflag:' + flag)
})
return content_hash(text(parts, '\n'))
return text(parts, '\n')
}
// Build a per-module dynamic library for a single C file
@@ -283,7 +376,6 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
if (!obj) return null
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.cpp || tc.c
var local_dir = get_local_dir()
var pkg_dir = shop.get_package_dir(pkg)
@@ -307,10 +399,10 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
// Content-addressed output: hash of (all objects + link flags + target)
var all_objects = [obj]
all_objects = array(all_objects, _extra)
var link_key = compute_link_key(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
var link_content = compute_link_content(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + link_key + '.' + _target + dylib_ext
var dylib_path = cache_path(link_content, SALT_DYLIB)
var cmd_parts = null
var cmd_str = null
var ret = null
@@ -352,29 +444,17 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
cmd_str = text(cmd_parts, ' ')
if (_opts.verbose) print('[verbose] link: ' + cmd_str)
log.shop('linking ' + file)
log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path))
ret = os.system(cmd_str)
if (ret != 0) {
print('Linking failed: ' + file)
return null
}
} else {
log.shop('link cache hit ' + file)
}
// Always install to deterministic lib/<pkg>/<stem>.dylib
// Strip .c/.cpp extension so the loader can find it by module name
var file_stem = file
if (ends_with(file_stem, '.cpp')) file_stem = text(file_stem, 0, -4)
else if (ends_with(file_stem, '.c')) file_stem = text(file_stem, 0, -2)
var install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
var stem_dir = fd.dirname(file_stem)
if (stem_dir && stem_dir != '.') {
install_dir = install_dir + '/' + stem_dir
}
ensure_dir(install_dir)
var install_path = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) + '/' + file_stem + dylib_ext
fd.slurpwrite(install_path, fd.slurp(dylib_path))
if (_opts.verbose) print('[verbose] install: ' + install_path)
return dylib_path
}
@@ -602,16 +682,12 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
var _target = target || Build.detect_host_target()
var _buildtype = buildtype || 'release'
var qbe_rt_path = null
var native_stem = null
var native_install_dir = null
var native_install_path = null
if (!fd.is_file(src_path)) {
print('Source file not found: ' + src_path); disrupt
}
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.c
// Step 1: Compile through pipeline
@@ -627,16 +703,15 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
var il_parts = qbe_emit(optimized, qbe_macros, sym_name)
// Content hash for cache key
var hash = content_hash(text(fd.slurp(src_path)) + '\n' + _target + '\nnative')
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
var dylib_path = cache_path(text(fd.slurp(src_path)) + '\n' + _target, SALT_NATIVE)
if (fd.is_file(dylib_path))
return dylib_path
// Compile and assemble via batched parallel pipeline
var tmp = '/tmp/cell_native_' + hash
var tmp = '/tmp/cell_native_' + content_hash(src_path)
var rt_o_path = '/tmp/cell_qbe_rt.o'
var o_paths = compile_native_batched(il_parts, cc, tmp)
@@ -672,15 +747,6 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
log.console('Built native: ' + fd.basename(dylib_path))
// Install to deterministic lib/<pkg>/<stem>.dylib
if (pkg) {
native_stem = fd.basename(src_path)
native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
ensure_dir(native_install_dir)
native_install_path = native_install_dir + '/' + native_stem + dylib_ext
fd.slurpwrite(native_install_path, fd.slurp(dylib_path))
}
return dylib_path
}
@@ -692,12 +758,8 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
var _buildtype = (opts && opts.buildtype) || 'release'
var pkg = opts && opts.pkg
var qbe_rt_path = null
var native_stem = null
var native_install_dir = null
var native_install_path = null
var tc = toolchains[_target]
var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
var cc = tc.c
var qbe_macros = use('qbe')
@@ -710,16 +772,15 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
var il_parts = qbe_emit(optimized, qbe_macros, sym_name)
var src = text(fd.slurp(src_path))
var hash = content_hash(src + '\n' + _target + '\nnative')
var build_dir = get_build_dir()
ensure_dir(build_dir)
var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
var dylib_path = cache_path(src + '\n' + _target, SALT_NATIVE)
if (fd.is_file(dylib_path))
return dylib_path
// Compile and assemble via batched parallel pipeline
var tmp = '/tmp/cell_native_' + hash
var tmp = '/tmp/cell_native_' + content_hash(src_path)
var rt_o_path = '/tmp/cell_qbe_rt.o'
var o_paths = compile_native_batched(il_parts, cc, tmp)
@@ -755,14 +816,6 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
log.console('Built native: ' + fd.basename(dylib_path))
if (pkg) {
native_stem = fd.basename(src_path)
native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
ensure_dir(native_install_dir)
native_install_path = native_install_dir + '/' + native_stem + dylib_ext
fd.slurpwrite(native_install_path, fd.slurp(dylib_path))
}
return dylib_path
}
@@ -884,4 +937,13 @@ Build.build_all_dynamic = function(target, buildtype, opts) {
return results
}
// Export salt constants and cache_path for shop.cm and others
Build.SALT_OBJ = SALT_OBJ
Build.SALT_DYLIB = SALT_DYLIB
Build.SALT_NATIVE = SALT_NATIVE
Build.SALT_MACH = SALT_MACH
Build.SALT_MCODE = SALT_MCODE
Build.SALT_DEPS = SALT_DEPS
Build.cache_path = cache_path
return Build

38
clean.ce
View File

@@ -119,37 +119,13 @@ var build_dir = shop.get_build_dir()
var packages_dir = replace(shop.get_package_dir(''), /\/$/, '') // Get base packages dir
if (clean_build) {
if (is_shop_scope) {
// Clean entire build and lib directories
if (fd.is_dir(build_dir)) {
push(dirs_to_delete, build_dir)
}
if (fd.is_dir(lib_dir)) {
push(dirs_to_delete, lib_dir)
}
} else {
// Clean specific package libraries
arrfor(packages_to_clean, function(p) {
if (p == 'core') return
var lib_name = shop.lib_name_for_package(p)
var dylib_ext = '.dylib'
var lib_path = lib_dir + '/' + lib_name + dylib_ext
if (fd.is_file(lib_path)) {
push(files_to_delete, lib_path)
}
// Also check for .so and .dll
var so_path = lib_dir + '/' + lib_name + '.so'
var dll_path = lib_dir + '/' + lib_name + '.dll'
if (fd.is_file(so_path)) {
push(files_to_delete, so_path)
}
if (fd.is_file(dll_path)) {
push(files_to_delete, dll_path)
}
})
// Nuke entire build cache (content-addressed, per-package clean impractical)
if (fd.is_dir(build_dir)) {
push(dirs_to_delete, build_dir)
}
// Clean orphaned lib/ directory if it exists (legacy)
if (fd.is_dir(lib_dir)) {
push(dirs_to_delete, lib_dir)
}
}

193
internal/shop.cm
View File

@@ -54,9 +54,10 @@ function ensure_dir(path) {
}
}
function hash_path(content)
function hash_path(content, salt)
{
return global_shop_path + '/build' + '/' + content_hash(content)
var s = salt || 'mach'
return global_shop_path + '/build/' + content_hash(stone(blob(text(content) + '\n' + s)))
}
var Shop = {}
@@ -418,6 +419,7 @@ Shop.extract_commit_hash = function(pkg, response) {
}
var open_dls = {}
var package_dylibs = {} // pkg -> [{file, symbol, dylib}, ...]
// Host target detection for native dylib resolution
function detect_host_target() {
@@ -434,37 +436,26 @@ function detect_host_target() {
var host_target = detect_host_target()
// Check for a native .cm dylib at the deterministic lib path
// Check for a native .cm dylib in the build cache
// Returns a native descriptor {_native, _handle, _sym}, or null if no native dylib exists
// Also checks staleness: if source has changed, the content-addressed build artifact
// won't exist for the new hash, so the installed dylib is treated as stale.
function try_native_mod_dylib(pkg, stem) {
var dylib_path = get_dylib_path(pkg, stem)
var src_path = null
var src = null
var host = null
var hash = null
var tc_ext = null
var build_path = null
var handle = null
var sym = null
var build_mod = use_cache['core/build']
if (!build_mod) return null
if (!fd.is_file(dylib_path)) return null
var src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem
if (!fd.is_file(src_path)) return null
// Staleness check: verify the content-addressed build artifact exists
src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem
if (fd.is_file(src_path)) {
src = text(fd.slurp(src_path))
host = detect_host_target()
hash = content_hash(src + '\n' + host + '\nnative')
tc_ext = dylib_ext
build_path = global_shop_path + '/build/' + hash + '.' + host + tc_ext
if (!fd.is_file(build_path)) return null
}
var src = text(fd.slurp(src_path))
var host = detect_host_target()
if (!host) return null
handle = os.dylib_open(dylib_path)
var build_path = build_mod.cache_path(src + '\n' + host, build_mod.SALT_NATIVE)
if (!fd.is_file(build_path)) return null
log.shop('native dylib cache hit: ' + stem)
var handle = os.dylib_open(build_path)
if (!handle) return null
sym = Shop.c_symbol_for_file(pkg, stem)
var sym = Shop.c_symbol_for_file(pkg, stem)
return {_native: true, _handle: handle, _sym: sym}
}
@@ -675,9 +666,6 @@ function resolve_mod_fn(path, pkg) {
var cached = null
var ast = null
var compiled = null
var mach_path = null
var mach_blob = null
var mcode_path = null
var ir = null
var optimized = null
var mcode_json = null
@@ -729,9 +717,9 @@ function resolve_mod_fn(path, pkg) {
}
}
// Check for cached mcode in content-addressed store (salted hash to distinguish from mach)
// Check for cached mcode in content-addressed store
if (policy.allow_compile) {
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
cached_mcode_path = hash_path(content_key, 'mcode')
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
@@ -855,52 +843,67 @@ function make_c_symbol(pkg, file) {
return 'js_' + pkg_safe + '_' + file_safe + '_use'
}
// Get the deterministic dylib path for a module in lib/<pkg>/<stem>.dylib
function get_dylib_path(pkg, stem) {
return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + dylib_ext
}
// Ensure all C modules for a package are built and loaded.
// Returns the array of {file, symbol, dylib} results, cached per package.
function ensure_package_dylibs(pkg) {
if (package_dylibs[pkg]) return package_dylibs[pkg]
// Get the deterministic mach path for a module in lib/<pkg>/<stem>.mach
function get_mach_path(pkg, stem) {
return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + '.mach'
}
var build_mod = use_cache['core/build']
if (!build_mod) return null
// Open a per-module dylib and return the dlopen handle
// Pre-loads sibling dylibs with RTLD_LAZY|RTLD_GLOBAL so cross-dylib symbols resolve
function open_module_dylib(dylib_path) {
if (open_dls[dylib_path]) return open_dls[dylib_path]
if (!fd.is_file(dylib_path)) return null
var dir = fd.dirname(dylib_path)
var entries = fd.readdir(dir)
var i = 0
var sibling = null
var handle = null
while (i < length(entries)) {
if (ends_with(entries[i], dylib_ext) && entries[i] != fd.basename(dylib_path)) {
sibling = dir + '/' + entries[i]
if (!open_dls[sibling]) {
handle = os.dylib_preload(sibling)
if (handle) open_dls[sibling] = handle
}
}
i = i + 1
var target = detect_host_target()
if (!target) return null
var c_files = pkg_tools.get_c_files(pkg, target, true)
if (!c_files || length(c_files) == 0) {
package_dylibs[pkg] = []
return []
}
open_dls[dylib_path] = os.dylib_open(dylib_path)
return open_dls[dylib_path]
log.shop('ensuring C modules for ' + pkg)
var results = build_mod.build_dynamic(pkg, target, 'release', {})
package_dylibs[pkg] = results
// Preload all sibling dylibs with RTLD_LAZY|RTLD_GLOBAL
arrfor(results, function(r) {
var handle = null
if (r.dylib && !open_dls[r.dylib]) {
handle = os.dylib_preload(r.dylib)
if (handle) open_dls[r.dylib] = handle
}
})
log.shop('built ' + text(length(results)) + ' C module(s) for ' + pkg)
return results
}
// Try to resolve a C symbol from the deterministic dylib path
// Try to resolve a C symbol by building the package on demand
// Returns a loader function or null
function try_dylib_symbol(sym, pkg, file_stem) {
var dylib_path = get_dylib_path(pkg, file_stem)
var handle = open_module_dylib(dylib_path)
var dylibs = ensure_package_dylibs(pkg)
if (!dylibs || length(dylibs) == 0) return null
var c_file = file_stem + '.c'
var cpp_file = file_stem + '.cpp'
var entry = find(dylibs, function(r) {
return r.file == c_file || r.file == cpp_file
})
if (!entry || !entry.dylib) return null
var handle = open_dls[entry.dylib]
if (!handle) {
handle = os.dylib_open(entry.dylib)
if (handle) open_dls[entry.dylib] = handle
}
if (!handle) return null
if (!os.dylib_has_symbol(handle, sym)) return null
log.shop('resolved ' + sym + ' from build cache')
return function() { return os.dylib_symbol(handle, sym) }
}
// Resolve a C symbol by searching:
// At each scope: check lib/ dylib first, then internal (static)
// At each scope: check build-cache dylib first, then internal (static)
function resolve_c_symbol(path, package_context) {
var explicit = split_explicit_package_import(path)
var sym = null
@@ -922,7 +925,7 @@ function resolve_c_symbol(path, package_context) {
sym = make_c_symbol(explicit.package, explicit.path)
file_stem = replace(explicit.path, '.c', '')
// Check lib/ dylib first
// Check build-cache dylib first
if (policy.allow_dylib) {
loader = try_dylib_symbol(sym, explicit.package, file_stem)
if (loader) {
@@ -950,7 +953,7 @@ function resolve_c_symbol(path, package_context) {
if (!package_context || package_context == 'core') {
core_sym = make_c_symbol('core', path)
// Check lib/ dylib first for core
// Check build-cache dylib first for core
if (policy.allow_dylib) {
loader = try_dylib_symbol(core_sym, 'core', path)
if (loader) {
@@ -972,7 +975,7 @@ function resolve_c_symbol(path, package_context) {
return null
}
// 1. Check own package (dylib first, then internal)
// 1. Check own package (build-cache dylib first, then internal)
sym = make_c_symbol(package_context, path)
if (policy.allow_dylib) {
@@ -1028,7 +1031,7 @@ function resolve_c_symbol(path, package_context) {
}
}
// 3. Check core (dylib first, then internal)
// 3. Check core (build-cache dylib first, then internal)
core_sym = make_c_symbol('core', path)
if (policy.allow_dylib) {
@@ -1541,11 +1544,8 @@ Shop.remove = function(pkg) {
fd.rmdir(pkg_dir, 1)
}
// Remove built dylibs
var lib_dir = global_shop_path + '/lib/' + safe_package_path(pkg)
if (fd.is_dir(lib_dir)) {
fd.rmdir(lib_dir, 1)
}
// Invalidate package dylib cache
package_dylibs[pkg] = null
log.console("Removed " + pkg)
return true
@@ -1599,14 +1599,9 @@ Shop.module_reload = function(path, package) {
var lookup_key = package ? package + ':' + path : ':' + path
module_info_cache[lookup_key] = null
// Close old dylib handle if any
var old_dylib_path = null
// Invalidate package dylib cache so next resolve triggers rebuild
if (package) {
old_dylib_path = get_dylib_path(package, path)
if (open_dls[old_dylib_path]) {
os.dylib_close(open_dls[old_dylib_path])
open_dls[old_dylib_path] = null
}
package_dylibs[package] = null
}
var info = resolve_module_info(path, package)
@@ -1717,17 +1712,6 @@ Shop.lib_name_for_package = function(pkg) {
return safe_package_path(pkg)
}
// Returns { ok: bool, results: [{pkg, ok, error}] }
// Get the deterministic dylib path for a module (public API)
Shop.get_dylib_path = function(pkg, stem) {
return get_dylib_path(pkg, stem)
}
// Get the deterministic mach path for a module (public API)
Shop.get_mach_path = function(pkg, stem) {
return get_mach_path(pkg, stem)
}
// Load a module explicitly as mach bytecode, bypassing dylib resolution.
// Returns the loaded module value. Disrupts if the module cannot be found.
Shop.load_as_mach = function(path, pkg) {
@@ -1742,9 +1726,6 @@ Shop.load_as_mach = function(path, pkg) {
var ast = null
var ir = null
var optimized = null
var pkg_dir = null
var stem = null
var mach_path = null
var file_info = null
var inject = null
var env = null
@@ -1755,27 +1736,13 @@ Shop.load_as_mach = function(path, pkg) {
content = text(fd.slurp(file_path))
content_key = stone(blob(content))
// Try installed .mach in lib/
if (pkg) {
pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
if (starts_with(file_path, pkg_dir + '/')) {
stem = text(file_path, length(pkg_dir) + 1)
mach_path = get_mach_path(pkg, stem)
if (fd.is_file(mach_path)) {
compiled = fd.slurp(mach_path)
}
}
}
// Try cached mach blob
if (!compiled) {
cached = pull_from_cache(content_key)
if (cached) compiled = cached
}
cached = pull_from_cache(content_key)
if (cached) compiled = cached
// Try cached mcode -> compile to mach
if (!compiled) {
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
cached_mcode_path = hash_path(content_key, 'mcode')
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
@@ -1795,7 +1762,7 @@ Shop.load_as_mach = function(path, pkg) {
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
cached_mcode_path = hash_path(content_key, 'mcode')
ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
compiled = mach_compile_mcode_bin(file_path, mcode_json)

21
resolve.ce
View File

@@ -130,11 +130,7 @@ var is_linked = false
var is_in_lock = false
var is_local = false
var is_fetched = false
var lib_dir = null
var lib_name = null
var dylib_ext = null
var lib_path = null
var is_built = false
var has_c_files = false
var status_parts = null
var commit_str = null
var line = null
@@ -164,12 +160,13 @@ for (i = 0; i < length(sorted); i++) {
pkg_dir = shop.get_package_dir(locator)
is_fetched = fd.is_dir(pkg_dir) || fd.is_link(pkg_dir)
// Check if built (library exists)
lib_dir = shop.get_lib_dir()
lib_name = shop.lib_name_for_package(locator)
dylib_ext = '.dylib' // TODO: detect from target
lib_path = lib_dir + '/' + lib_name + dylib_ext
is_built = fd.is_file(lib_path)
// Check if package has C modules (built on demand)
has_c_files = false
var _check_c = function() {
var c_files = pkg.get_c_files(locator, target_triple, true)
if (c_files && length(c_files) > 0) has_c_files = true
} disruption {}
_check_c()
// Format output
status_parts = []
@@ -177,7 +174,7 @@ for (i = 0; i < length(sorted); i++) {
if (is_local) push(status_parts, "local")
if (!is_in_lock) push(status_parts, "not in lock")
if (!is_fetched) push(status_parts, "not fetched")
if (is_built) push(status_parts, "built")
if (has_c_files) push(status_parts, "has C modules")
commit_str = ""
if (lock_entry && lock_entry.commit) {

20
tests/build_audit.cm
View File

@@ -5,26 +5,6 @@ var shop = use('internal/shop')
var runtime = use('runtime')
return {
// ========================================================================
// DETERMINISTIC DYLIB PATHS
// ========================================================================
test_dylib_path_deterministic: function() {
var path = shop.get_dylib_path('core', 'time')
if (!ends_with(path, '/lib/core/time.dylib')) return "dylib path should end with /lib/core/time.dylib, got: " + path
},
test_dylib_path_internal: function() {
var path = shop.get_dylib_path('core', 'internal/os')
if (!ends_with(path, '/lib/core/internal/os.dylib')) return "dylib path should end with /lib/core/internal/os.dylib, got: " + path
},
test_dylib_path_external_package: function() {
var path = shop.get_dylib_path('gitea.pockle.world/john/prosperon', 'sprite')
if (!ends_with(path, '/lib/gitea.pockle.world/john/prosperon/sprite.dylib'))
return "dylib path should mirror package layout, got: " + path
},
// ========================================================================
// SYMBOL NAMING
// ========================================================================

17
verify.ce
View File

@@ -85,10 +85,6 @@ function verify_package(locator) {
var current_target = null
var expected_target = null
var target_dir = null
var lib_dir = null
var lib_name = null
var dylib_ext = null
var lib_path = null
var c_files = null
checked++
@@ -147,20 +143,9 @@ function verify_package(locator) {
}
}
// Check build output exists
lib_dir = shop.get_lib_dir()
lib_name = shop.lib_name_for_package(locator)
dylib_ext = '.dylib' // TODO: detect from target
lib_path = lib_dir + '/' + lib_name + dylib_ext
// Only check for builds if package has C files
// Check if package has C files (builds happen lazily on first use)
var _check_build = function() {
c_files = pkg.get_c_files(locator, target_triple, true)
if (c_files && length(c_files) > 0) {
if (!fd.is_file(lib_path)) {
add_warning(locator + ": library not built at " + lib_path)
}
}
} disruption {
// Skip build check if can't determine C files
}