diff --git a/build.cm b/build.cm index 8f343470..ab9fe5fd 100644 --- a/build.cm +++ b/build.cm @@ -80,6 +80,22 @@ function content_hash(str) { return text(crypto.blake2(bb, 32), 'h') } +// ============================================================================ +// Cache key salts — canonical registry +// Every artifact type has a unique salt so hash collisions between types +// are impossible, and no file extensions are needed in build/. +// ============================================================================ +var SALT_OBJ = 'obj' // compiled C object file +var SALT_DYLIB = 'dylib' // linked dynamic library +var SALT_NATIVE = 'native' // native-compiled .cm dylib +var SALT_MACH = 'mach' // mach bytecode blob +var SALT_MCODE = 'mcode' // mcode IR (JSON) +var SALT_DEPS = 'deps' // cached cc -MM dependency list + +function cache_path(content, salt) { + return get_build_dir() + '/' + content_hash(content + '\n' + salt) +} + function get_build_dir() { return shop.get_build_dir() } @@ -98,6 +114,52 @@ function ensure_dir(path) { Build.ensure_dir = ensure_dir +// ============================================================================ +// Dependency scanning helpers +// ============================================================================ + +// Parse make-style dependency output: +// foo.o: foo.c header1.h \ +// header2.h +// Returns array of dependency file paths (skips the target) +function parse_makefile_deps(dep_text) { + var joined = replace(dep_text, /\\\n\s*/, ' ') + var colon_pos = search(joined, ':') + if (colon_pos == null) return [] + var rest = trim(text(joined, colon_pos + 1)) + var parts = filter(array(rest, /\s+/), function(p) { + return length(p) > 0 + }) + return parts +} + +// Run cc -MM to get the preprocessor dependency list. +// Returns array of dependency file paths. +function get_c_deps(cc, flags, src_path) { + var dep_file = '/tmp/cell_deps_' + content_hash(src_path) + '.d' + var dep_cmd = [cc, '-MM', '-MG', '-MF', '"' + dep_file + '"'] + dep_cmd = array(dep_cmd, flags) + push(dep_cmd, '"' + src_path + '"') + var ret = os.system(text(dep_cmd, ' ') + ' 2>/dev/null') + if (ret != 0) return [src_path] + if (!fd.is_file(dep_file)) return [src_path] + var dep_text = text(fd.slurp(dep_file)) + return parse_makefile_deps(dep_text) +} + +// Build a full hash string from the compilation command and all dependency +// file contents. This is the content key for the object file. +function hash_all_deps(cmd_str, deps) { + var parts = [cmd_str] + arrfor(deps, function(dep_path) { + if (fd.is_file(dep_path)) + push(parts, dep_path + '\n' + text(fd.slurp(dep_path))) + else + push(parts, dep_path + '\n') + }) + return text(parts, '\n') +} + // ============================================================================ // Compilation // ============================================================================ @@ -124,30 +186,30 @@ Build.compile_file = function(pkg, file, target, opts) { // Symbol name for this file var sym_name = shop.c_symbol_for_file(pkg, file) - // Build command - var cmd_parts = [cc, '-c', '-fPIC'] + // Build common flags (shared between dep scan and compilation) + var common_flags = [] // Add buildtype-specific flags if (_buildtype == 'release') { - cmd_parts = array(cmd_parts, ['-O3', '-DNDEBUG']) + common_flags = array(common_flags, ['-O3', '-DNDEBUG']) } else if (_buildtype == 'debug') { - cmd_parts = array(cmd_parts, ['-O2', '-g']) + common_flags = array(common_flags, ['-O2', '-g']) } else if (_buildtype == 'minsize') { - cmd_parts = array(cmd_parts, ['-Os', '-DNDEBUG']) + common_flags = array(common_flags, ['-Os', '-DNDEBUG']) } - push(cmd_parts, '-DCELL_USE_NAME=' + sym_name) - push(cmd_parts, '-I"' + pkg_dir + '"') + push(common_flags, '-DCELL_USE_NAME=' + sym_name) + push(common_flags, '-I"' + pkg_dir + '"') // Auto-discover include/ directory if (fd.is_dir(pkg_dir + '/include')) { - push(cmd_parts, '-I"' + pkg_dir + '/include"') + push(common_flags, '-I"' + pkg_dir + '/include"') } // External packages need core's source dir for cell.h, quickjs.h, blob.h if (pkg != 'core') { core_dir = shop.get_package_dir('core') - push(cmd_parts, '-I"' + core_dir + '/source"') + push(common_flags, '-I"' + core_dir + '/source"') } // Add package CFLAGS (resolve relative -I paths) @@ -160,16 +222,19 @@ Build.compile_file = function(pkg, file, target, opts) { f = '-I"' + pkg_dir + '/' + ipath + '"' } } - push(cmd_parts, f) + push(common_flags, f) }) - + // Add target CFLAGS arrfor(target_cflags, function(flag) { - push(cmd_parts, flag) + push(common_flags, flag) }) - + + // Build full compilation command + var cmd_parts = [cc, '-c', '-fPIC'] + cmd_parts = array(cmd_parts, common_flags) push(cmd_parts, '"' + src_path + '"') - + var cmd_str = text(cmd_parts, ' ') if (_opts.verbose) { @@ -177,30 +242,56 @@ Build.compile_file = function(pkg, file, target, opts) { print('[verbose] compile: ' + cmd_str) } - // Content hash: command + file content + // Two-level cache: quick hash for deps file, full hash for object var file_content = fd.slurp(src_path) - var hash_input = cmd_str + '\n' + text(file_content) - var hash = content_hash(hash_input) - + var quick_content = cmd_str + '\n' + text(file_content) + var deps_path = cache_path(quick_content, SALT_DEPS) + var build_dir = get_build_dir() ensure_dir(build_dir) - var obj_path = build_dir + '/' + hash - - // Check if already compiled + + var deps = null + var full_content = null + var obj_path = null + + // Warm path: read cached dep list, verify by hashing all deps + if (fd.is_file(deps_path)) { + deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) { + return length(p) > 0 + }) + full_content = hash_all_deps(cmd_str, deps) + obj_path = cache_path(full_content, SALT_OBJ) + if (fd.is_file(obj_path)) { + if (_opts.verbose) print('[verbose] cache hit: ' + file) + log.shop('cache hit ' + file) + return obj_path + } + log.shop('cache stale ' + file + ' (header changed)') + } + + // Cold path: run cc -MM to discover deps + log.shop('dep scan ' + file) + deps = get_c_deps(cc, common_flags, src_path) + full_content = hash_all_deps(cmd_str, deps) + obj_path = cache_path(full_content, SALT_OBJ) + + // Check if object exists (might exist from previous build with same deps) if (fd.is_file(obj_path)) { - if (_opts.verbose) print('[verbose] cache hit: ' + file) + fd.slurpwrite(deps_path, stone(blob(text(deps, '\n')))) + if (_opts.verbose) print('[verbose] cache hit: ' + file + ' (after dep scan)') + log.shop('cache hit ' + file + ' (after dep scan)') return obj_path } - if (_opts.verbose) print('[verbose] cache miss: ' + file) - - // Compile — capture stderr to detect missing-header vs real errors - var err_path = '/tmp/cell_build_err_' + hash + '.log' + + // Compile + log.shop('compiling ' + file) + log.console('Compiling ' + file) + var err_path = '/tmp/cell_build_err_' + content_hash(src_path) + '.log' var full_cmd = cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"' var err_text = null var missing = null var err_lines = null var first_err = null - log.console('Compiling ' + file) var ret = os.system(full_cmd) if (ret != 0) { if (fd.is_file(err_path)) { @@ -222,6 +313,8 @@ Build.compile_file = function(pkg, file, target, opts) { return null } + // Save deps for future warm-path lookups + fd.slurpwrite(deps_path, stone(blob(text(deps, '\n')))) return obj_path } @@ -249,8 +342,8 @@ Build.build_package = function(pkg, target, exclude_main, buildtype) { // Dynamic library building // ============================================================================ -// Compute link key from all inputs that affect the dylib output -function compute_link_key(objects, ldflags, target_ldflags, opts) { +// Compute link content string from all inputs that affect the dylib output +function compute_link_content(objects, ldflags, target_ldflags, opts) { // Sort objects for deterministic hash var sorted_objects = sort(objects) @@ -269,7 +362,7 @@ function compute_link_key(objects, ldflags, target_ldflags, opts) { push(parts, 'target_ldflag:' + flag) }) - return content_hash(text(parts, '\n')) + return text(parts, '\n') } // Build a per-module dynamic library for a single C file @@ -283,7 +376,6 @@ Build.build_module_dylib = function(pkg, file, target, opts) { if (!obj) return null var tc = toolchains[_target] - var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so') var cc = tc.cpp || tc.c var local_dir = get_local_dir() var pkg_dir = shop.get_package_dir(pkg) @@ -307,10 +399,10 @@ Build.build_module_dylib = function(pkg, file, target, opts) { // Content-addressed output: hash of (all objects + link flags + target) var all_objects = [obj] all_objects = array(all_objects, _extra) - var link_key = compute_link_key(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc}) + var link_content = compute_link_content(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc}) var build_dir = get_build_dir() ensure_dir(build_dir) - var dylib_path = build_dir + '/' + link_key + '.' + _target + dylib_ext + var dylib_path = cache_path(link_content, SALT_DYLIB) var cmd_parts = null var cmd_str = null var ret = null @@ -352,29 +444,17 @@ Build.build_module_dylib = function(pkg, file, target, opts) { cmd_str = text(cmd_parts, ' ') if (_opts.verbose) print('[verbose] link: ' + cmd_str) + log.shop('linking ' + file) log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path)) ret = os.system(cmd_str) if (ret != 0) { print('Linking failed: ' + file) return null } + } else { + log.shop('link cache hit ' + file) } - // Always install to deterministic lib//.dylib - // Strip .c/.cpp extension so the loader can find it by module name - var file_stem = file - if (ends_with(file_stem, '.cpp')) file_stem = text(file_stem, 0, -4) - else if (ends_with(file_stem, '.c')) file_stem = text(file_stem, 0, -2) - var install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) - var stem_dir = fd.dirname(file_stem) - if (stem_dir && stem_dir != '.') { - install_dir = install_dir + '/' + stem_dir - } - ensure_dir(install_dir) - var install_path = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) + '/' + file_stem + dylib_ext - fd.slurpwrite(install_path, fd.slurp(dylib_path)) - if (_opts.verbose) print('[verbose] install: ' + install_path) - return dylib_path } @@ -602,16 +682,12 @@ Build.compile_native = function(src_path, target, buildtype, pkg) { var _target = target || Build.detect_host_target() var _buildtype = buildtype || 'release' var qbe_rt_path = null - var native_stem = null - var native_install_dir = null - var native_install_path = null if (!fd.is_file(src_path)) { print('Source file not found: ' + src_path); disrupt } var tc = toolchains[_target] - var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so') var cc = tc.c // Step 1: Compile through pipeline @@ -627,16 +703,15 @@ Build.compile_native = function(src_path, target, buildtype, pkg) { var il_parts = qbe_emit(optimized, qbe_macros, sym_name) // Content hash for cache key - var hash = content_hash(text(fd.slurp(src_path)) + '\n' + _target + '\nnative') var build_dir = get_build_dir() ensure_dir(build_dir) - var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext + var dylib_path = cache_path(text(fd.slurp(src_path)) + '\n' + _target, SALT_NATIVE) if (fd.is_file(dylib_path)) return dylib_path // Compile and assemble via batched parallel pipeline - var tmp = '/tmp/cell_native_' + hash + var tmp = '/tmp/cell_native_' + content_hash(src_path) var rt_o_path = '/tmp/cell_qbe_rt.o' var o_paths = compile_native_batched(il_parts, cc, tmp) @@ -672,15 +747,6 @@ Build.compile_native = function(src_path, target, buildtype, pkg) { log.console('Built native: ' + fd.basename(dylib_path)) - // Install to deterministic lib//.dylib - if (pkg) { - native_stem = fd.basename(src_path) - native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) - ensure_dir(native_install_dir) - native_install_path = native_install_dir + '/' + native_stem + dylib_ext - fd.slurpwrite(native_install_path, fd.slurp(dylib_path)) - } - return dylib_path } @@ -692,12 +758,8 @@ Build.compile_native_ir = function(optimized, src_path, opts) { var _buildtype = (opts && opts.buildtype) || 'release' var pkg = opts && opts.pkg var qbe_rt_path = null - var native_stem = null - var native_install_dir = null - var native_install_path = null var tc = toolchains[_target] - var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so') var cc = tc.c var qbe_macros = use('qbe') @@ -710,16 +772,15 @@ Build.compile_native_ir = function(optimized, src_path, opts) { var il_parts = qbe_emit(optimized, qbe_macros, sym_name) var src = text(fd.slurp(src_path)) - var hash = content_hash(src + '\n' + _target + '\nnative') var build_dir = get_build_dir() ensure_dir(build_dir) - var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext + var dylib_path = cache_path(src + '\n' + _target, SALT_NATIVE) if (fd.is_file(dylib_path)) return dylib_path // Compile and assemble via batched parallel pipeline - var tmp = '/tmp/cell_native_' + hash + var tmp = '/tmp/cell_native_' + content_hash(src_path) var rt_o_path = '/tmp/cell_qbe_rt.o' var o_paths = compile_native_batched(il_parts, cc, tmp) @@ -755,14 +816,6 @@ Build.compile_native_ir = function(optimized, src_path, opts) { log.console('Built native: ' + fd.basename(dylib_path)) - if (pkg) { - native_stem = fd.basename(src_path) - native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) - ensure_dir(native_install_dir) - native_install_path = native_install_dir + '/' + native_stem + dylib_ext - fd.slurpwrite(native_install_path, fd.slurp(dylib_path)) - } - return dylib_path } @@ -884,4 +937,13 @@ Build.build_all_dynamic = function(target, buildtype, opts) { return results } +// Export salt constants and cache_path for shop.cm and others +Build.SALT_OBJ = SALT_OBJ +Build.SALT_DYLIB = SALT_DYLIB +Build.SALT_NATIVE = SALT_NATIVE +Build.SALT_MACH = SALT_MACH +Build.SALT_MCODE = SALT_MCODE +Build.SALT_DEPS = SALT_DEPS +Build.cache_path = cache_path + return Build diff --git a/clean.ce b/clean.ce index f3ab6b10..89c80bc7 100644 --- a/clean.ce +++ b/clean.ce @@ -119,37 +119,13 @@ var build_dir = shop.get_build_dir() var packages_dir = replace(shop.get_package_dir(''), /\/$/, '') // Get base packages dir if (clean_build) { - if (is_shop_scope) { - // Clean entire build and lib directories - if (fd.is_dir(build_dir)) { - push(dirs_to_delete, build_dir) - } - if (fd.is_dir(lib_dir)) { - push(dirs_to_delete, lib_dir) - } - } else { - // Clean specific package libraries - arrfor(packages_to_clean, function(p) { - if (p == 'core') return - - var lib_name = shop.lib_name_for_package(p) - var dylib_ext = '.dylib' - var lib_path = lib_dir + '/' + lib_name + dylib_ext - - if (fd.is_file(lib_path)) { - push(files_to_delete, lib_path) - } - - // Also check for .so and .dll - var so_path = lib_dir + '/' + lib_name + '.so' - var dll_path = lib_dir + '/' + lib_name + '.dll' - if (fd.is_file(so_path)) { - push(files_to_delete, so_path) - } - if (fd.is_file(dll_path)) { - push(files_to_delete, dll_path) - } - }) + // Nuke entire build cache (content-addressed, per-package clean impractical) + if (fd.is_dir(build_dir)) { + push(dirs_to_delete, build_dir) + } + // Clean orphaned lib/ directory if it exists (legacy) + if (fd.is_dir(lib_dir)) { + push(dirs_to_delete, lib_dir) } } diff --git a/internal/shop.cm b/internal/shop.cm index 9d6285c8..09796818 100644 --- a/internal/shop.cm +++ b/internal/shop.cm @@ -54,9 +54,10 @@ function ensure_dir(path) { } } -function hash_path(content) +function hash_path(content, salt) { - return global_shop_path + '/build' + '/' + content_hash(content) + var s = salt || 'mach' + return global_shop_path + '/build/' + content_hash(stone(blob(text(content) + '\n' + s))) } var Shop = {} @@ -418,6 +419,7 @@ Shop.extract_commit_hash = function(pkg, response) { } var open_dls = {} +var package_dylibs = {} // pkg -> [{file, symbol, dylib}, ...] // Host target detection for native dylib resolution function detect_host_target() { @@ -434,37 +436,26 @@ function detect_host_target() { var host_target = detect_host_target() -// Check for a native .cm dylib at the deterministic lib path +// Check for a native .cm dylib in the build cache // Returns a native descriptor {_native, _handle, _sym}, or null if no native dylib exists -// Also checks staleness: if source has changed, the content-addressed build artifact -// won't exist for the new hash, so the installed dylib is treated as stale. function try_native_mod_dylib(pkg, stem) { - var dylib_path = get_dylib_path(pkg, stem) - var src_path = null - var src = null - var host = null - var hash = null - var tc_ext = null - var build_path = null - var handle = null - var sym = null + var build_mod = use_cache['core/build'] + if (!build_mod) return null - if (!fd.is_file(dylib_path)) return null + var src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem + if (!fd.is_file(src_path)) return null - // Staleness check: verify the content-addressed build artifact exists - src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem - if (fd.is_file(src_path)) { - src = text(fd.slurp(src_path)) - host = detect_host_target() - hash = content_hash(src + '\n' + host + '\nnative') - tc_ext = dylib_ext - build_path = global_shop_path + '/build/' + hash + '.' + host + tc_ext - if (!fd.is_file(build_path)) return null - } + var src = text(fd.slurp(src_path)) + var host = detect_host_target() + if (!host) return null - handle = os.dylib_open(dylib_path) + var build_path = build_mod.cache_path(src + '\n' + host, build_mod.SALT_NATIVE) + if (!fd.is_file(build_path)) return null + + log.shop('native dylib cache hit: ' + stem) + var handle = os.dylib_open(build_path) if (!handle) return null - sym = Shop.c_symbol_for_file(pkg, stem) + var sym = Shop.c_symbol_for_file(pkg, stem) return {_native: true, _handle: handle, _sym: sym} } @@ -675,9 +666,6 @@ function resolve_mod_fn(path, pkg) { var cached = null var ast = null var compiled = null - var mach_path = null - var mach_blob = null - var mcode_path = null var ir = null var optimized = null var mcode_json = null @@ -729,9 +717,9 @@ function resolve_mod_fn(path, pkg) { } } - // Check for cached mcode in content-addressed store (salted hash to distinguish from mach) + // Check for cached mcode in content-addressed store if (policy.allow_compile) { - cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) + cached_mcode_path = hash_path(content_key, 'mcode') if (fd.is_file(cached_mcode_path)) { mcode_json = text(fd.slurp(cached_mcode_path)) compiled = mach_compile_mcode_bin(path, mcode_json) @@ -855,52 +843,67 @@ function make_c_symbol(pkg, file) { return 'js_' + pkg_safe + '_' + file_safe + '_use' } -// Get the deterministic dylib path for a module in lib//.dylib -function get_dylib_path(pkg, stem) { - return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + dylib_ext -} +// Ensure all C modules for a package are built and loaded. +// Returns the array of {file, symbol, dylib} results, cached per package. +function ensure_package_dylibs(pkg) { + if (package_dylibs[pkg]) return package_dylibs[pkg] -// Get the deterministic mach path for a module in lib//.mach -function get_mach_path(pkg, stem) { - return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + '.mach' -} + var build_mod = use_cache['core/build'] + if (!build_mod) return null -// Open a per-module dylib and return the dlopen handle -// Pre-loads sibling dylibs with RTLD_LAZY|RTLD_GLOBAL so cross-dylib symbols resolve -function open_module_dylib(dylib_path) { - if (open_dls[dylib_path]) return open_dls[dylib_path] - if (!fd.is_file(dylib_path)) return null - var dir = fd.dirname(dylib_path) - var entries = fd.readdir(dir) - var i = 0 - var sibling = null - var handle = null - while (i < length(entries)) { - if (ends_with(entries[i], dylib_ext) && entries[i] != fd.basename(dylib_path)) { - sibling = dir + '/' + entries[i] - if (!open_dls[sibling]) { - handle = os.dylib_preload(sibling) - if (handle) open_dls[sibling] = handle - } - } - i = i + 1 + var target = detect_host_target() + if (!target) return null + + var c_files = pkg_tools.get_c_files(pkg, target, true) + if (!c_files || length(c_files) == 0) { + package_dylibs[pkg] = [] + return [] } - open_dls[dylib_path] = os.dylib_open(dylib_path) - return open_dls[dylib_path] + + log.shop('ensuring C modules for ' + pkg) + var results = build_mod.build_dynamic(pkg, target, 'release', {}) + package_dylibs[pkg] = results + + // Preload all sibling dylibs with RTLD_LAZY|RTLD_GLOBAL + arrfor(results, function(r) { + var handle = null + if (r.dylib && !open_dls[r.dylib]) { + handle = os.dylib_preload(r.dylib) + if (handle) open_dls[r.dylib] = handle + } + }) + + log.shop('built ' + text(length(results)) + ' C module(s) for ' + pkg) + return results } -// Try to resolve a C symbol from the deterministic dylib path +// Try to resolve a C symbol by building the package on demand // Returns a loader function or null function try_dylib_symbol(sym, pkg, file_stem) { - var dylib_path = get_dylib_path(pkg, file_stem) - var handle = open_module_dylib(dylib_path) + var dylibs = ensure_package_dylibs(pkg) + if (!dylibs || length(dylibs) == 0) return null + + var c_file = file_stem + '.c' + var cpp_file = file_stem + '.cpp' + var entry = find(dylibs, function(r) { + return r.file == c_file || r.file == cpp_file + }) + if (!entry || !entry.dylib) return null + + var handle = open_dls[entry.dylib] + if (!handle) { + handle = os.dylib_open(entry.dylib) + if (handle) open_dls[entry.dylib] = handle + } if (!handle) return null if (!os.dylib_has_symbol(handle, sym)) return null + + log.shop('resolved ' + sym + ' from build cache') return function() { return os.dylib_symbol(handle, sym) } } // Resolve a C symbol by searching: -// At each scope: check lib/ dylib first, then internal (static) +// At each scope: check build-cache dylib first, then internal (static) function resolve_c_symbol(path, package_context) { var explicit = split_explicit_package_import(path) var sym = null @@ -922,7 +925,7 @@ function resolve_c_symbol(path, package_context) { sym = make_c_symbol(explicit.package, explicit.path) file_stem = replace(explicit.path, '.c', '') - // Check lib/ dylib first + // Check build-cache dylib first if (policy.allow_dylib) { loader = try_dylib_symbol(sym, explicit.package, file_stem) if (loader) { @@ -950,7 +953,7 @@ function resolve_c_symbol(path, package_context) { if (!package_context || package_context == 'core') { core_sym = make_c_symbol('core', path) - // Check lib/ dylib first for core + // Check build-cache dylib first for core if (policy.allow_dylib) { loader = try_dylib_symbol(core_sym, 'core', path) if (loader) { @@ -972,7 +975,7 @@ function resolve_c_symbol(path, package_context) { return null } - // 1. Check own package (dylib first, then internal) + // 1. Check own package (build-cache dylib first, then internal) sym = make_c_symbol(package_context, path) if (policy.allow_dylib) { @@ -1028,7 +1031,7 @@ function resolve_c_symbol(path, package_context) { } } - // 3. Check core (dylib first, then internal) + // 3. Check core (build-cache dylib first, then internal) core_sym = make_c_symbol('core', path) if (policy.allow_dylib) { @@ -1541,11 +1544,8 @@ Shop.remove = function(pkg) { fd.rmdir(pkg_dir, 1) } - // Remove built dylibs - var lib_dir = global_shop_path + '/lib/' + safe_package_path(pkg) - if (fd.is_dir(lib_dir)) { - fd.rmdir(lib_dir, 1) - } + // Invalidate package dylib cache + package_dylibs[pkg] = null log.console("Removed " + pkg) return true @@ -1599,14 +1599,9 @@ Shop.module_reload = function(path, package) { var lookup_key = package ? package + ':' + path : ':' + path module_info_cache[lookup_key] = null - // Close old dylib handle if any - var old_dylib_path = null + // Invalidate package dylib cache so next resolve triggers rebuild if (package) { - old_dylib_path = get_dylib_path(package, path) - if (open_dls[old_dylib_path]) { - os.dylib_close(open_dls[old_dylib_path]) - open_dls[old_dylib_path] = null - } + package_dylibs[package] = null } var info = resolve_module_info(path, package) @@ -1717,17 +1712,6 @@ Shop.lib_name_for_package = function(pkg) { return safe_package_path(pkg) } -// Returns { ok: bool, results: [{pkg, ok, error}] } -// Get the deterministic dylib path for a module (public API) -Shop.get_dylib_path = function(pkg, stem) { - return get_dylib_path(pkg, stem) -} - -// Get the deterministic mach path for a module (public API) -Shop.get_mach_path = function(pkg, stem) { - return get_mach_path(pkg, stem) -} - // Load a module explicitly as mach bytecode, bypassing dylib resolution. // Returns the loaded module value. Disrupts if the module cannot be found. Shop.load_as_mach = function(path, pkg) { @@ -1742,9 +1726,6 @@ Shop.load_as_mach = function(path, pkg) { var ast = null var ir = null var optimized = null - var pkg_dir = null - var stem = null - var mach_path = null var file_info = null var inject = null var env = null @@ -1755,27 +1736,13 @@ Shop.load_as_mach = function(path, pkg) { content = text(fd.slurp(file_path)) content_key = stone(blob(content)) - // Try installed .mach in lib/ - if (pkg) { - pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg) - if (starts_with(file_path, pkg_dir + '/')) { - stem = text(file_path, length(pkg_dir) + 1) - mach_path = get_mach_path(pkg, stem) - if (fd.is_file(mach_path)) { - compiled = fd.slurp(mach_path) - } - } - } - // Try cached mach blob - if (!compiled) { - cached = pull_from_cache(content_key) - if (cached) compiled = cached - } + cached = pull_from_cache(content_key) + if (cached) compiled = cached // Try cached mcode -> compile to mach if (!compiled) { - cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) + cached_mcode_path = hash_path(content_key, 'mcode') if (fd.is_file(cached_mcode_path)) { mcode_json = text(fd.slurp(cached_mcode_path)) compiled = mach_compile_mcode_bin(file_path, mcode_json) @@ -1795,7 +1762,7 @@ Shop.load_as_mach = function(path, pkg) { ir = _mcode_mod(ast) optimized = _streamline_mod(ir) mcode_json = shop_json.encode(optimized) - cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) + cached_mcode_path = hash_path(content_key, 'mcode') ensure_dir(global_shop_path + '/build') fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json))) compiled = mach_compile_mcode_bin(file_path, mcode_json) diff --git a/resolve.ce b/resolve.ce index 8cee4696..e67dba7f 100644 --- a/resolve.ce +++ b/resolve.ce @@ -130,11 +130,7 @@ var is_linked = false var is_in_lock = false var is_local = false var is_fetched = false -var lib_dir = null -var lib_name = null -var dylib_ext = null -var lib_path = null -var is_built = false +var has_c_files = false var status_parts = null var commit_str = null var line = null @@ -164,12 +160,13 @@ for (i = 0; i < length(sorted); i++) { pkg_dir = shop.get_package_dir(locator) is_fetched = fd.is_dir(pkg_dir) || fd.is_link(pkg_dir) - // Check if built (library exists) - lib_dir = shop.get_lib_dir() - lib_name = shop.lib_name_for_package(locator) - dylib_ext = '.dylib' // TODO: detect from target - lib_path = lib_dir + '/' + lib_name + dylib_ext - is_built = fd.is_file(lib_path) + // Check if package has C modules (built on demand) + has_c_files = false + var _check_c = function() { + var c_files = pkg.get_c_files(locator, target_triple, true) + if (c_files && length(c_files) > 0) has_c_files = true + } disruption {} + _check_c() // Format output status_parts = [] @@ -177,7 +174,7 @@ for (i = 0; i < length(sorted); i++) { if (is_local) push(status_parts, "local") if (!is_in_lock) push(status_parts, "not in lock") if (!is_fetched) push(status_parts, "not fetched") - if (is_built) push(status_parts, "built") + if (has_c_files) push(status_parts, "has C modules") commit_str = "" if (lock_entry && lock_entry.commit) { diff --git a/tests/build_audit.cm b/tests/build_audit.cm index 375aff23..947342e0 100644 --- a/tests/build_audit.cm +++ b/tests/build_audit.cm @@ -5,26 +5,6 @@ var shop = use('internal/shop') var runtime = use('runtime') return { - // ======================================================================== - // DETERMINISTIC DYLIB PATHS - // ======================================================================== - - test_dylib_path_deterministic: function() { - var path = shop.get_dylib_path('core', 'time') - if (!ends_with(path, '/lib/core/time.dylib')) return "dylib path should end with /lib/core/time.dylib, got: " + path - }, - - test_dylib_path_internal: function() { - var path = shop.get_dylib_path('core', 'internal/os') - if (!ends_with(path, '/lib/core/internal/os.dylib')) return "dylib path should end with /lib/core/internal/os.dylib, got: " + path - }, - - test_dylib_path_external_package: function() { - var path = shop.get_dylib_path('gitea.pockle.world/john/prosperon', 'sprite') - if (!ends_with(path, '/lib/gitea.pockle.world/john/prosperon/sprite.dylib')) - return "dylib path should mirror package layout, got: " + path - }, - // ======================================================================== // SYMBOL NAMING // ======================================================================== diff --git a/verify.ce b/verify.ce index ee0ed4cf..cda42bba 100644 --- a/verify.ce +++ b/verify.ce @@ -85,10 +85,6 @@ function verify_package(locator) { var current_target = null var expected_target = null var target_dir = null - var lib_dir = null - var lib_name = null - var dylib_ext = null - var lib_path = null var c_files = null checked++ @@ -147,20 +143,9 @@ function verify_package(locator) { } } - // Check build output exists - lib_dir = shop.get_lib_dir() - lib_name = shop.lib_name_for_package(locator) - dylib_ext = '.dylib' // TODO: detect from target - lib_path = lib_dir + '/' + lib_name + dylib_ext - - // Only check for builds if package has C files + // Check if package has C files (builds happen lazily on first use) var _check_build = function() { c_files = pkg.get_c_files(locator, target_triple, true) - if (c_files && length(c_files) > 0) { - if (!fd.is_file(lib_path)) { - add_warning(locator + ": library not built at " + lib_path) - } - } } disruption { // Skip build check if can't determine C files }