Merge branch 'dylib_cache'

build.cm

@@ -80,6 +80,22 @@ function content_hash(str) {
   return text(crypto.blake2(bb, 32), 'h')
 }
 
+// ============================================================================
+// Cache key salts — canonical registry
+// Every artifact type has a unique salt so hash collisions between types
+// are impossible, and no file extensions are needed in build/.
+// ============================================================================
+var SALT_OBJ = 'obj'          // compiled C object file
+var SALT_DYLIB = 'dylib'      // linked dynamic library
+var SALT_NATIVE = 'native'    // native-compiled .cm dylib
+var SALT_MACH = 'mach'        // mach bytecode blob
+var SALT_MCODE = 'mcode'      // mcode IR (JSON)
+var SALT_DEPS = 'deps'        // cached cc -MM dependency list
+
+function cache_path(content, salt) {
+  return get_build_dir() + '/' + content_hash(content + '\n' + salt)
+}
+
 function get_build_dir() {
   return shop.get_build_dir()
 }
@@ -98,6 +114,52 @@ function ensure_dir(path) {
 
 Build.ensure_dir = ensure_dir
 
+// ============================================================================
+// Dependency scanning helpers
+// ============================================================================
+
+// Parse make-style dependency output:
+//   foo.o: foo.c header1.h \
+//     header2.h
+// Returns array of dependency file paths (skips the target)
+function parse_makefile_deps(dep_text) {
+  var joined = replace(dep_text, /\\\n\s*/, ' ')
+  var colon_pos = search(joined, ':')
+  if (colon_pos == null) return []
+  var rest = trim(text(joined, colon_pos + 1))
+  var parts = filter(array(rest, /\s+/), function(p) {
+    return length(p) > 0
+  })
+  return parts
+}
+
+// Run cc -MM to get the preprocessor dependency list.
+// Returns array of dependency file paths.
+function get_c_deps(cc, flags, src_path) {
+  var dep_file = '/tmp/cell_deps_' + content_hash(src_path) + '.d'
+  var dep_cmd = [cc, '-MM', '-MG', '-MF', '"' + dep_file + '"']
+  dep_cmd = array(dep_cmd, flags)
+  push(dep_cmd, '"' + src_path + '"')
+  var ret = os.system(text(dep_cmd, ' ') + ' 2>/dev/null')
+  if (ret != 0) return [src_path]
+  if (!fd.is_file(dep_file)) return [src_path]
+  var dep_text = text(fd.slurp(dep_file))
+  return parse_makefile_deps(dep_text)
+}
+
+// Build a full hash string from the compilation command and all dependency
+// file contents. This is the content key for the object file.
+function hash_all_deps(cmd_str, deps) {
+  var parts = [cmd_str]
+  arrfor(deps, function(dep_path) {
+    if (fd.is_file(dep_path))
+      push(parts, dep_path + '\n' + text(fd.slurp(dep_path)))
+    else
+      push(parts, dep_path + '\n<missing>')
+  })
+  return text(parts, '\n')
+}
+
 // ============================================================================
 // Compilation
 // ============================================================================
@@ -124,30 +186,30 @@ Build.compile_file = function(pkg, file, target, opts) {
   // Symbol name for this file
   var sym_name = shop.c_symbol_for_file(pkg, file)
 
-  // Build command
+  // Build common flags (shared between dep scan and compilation)
-  var cmd_parts = [cc, '-c', '-fPIC']
+  var common_flags = []
 
   // Add buildtype-specific flags
   if (_buildtype == 'release') {
-    cmd_parts = array(cmd_parts, ['-O3', '-DNDEBUG'])
+    common_flags = array(common_flags, ['-O3', '-DNDEBUG'])
   } else if (_buildtype == 'debug') {
-    cmd_parts = array(cmd_parts, ['-O2', '-g'])
+    common_flags = array(common_flags, ['-O2', '-g'])
   } else if (_buildtype == 'minsize') {
-    cmd_parts = array(cmd_parts, ['-Os', '-DNDEBUG'])
+    common_flags = array(common_flags, ['-Os', '-DNDEBUG'])
   }
 
-  push(cmd_parts, '-DCELL_USE_NAME=' + sym_name)
+  push(common_flags, '-DCELL_USE_NAME=' + sym_name)
-  push(cmd_parts, '-I"' + pkg_dir + '"')
+  push(common_flags, '-I"' + pkg_dir + '"')
 
   // Auto-discover include/ directory
   if (fd.is_dir(pkg_dir + '/include')) {
-    push(cmd_parts, '-I"' + pkg_dir + '/include"')
+    push(common_flags, '-I"' + pkg_dir + '/include"')
   }
 
   // External packages need core's source dir for cell.h, quickjs.h, blob.h
   if (pkg != 'core') {
     core_dir = shop.get_package_dir('core')
-    push(cmd_parts, '-I"' + core_dir + '/source"')
+    push(common_flags, '-I"' + core_dir + '/source"')
   }
 
   // Add package CFLAGS (resolve relative -I paths)
@@ -160,14 +222,17 @@ Build.compile_file = function(pkg, file, target, opts) {
         f = '-I"' + pkg_dir + '/' + ipath + '"'
       }
     }
-    push(cmd_parts, f)
+    push(common_flags, f)
   })
 
   // Add target CFLAGS
   arrfor(target_cflags, function(flag) {
-    push(cmd_parts, flag)
+    push(common_flags, flag)
   })
 
+  // Build full compilation command
+  var cmd_parts = [cc, '-c', '-fPIC']
+  cmd_parts = array(cmd_parts, common_flags)
   push(cmd_parts, '"' + src_path + '"')
 
   var cmd_str = text(cmd_parts, ' ')
@@ -177,30 +242,56 @@ Build.compile_file = function(pkg, file, target, opts) {
     print('[verbose] compile: ' + cmd_str)
   }
 
-  // Content hash: command + file content
+  // Two-level cache: quick hash for deps file, full hash for object
   var file_content = fd.slurp(src_path)
-  var hash_input = cmd_str + '\n' + text(file_content)
+  var quick_content = cmd_str + '\n' + text(file_content)
-  var hash = content_hash(hash_input)
+  var deps_path = cache_path(quick_content, SALT_DEPS)
 
   var build_dir = get_build_dir()
   ensure_dir(build_dir)
-  var obj_path = build_dir + '/' + hash
 
-  // Check if already compiled
+  var deps = null
+  var full_content = null
+  var obj_path = null
+
+  // Warm path: read cached dep list, verify by hashing all deps
+  if (fd.is_file(deps_path)) {
+    deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) {
+      return length(p) > 0
+    })
+    full_content = hash_all_deps(cmd_str, deps)
+    obj_path = cache_path(full_content, SALT_OBJ)
     if (fd.is_file(obj_path)) {
       if (_opts.verbose) print('[verbose] cache hit: ' + file)
+      log.shop('cache hit ' + file)
       return obj_path
     }
-  if (_opts.verbose) print('[verbose] cache miss: ' + file)
+    log.shop('cache stale ' + file + ' (header changed)')
+  }
 
-  // Compile — capture stderr to detect missing-header vs real errors
+  // Cold path: run cc -MM to discover deps
-  var err_path = '/tmp/cell_build_err_' + hash + '.log'
+  log.shop('dep scan ' + file)
+  deps = get_c_deps(cc, common_flags, src_path)
+  full_content = hash_all_deps(cmd_str, deps)
+  obj_path = cache_path(full_content, SALT_OBJ)
+
+  // Check if object exists (might exist from previous build with same deps)
+  if (fd.is_file(obj_path)) {
+    fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
+    if (_opts.verbose) print('[verbose] cache hit: ' + file + ' (after dep scan)')
+    log.shop('cache hit ' + file + ' (after dep scan)')
+    return obj_path
+  }
+
+  // Compile
+  log.shop('compiling ' + file)
+  log.console('Compiling ' + file)
+  var err_path = '/tmp/cell_build_err_' + content_hash(src_path) + '.log'
   var full_cmd = cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"'
   var err_text = null
   var missing = null
   var err_lines = null
   var first_err = null
-  log.console('Compiling ' + file)
   var ret = os.system(full_cmd)
   if (ret != 0) {
     if (fd.is_file(err_path)) {
@@ -222,6 +313,8 @@ Build.compile_file = function(pkg, file, target, opts) {
     return null
   }
 
+  // Save deps for future warm-path lookups
+  fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
   return obj_path
 }
 
@@ -249,8 +342,8 @@ Build.build_package = function(pkg, target, exclude_main, buildtype) {
 // Dynamic library building
 // ============================================================================
 
-// Compute link key from all inputs that affect the dylib output
+// Compute link content string from all inputs that affect the dylib output
-function compute_link_key(objects, ldflags, target_ldflags, opts) {
+function compute_link_content(objects, ldflags, target_ldflags, opts) {
   // Sort objects for deterministic hash
   var sorted_objects = sort(objects)
 
@@ -269,7 +362,7 @@ function compute_link_key(objects, ldflags, target_ldflags, opts) {
     push(parts, 'target_ldflag:' + flag)
   })
 
-  return content_hash(text(parts, '\n'))
+  return text(parts, '\n')
 }
 
 // Build a per-module dynamic library for a single C file
@@ -283,7 +376,6 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
   if (!obj) return null
 
   var tc = toolchains[_target]
-  var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
   var cc = tc.cpp || tc.c
   var local_dir = get_local_dir()
   var pkg_dir = shop.get_package_dir(pkg)
@@ -307,10 +399,10 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
   // Content-addressed output: hash of (all objects + link flags + target)
   var all_objects = [obj]
   all_objects = array(all_objects, _extra)
-  var link_key = compute_link_key(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
+  var link_content = compute_link_content(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
   var build_dir = get_build_dir()
   ensure_dir(build_dir)
-  var dylib_path = build_dir + '/' + link_key + '.' + _target + dylib_ext
+  var dylib_path = cache_path(link_content, SALT_DYLIB)
   var cmd_parts = null
   var cmd_str = null
   var ret = null
@@ -352,29 +444,17 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
 
     cmd_str = text(cmd_parts, ' ')
     if (_opts.verbose) print('[verbose] link: ' + cmd_str)
+    log.shop('linking ' + file)
     log.console('Linking module ' + file + ' -> ' + fd.basename(dylib_path))
     ret = os.system(cmd_str)
     if (ret != 0) {
       print('Linking failed: ' + file)
       return null
     }
+  } else {
+    log.shop('link cache hit ' + file)
   }
 
-  // Always install to deterministic lib/<pkg>/<stem>.dylib
-  // Strip .c/.cpp extension so the loader can find it by module name
-  var file_stem = file
-  if (ends_with(file_stem, '.cpp')) file_stem = text(file_stem, 0, -4)
-  else if (ends_with(file_stem, '.c')) file_stem = text(file_stem, 0, -2)
-  var install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
-  var stem_dir = fd.dirname(file_stem)
-  if (stem_dir && stem_dir != '.') {
-    install_dir = install_dir + '/' + stem_dir
-  }
-  ensure_dir(install_dir)
-  var install_path = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) + '/' + file_stem + dylib_ext
-  fd.slurpwrite(install_path, fd.slurp(dylib_path))
-  if (_opts.verbose) print('[verbose] install: ' + install_path)
-
   return dylib_path
 }
 
@@ -602,16 +682,12 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
   var _target = target || Build.detect_host_target()
   var _buildtype = buildtype || 'release'
   var qbe_rt_path = null
-  var native_stem = null
-  var native_install_dir = null
-  var native_install_path = null
 
   if (!fd.is_file(src_path)) {
     print('Source file not found: ' + src_path); disrupt
   }
 
   var tc = toolchains[_target]
-  var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
   var cc = tc.c
 
   // Step 1: Compile through pipeline
@@ -627,16 +703,15 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
   var il_parts = qbe_emit(optimized, qbe_macros, sym_name)
 
   // Content hash for cache key
-  var hash = content_hash(text(fd.slurp(src_path)) + '\n' + _target + '\nnative')
   var build_dir = get_build_dir()
   ensure_dir(build_dir)
 
-  var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
+  var dylib_path = cache_path(text(fd.slurp(src_path)) + '\n' + _target, SALT_NATIVE)
   if (fd.is_file(dylib_path))
     return dylib_path
 
   // Compile and assemble via batched parallel pipeline
-  var tmp = '/tmp/cell_native_' + hash
+  var tmp = '/tmp/cell_native_' + content_hash(src_path)
   var rt_o_path = '/tmp/cell_qbe_rt.o'
 
   var o_paths = compile_native_batched(il_parts, cc, tmp)
@@ -672,15 +747,6 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
 
   log.console('Built native: ' + fd.basename(dylib_path))
 
-  // Install to deterministic lib/<pkg>/<stem>.dylib
-  if (pkg) {
-    native_stem = fd.basename(src_path)
-    native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
-    ensure_dir(native_install_dir)
-    native_install_path = native_install_dir + '/' + native_stem + dylib_ext
-    fd.slurpwrite(native_install_path, fd.slurp(dylib_path))
-  }
-
   return dylib_path
 }
 
@@ -692,12 +758,8 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
   var _buildtype = (opts && opts.buildtype) || 'release'
   var pkg = opts && opts.pkg
   var qbe_rt_path = null
-  var native_stem = null
-  var native_install_dir = null
-  var native_install_path = null
 
   var tc = toolchains[_target]
-  var dylib_ext = tc.system == 'windows' ? '.dll' : (tc.system == 'darwin' ? '.dylib' : '.so')
   var cc = tc.c
 
   var qbe_macros = use('qbe')
@@ -710,16 +772,15 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
   var il_parts = qbe_emit(optimized, qbe_macros, sym_name)
 
   var src = text(fd.slurp(src_path))
-  var hash = content_hash(src + '\n' + _target + '\nnative')
   var build_dir = get_build_dir()
   ensure_dir(build_dir)
 
-  var dylib_path = build_dir + '/' + hash + '.' + _target + dylib_ext
+  var dylib_path = cache_path(src + '\n' + _target, SALT_NATIVE)
   if (fd.is_file(dylib_path))
     return dylib_path
 
   // Compile and assemble via batched parallel pipeline
-  var tmp = '/tmp/cell_native_' + hash
+  var tmp = '/tmp/cell_native_' + content_hash(src_path)
   var rt_o_path = '/tmp/cell_qbe_rt.o'
 
   var o_paths = compile_native_batched(il_parts, cc, tmp)
@@ -755,14 +816,6 @@ Build.compile_native_ir = function(optimized, src_path, opts) {
 
   log.console('Built native: ' + fd.basename(dylib_path))
 
-  if (pkg) {
-    native_stem = fd.basename(src_path)
-    native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
-    ensure_dir(native_install_dir)
-    native_install_path = native_install_dir + '/' + native_stem + dylib_ext
-    fd.slurpwrite(native_install_path, fd.slurp(dylib_path))
-  }
-
   return dylib_path
 }
 
@@ -884,4 +937,13 @@ Build.build_all_dynamic = function(target, buildtype, opts) {
   return results
 }
 
+// Export salt constants and cache_path for shop.cm and others
+Build.SALT_OBJ = SALT_OBJ
+Build.SALT_DYLIB = SALT_DYLIB
+Build.SALT_NATIVE = SALT_NATIVE
+Build.SALT_MACH = SALT_MACH
+Build.SALT_MCODE = SALT_MCODE
+Build.SALT_DEPS = SALT_DEPS
+Build.cache_path = cache_path
+
 return Build

clean.ce

@@ -119,38 +119,14 @@ var build_dir = shop.get_build_dir()
 var packages_dir = replace(shop.get_package_dir(''), /\/$/, '') // Get base packages dir
 
 if (clean_build) {
-  if (is_shop_scope) {
+  // Nuke entire build cache (content-addressed, per-package clean impractical)
-    // Clean entire build and lib directories
   if (fd.is_dir(build_dir)) {
     push(dirs_to_delete, build_dir)
   }
+  // Clean orphaned lib/ directory if it exists (legacy)
   if (fd.is_dir(lib_dir)) {
     push(dirs_to_delete, lib_dir)
   }
-  } else {
-    // Clean specific package libraries
-    arrfor(packages_to_clean, function(p) {
-      if (p == 'core') return
-
-      var lib_name = shop.lib_name_for_package(p)
-      var dylib_ext = '.dylib'
-      var lib_path = lib_dir + '/' + lib_name + dylib_ext
-
-      if (fd.is_file(lib_path)) {
-        push(files_to_delete, lib_path)
-      }
-
-      // Also check for .so and .dll
-      var so_path = lib_dir + '/' + lib_name + '.so'
-      var dll_path = lib_dir + '/' + lib_name + '.dll'
-      if (fd.is_file(so_path)) {
-        push(files_to_delete, so_path)
-      }
-      if (fd.is_file(dll_path)) {
-        push(files_to_delete, dll_path)
-      }
-    })
-  }
 }
 
 if (clean_fetch) {

internal/shop.cm

@@ -54,9 +54,10 @@ function ensure_dir(path) {
   }
 }
 
-function hash_path(content)
+function hash_path(content, salt)
 {
-  return global_shop_path + '/build' + '/' + content_hash(content)
+  var s = salt || 'mach'
+  return global_shop_path + '/build/' + content_hash(stone(blob(text(content) + '\n' + s)))
 }
 
 var Shop = {}
@@ -418,6 +419,7 @@ Shop.extract_commit_hash = function(pkg, response) {
 }
 
 var open_dls = {}
+var package_dylibs = {}  // pkg -> [{file, symbol, dylib}, ...]
 
 // Host target detection for native dylib resolution
 function detect_host_target() {
@@ -434,37 +436,26 @@ function detect_host_target() {
 
 var host_target = detect_host_target()
 
-// Check for a native .cm dylib at the deterministic lib path
+// Check for a native .cm dylib in the build cache
 // Returns a native descriptor {_native, _handle, _sym}, or null if no native dylib exists
-// Also checks staleness: if source has changed, the content-addressed build artifact
-// won't exist for the new hash, so the installed dylib is treated as stale.
 function try_native_mod_dylib(pkg, stem) {
-  var dylib_path = get_dylib_path(pkg, stem)
+  var build_mod = use_cache['core/build']
-  var src_path = null
+  if (!build_mod) return null
-  var src = null
-  var host = null
-  var hash = null
-  var tc_ext = null
-  var build_path = null
-  var handle = null
-  var sym = null
 
-  if (!fd.is_file(dylib_path)) return null
+  var src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem
+  if (!fd.is_file(src_path)) return null
 
-  // Staleness check: verify the content-addressed build artifact exists
+  var src = text(fd.slurp(src_path))
-  src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem
+  var host = detect_host_target()
-  if (fd.is_file(src_path)) {
+  if (!host) return null
-    src = text(fd.slurp(src_path))
+
-    host = detect_host_target()
+  var build_path = build_mod.cache_path(src + '\n' + host, build_mod.SALT_NATIVE)
-    hash = content_hash(src + '\n' + host + '\nnative')
-    tc_ext = dylib_ext
-    build_path = global_shop_path + '/build/' + hash + '.' + host + tc_ext
   if (!fd.is_file(build_path)) return null
-  }
 
-  handle = os.dylib_open(dylib_path)
+  log.shop('native dylib cache hit: ' + stem)
+  var handle = os.dylib_open(build_path)
   if (!handle) return null
-  sym = Shop.c_symbol_for_file(pkg, stem)
+  var sym = Shop.c_symbol_for_file(pkg, stem)
   return {_native: true, _handle: handle, _sym: sym}
 }
 
@@ -675,9 +666,6 @@ function resolve_mod_fn(path, pkg) {
   var cached = null
   var ast = null
   var compiled = null
-  var mach_path = null
-  var mach_blob = null
-  var mcode_path = null
   var ir = null
   var optimized = null
   var mcode_json = null
@@ -729,9 +717,9 @@ function resolve_mod_fn(path, pkg) {
     }
   }
 
-  // Check for cached mcode in content-addressed store (salted hash to distinguish from mach)
+  // Check for cached mcode in content-addressed store
   if (policy.allow_compile) {
-    cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
+    cached_mcode_path = hash_path(content_key, 'mcode')
     if (fd.is_file(cached_mcode_path)) {
       mcode_json = text(fd.slurp(cached_mcode_path))
       compiled = mach_compile_mcode_bin(path, mcode_json)
@@ -855,52 +843,67 @@ function make_c_symbol(pkg, file) {
   return 'js_' + pkg_safe + '_' + file_safe + '_use'
 }
 
-// Get the deterministic dylib path for a module in lib/<pkg>/<stem>.dylib
+// Ensure all C modules for a package are built and loaded.
-function get_dylib_path(pkg, stem) {
+// Returns the array of {file, symbol, dylib} results, cached per package.
-  return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + dylib_ext
+function ensure_package_dylibs(pkg) {
+  if (package_dylibs[pkg]) return package_dylibs[pkg]
+
+  var build_mod = use_cache['core/build']
+  if (!build_mod) return null
+
+  var target = detect_host_target()
+  if (!target) return null
+
+  var c_files = pkg_tools.get_c_files(pkg, target, true)
+  if (!c_files || length(c_files) == 0) {
+    package_dylibs[pkg] = []
+    return []
   }
 
-// Get the deterministic mach path for a module in lib/<pkg>/<stem>.mach
+  log.shop('ensuring C modules for ' + pkg)
-function get_mach_path(pkg, stem) {
+  var results = build_mod.build_dynamic(pkg, target, 'release', {})
-  return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + '.mach'
+  package_dylibs[pkg] = results
-}
 
-// Open a per-module dylib and return the dlopen handle
+  // Preload all sibling dylibs with RTLD_LAZY|RTLD_GLOBAL
-// Pre-loads sibling dylibs with RTLD_LAZY|RTLD_GLOBAL so cross-dylib symbols resolve
+  arrfor(results, function(r) {
-function open_module_dylib(dylib_path) {
-  if (open_dls[dylib_path]) return open_dls[dylib_path]
-  if (!fd.is_file(dylib_path)) return null
-  var dir = fd.dirname(dylib_path)
-  var entries = fd.readdir(dir)
-  var i = 0
-  var sibling = null
     var handle = null
-  while (i < length(entries)) {
+    if (r.dylib && !open_dls[r.dylib]) {
-    if (ends_with(entries[i], dylib_ext) && entries[i] != fd.basename(dylib_path)) {
+      handle = os.dylib_preload(r.dylib)
-      sibling = dir + '/' + entries[i]
+      if (handle) open_dls[r.dylib] = handle
-      if (!open_dls[sibling]) {
-        handle = os.dylib_preload(sibling)
-        if (handle) open_dls[sibling] = handle
     }
-    }
+  })
-    i = i + 1
+
-  }
+  log.shop('built ' + text(length(results)) + ' C module(s) for ' + pkg)
-  open_dls[dylib_path] = os.dylib_open(dylib_path)
+  return results
-  return open_dls[dylib_path]
 }
 
-// Try to resolve a C symbol from the deterministic dylib path
+// Try to resolve a C symbol by building the package on demand
 // Returns a loader function or null
 function try_dylib_symbol(sym, pkg, file_stem) {
-  var dylib_path = get_dylib_path(pkg, file_stem)
+  var dylibs = ensure_package_dylibs(pkg)
-  var handle = open_module_dylib(dylib_path)
+  if (!dylibs || length(dylibs) == 0) return null
+
+  var c_file = file_stem + '.c'
+  var cpp_file = file_stem + '.cpp'
+  var entry = find(dylibs, function(r) {
+    return r.file == c_file || r.file == cpp_file
+  })
+  if (!entry || !entry.dylib) return null
+
+  var handle = open_dls[entry.dylib]
+  if (!handle) {
+    handle = os.dylib_open(entry.dylib)
+    if (handle) open_dls[entry.dylib] = handle
+  }
   if (!handle) return null
   if (!os.dylib_has_symbol(handle, sym)) return null
+
+  log.shop('resolved ' + sym + ' from build cache')
   return function() { return os.dylib_symbol(handle, sym) }
 }
 
 // Resolve a C symbol by searching:
-// At each scope: check lib/ dylib first, then internal (static)
+// At each scope: check build-cache dylib first, then internal (static)
 function resolve_c_symbol(path, package_context) {
   var explicit = split_explicit_package_import(path)
   var sym = null
@@ -922,7 +925,7 @@ function resolve_c_symbol(path, package_context) {
     sym = make_c_symbol(explicit.package, explicit.path)
     file_stem = replace(explicit.path, '.c', '')
 
-    // Check lib/ dylib first
+    // Check build-cache dylib first
     if (policy.allow_dylib) {
       loader = try_dylib_symbol(sym, explicit.package, file_stem)
       if (loader) {
@@ -950,7 +953,7 @@ function resolve_c_symbol(path, package_context) {
   if (!package_context || package_context == 'core') {
     core_sym = make_c_symbol('core', path)
 
-    // Check lib/ dylib first for core
+    // Check build-cache dylib first for core
     if (policy.allow_dylib) {
       loader = try_dylib_symbol(core_sym, 'core', path)
       if (loader) {
@@ -972,7 +975,7 @@ function resolve_c_symbol(path, package_context) {
     return null
   }
 
-  // 1. Check own package (dylib first, then internal)
+  // 1. Check own package (build-cache dylib first, then internal)
   sym = make_c_symbol(package_context, path)
 
   if (policy.allow_dylib) {
@@ -1028,7 +1031,7 @@ function resolve_c_symbol(path, package_context) {
     }
   }
 
-  // 3. Check core (dylib first, then internal)
+  // 3. Check core (build-cache dylib first, then internal)
   core_sym = make_c_symbol('core', path)
 
   if (policy.allow_dylib) {
@@ -1541,11 +1544,8 @@ Shop.remove = function(pkg) {
     fd.rmdir(pkg_dir, 1)
   }
 
-  // Remove built dylibs
+  // Invalidate package dylib cache
-  var lib_dir = global_shop_path + '/lib/' + safe_package_path(pkg)
+  package_dylibs[pkg] = null
-  if (fd.is_dir(lib_dir)) {
-    fd.rmdir(lib_dir, 1)
-  }
 
   log.console("Removed " + pkg)
   return true
@@ -1599,14 +1599,9 @@ Shop.module_reload = function(path, package) {
   var lookup_key = package ? package + ':' + path : ':' + path
   module_info_cache[lookup_key] = null
 
-  // Close old dylib handle if any
+  // Invalidate package dylib cache so next resolve triggers rebuild
-  var old_dylib_path = null
   if (package) {
-    old_dylib_path = get_dylib_path(package, path)
+    package_dylibs[package] = null
-    if (open_dls[old_dylib_path]) {
-      os.dylib_close(open_dls[old_dylib_path])
-      open_dls[old_dylib_path] = null
-    }
   }
 
   var info = resolve_module_info(path, package)
@@ -1717,17 +1712,6 @@ Shop.lib_name_for_package = function(pkg) {
   return safe_package_path(pkg)
 }
 
-// Returns { ok: bool, results: [{pkg, ok, error}] }
-// Get the deterministic dylib path for a module (public API)
-Shop.get_dylib_path = function(pkg, stem) {
-  return get_dylib_path(pkg, stem)
-}
-
-// Get the deterministic mach path for a module (public API)
-Shop.get_mach_path = function(pkg, stem) {
-  return get_mach_path(pkg, stem)
-}
-
 // Load a module explicitly as mach bytecode, bypassing dylib resolution.
 // Returns the loaded module value. Disrupts if the module cannot be found.
 Shop.load_as_mach = function(path, pkg) {
@@ -1742,9 +1726,6 @@ Shop.load_as_mach = function(path, pkg) {
   var ast = null
   var ir = null
   var optimized = null
-  var pkg_dir = null
-  var stem = null
-  var mach_path = null
   var file_info = null
   var inject = null
   var env = null
@@ -1755,27 +1736,13 @@ Shop.load_as_mach = function(path, pkg) {
   content = text(fd.slurp(file_path))
   content_key = stone(blob(content))
 
-  // Try installed .mach in lib/
-  if (pkg) {
-    pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
-    if (starts_with(file_path, pkg_dir + '/')) {
-      stem = text(file_path, length(pkg_dir) + 1)
-      mach_path = get_mach_path(pkg, stem)
-      if (fd.is_file(mach_path)) {
-        compiled = fd.slurp(mach_path)
-      }
-    }
-  }
-
   // Try cached mach blob
-  if (!compiled) {
   cached = pull_from_cache(content_key)
   if (cached) compiled = cached
-  }
 
   // Try cached mcode -> compile to mach
   if (!compiled) {
-    cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
+    cached_mcode_path = hash_path(content_key, 'mcode')
     if (fd.is_file(cached_mcode_path)) {
       mcode_json = text(fd.slurp(cached_mcode_path))
       compiled = mach_compile_mcode_bin(file_path, mcode_json)
@@ -1795,7 +1762,7 @@ Shop.load_as_mach = function(path, pkg) {
     ir = _mcode_mod(ast)
     optimized = _streamline_mod(ir)
     mcode_json = shop_json.encode(optimized)
-    cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
+    cached_mcode_path = hash_path(content_key, 'mcode')
     ensure_dir(global_shop_path + '/build')
     fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
     compiled = mach_compile_mcode_bin(file_path, mcode_json)

resolve.ce

@@ -130,11 +130,7 @@ var is_linked = false
 var is_in_lock = false
 var is_local = false
 var is_fetched = false
-var lib_dir = null
+var has_c_files = false
-var lib_name = null
-var dylib_ext = null
-var lib_path = null
-var is_built = false
 var status_parts = null
 var commit_str = null
 var line = null
@@ -164,12 +160,13 @@ for (i = 0; i < length(sorted); i++) {
   pkg_dir = shop.get_package_dir(locator)
   is_fetched = fd.is_dir(pkg_dir) || fd.is_link(pkg_dir)
 
-  // Check if built (library exists)
+  // Check if package has C modules (built on demand)
-  lib_dir = shop.get_lib_dir()
+  has_c_files = false
-  lib_name = shop.lib_name_for_package(locator)
+  var _check_c = function() {
-  dylib_ext = '.dylib' // TODO: detect from target
+    var c_files = pkg.get_c_files(locator, target_triple, true)
-  lib_path = lib_dir + '/' + lib_name + dylib_ext
+    if (c_files && length(c_files) > 0) has_c_files = true
-  is_built = fd.is_file(lib_path)
+  } disruption {}
+  _check_c()
 
   // Format output
   status_parts = []
@@ -177,7 +174,7 @@ for (i = 0; i < length(sorted); i++) {
   if (is_local) push(status_parts, "local")
   if (!is_in_lock) push(status_parts, "not in lock")
   if (!is_fetched) push(status_parts, "not fetched")
-  if (is_built) push(status_parts, "built")
+  if (has_c_files) push(status_parts, "has C modules")
 
   commit_str = ""
   if (lock_entry && lock_entry.commit) {

tests/build_audit.cm

@@ -5,26 +5,6 @@ var shop = use('internal/shop')
 var runtime = use('runtime')
 
 return {
-  // ========================================================================
-  // DETERMINISTIC DYLIB PATHS
-  // ========================================================================
-
-  test_dylib_path_deterministic: function() {
-    var path = shop.get_dylib_path('core', 'time')
-    if (!ends_with(path, '/lib/core/time.dylib')) return "dylib path should end with /lib/core/time.dylib, got: " + path
-  },
-
-  test_dylib_path_internal: function() {
-    var path = shop.get_dylib_path('core', 'internal/os')
-    if (!ends_with(path, '/lib/core/internal/os.dylib')) return "dylib path should end with /lib/core/internal/os.dylib, got: " + path
-  },
-
-  test_dylib_path_external_package: function() {
-    var path = shop.get_dylib_path('gitea.pockle.world/john/prosperon', 'sprite')
-    if (!ends_with(path, '/lib/gitea.pockle.world/john/prosperon/sprite.dylib'))
-      return "dylib path should mirror package layout, got: " + path
-  },
-
   // ========================================================================
   // SYMBOL NAMING
   // ========================================================================

verify.ce

@@ -85,10 +85,6 @@ function verify_package(locator) {
   var current_target = null
   var expected_target = null
   var target_dir = null
-  var lib_dir = null
-  var lib_name = null
-  var dylib_ext = null
-  var lib_path = null
   var c_files = null
 
   checked++
@@ -147,20 +143,9 @@ function verify_package(locator) {
     }
   }
 
-  // Check build output exists
+  // Check if package has C files (builds happen lazily on first use)
-  lib_dir = shop.get_lib_dir()
-  lib_name = shop.lib_name_for_package(locator)
-  dylib_ext = '.dylib' // TODO: detect from target
-  lib_path = lib_dir + '/' + lib_name + dylib_ext
-
-  // Only check for builds if package has C files
   var _check_build = function() {
     c_files = pkg.get_c_files(locator, target_triple, true)
-    if (c_files && length(c_files) > 0) {
-      if (!fd.is_file(lib_path)) {
-        add_warning(locator + ": library not built at " + lib_path)
-      }
-    }
   } disruption {
     // Skip build check if can't determine C files
   }