better cache handling

internal/shop.cm

@@ -54,9 +54,10 @@ function ensure_dir(path) {
   }
 }
 
-function hash_path(content)
+function hash_path(content, salt)
 {
-  return global_shop_path + '/build' + '/' + content_hash(content)
+  var s = salt || 'mach'
+  return global_shop_path + '/build/' + content_hash(stone(blob(text(content) + '\n' + s)))
 }
 
 var Shop = {}
@@ -418,6 +419,7 @@ Shop.extract_commit_hash = function(pkg, response) {
 }
 
 var open_dls = {}
+var package_dylibs = {}  // pkg -> [{file, symbol, dylib}, ...]
 
 // Host target detection for native dylib resolution
 function detect_host_target() {
@@ -434,37 +436,26 @@ function detect_host_target() {
 
 var host_target = detect_host_target()
 
-// Check for a native .cm dylib at the deterministic lib path
+// Check for a native .cm dylib in the build cache
 // Returns a native descriptor {_native, _handle, _sym}, or null if no native dylib exists
-// Also checks staleness: if source has changed, the content-addressed build artifact
-// won't exist for the new hash, so the installed dylib is treated as stale.
 function try_native_mod_dylib(pkg, stem) {
-  var dylib_path = get_dylib_path(pkg, stem)
-  var src_path = null
-  var src = null
-  var host = null
-  var hash = null
-  var tc_ext = null
-  var build_path = null
-  var handle = null
-  var sym = null
+  var build_mod = use_cache['core/build']
+  if (!build_mod) return null
 
-  if (!fd.is_file(dylib_path)) return null
+  var src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem
+  if (!fd.is_file(src_path)) return null
 
-  // Staleness check: verify the content-addressed build artifact exists
-  src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem
-  if (fd.is_file(src_path)) {
-    src = text(fd.slurp(src_path))
-    host = detect_host_target()
-    hash = content_hash(src + '\n' + host + '\nnative')
-    tc_ext = dylib_ext
-    build_path = global_shop_path + '/build/' + hash + '.' + host + tc_ext
-    if (!fd.is_file(build_path)) return null
-  }
+  var src = text(fd.slurp(src_path))
+  var host = detect_host_target()
+  if (!host) return null
 
-  handle = os.dylib_open(dylib_path)
+  var build_path = build_mod.cache_path(src + '\n' + host, build_mod.SALT_NATIVE)
+  if (!fd.is_file(build_path)) return null
+
+  log.shop('native dylib cache hit: ' + stem)
+  var handle = os.dylib_open(build_path)
   if (!handle) return null
-  sym = Shop.c_symbol_for_file(pkg, stem)
+  var sym = Shop.c_symbol_for_file(pkg, stem)
   return {_native: true, _handle: handle, _sym: sym}
 }
 
@@ -675,9 +666,6 @@ function resolve_mod_fn(path, pkg) {
   var cached = null
   var ast = null
   var compiled = null
-  var mach_path = null
-  var mach_blob = null
-  var mcode_path = null
   var ir = null
   var optimized = null
   var mcode_json = null
@@ -729,9 +717,9 @@ function resolve_mod_fn(path, pkg) {
     }
   }
 
-  // Check for cached mcode in content-addressed store (salted hash to distinguish from mach)
+  // Check for cached mcode in content-addressed store
   if (policy.allow_compile) {
-    cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
+    cached_mcode_path = hash_path(content_key, 'mcode')
     if (fd.is_file(cached_mcode_path)) {
       mcode_json = text(fd.slurp(cached_mcode_path))
       compiled = mach_compile_mcode_bin(path, mcode_json)
@@ -855,52 +843,67 @@ function make_c_symbol(pkg, file) {
   return 'js_' + pkg_safe + '_' + file_safe + '_use'
 }
 
-// Get the deterministic dylib path for a module in lib/<pkg>/<stem>.dylib
-function get_dylib_path(pkg, stem) {
-  return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + dylib_ext
-}
+// Ensure all C modules for a package are built and loaded.
+// Returns the array of {file, symbol, dylib} results, cached per package.
+function ensure_package_dylibs(pkg) {
+  if (package_dylibs[pkg]) return package_dylibs[pkg]
 
-// Get the deterministic mach path for a module in lib/<pkg>/<stem>.mach
-function get_mach_path(pkg, stem) {
-  return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + '.mach'
-}
+  var build_mod = use_cache['core/build']
+  if (!build_mod) return null
 
-// Open a per-module dylib and return the dlopen handle
-// Pre-loads sibling dylibs with RTLD_LAZY|RTLD_GLOBAL so cross-dylib symbols resolve
-function open_module_dylib(dylib_path) {
-  if (open_dls[dylib_path]) return open_dls[dylib_path]
-  if (!fd.is_file(dylib_path)) return null
-  var dir = fd.dirname(dylib_path)
-  var entries = fd.readdir(dir)
-  var i = 0
-  var sibling = null
-  var handle = null
-  while (i < length(entries)) {
-    if (ends_with(entries[i], dylib_ext) && entries[i] != fd.basename(dylib_path)) {
-      sibling = dir + '/' + entries[i]
-      if (!open_dls[sibling]) {
-        handle = os.dylib_preload(sibling)
-        if (handle) open_dls[sibling] = handle
-      }
-    }
-    i = i + 1
+  var target = detect_host_target()
+  if (!target) return null
+
+  var c_files = pkg_tools.get_c_files(pkg, target, true)
+  if (!c_files || length(c_files) == 0) {
+    package_dylibs[pkg] = []
+    return []
   }
-  open_dls[dylib_path] = os.dylib_open(dylib_path)
-  return open_dls[dylib_path]
+
+  log.shop('ensuring C modules for ' + pkg)
+  var results = build_mod.build_dynamic(pkg, target, 'release', {})
+  package_dylibs[pkg] = results
+
+  // Preload all sibling dylibs with RTLD_LAZY|RTLD_GLOBAL
+  arrfor(results, function(r) {
+    var handle = null
+    if (r.dylib && !open_dls[r.dylib]) {
+      handle = os.dylib_preload(r.dylib)
+      if (handle) open_dls[r.dylib] = handle
+    }
+  })
+
+  log.shop('built ' + text(length(results)) + ' C module(s) for ' + pkg)
+  return results
 }
 
-// Try to resolve a C symbol from the deterministic dylib path
+// Try to resolve a C symbol by building the package on demand
 // Returns a loader function or null
 function try_dylib_symbol(sym, pkg, file_stem) {
-  var dylib_path = get_dylib_path(pkg, file_stem)
-  var handle = open_module_dylib(dylib_path)
+  var dylibs = ensure_package_dylibs(pkg)
+  if (!dylibs || length(dylibs) == 0) return null
+
+  var c_file = file_stem + '.c'
+  var cpp_file = file_stem + '.cpp'
+  var entry = find(dylibs, function(r) {
+    return r.file == c_file || r.file == cpp_file
+  })
+  if (!entry || !entry.dylib) return null
+
+  var handle = open_dls[entry.dylib]
+  if (!handle) {
+    handle = os.dylib_open(entry.dylib)
+    if (handle) open_dls[entry.dylib] = handle
+  }
   if (!handle) return null
   if (!os.dylib_has_symbol(handle, sym)) return null
+
+  log.shop('resolved ' + sym + ' from build cache')
   return function() { return os.dylib_symbol(handle, sym) }
 }
 
 // Resolve a C symbol by searching:
-// At each scope: check lib/ dylib first, then internal (static)
+// At each scope: check build-cache dylib first, then internal (static)
 function resolve_c_symbol(path, package_context) {
   var explicit = split_explicit_package_import(path)
   var sym = null
@@ -922,7 +925,7 @@ function resolve_c_symbol(path, package_context) {
     sym = make_c_symbol(explicit.package, explicit.path)
     file_stem = replace(explicit.path, '.c', '')
 
-    // Check lib/ dylib first
+    // Check build-cache dylib first
     if (policy.allow_dylib) {
       loader = try_dylib_symbol(sym, explicit.package, file_stem)
       if (loader) {
@@ -950,7 +953,7 @@ function resolve_c_symbol(path, package_context) {
   if (!package_context || package_context == 'core') {
     core_sym = make_c_symbol('core', path)
 
-    // Check lib/ dylib first for core
+    // Check build-cache dylib first for core
     if (policy.allow_dylib) {
       loader = try_dylib_symbol(core_sym, 'core', path)
       if (loader) {
@@ -972,7 +975,7 @@ function resolve_c_symbol(path, package_context) {
     return null
   }
 
-  // 1. Check own package (dylib first, then internal)
+  // 1. Check own package (build-cache dylib first, then internal)
   sym = make_c_symbol(package_context, path)
 
   if (policy.allow_dylib) {
@@ -1028,7 +1031,7 @@ function resolve_c_symbol(path, package_context) {
     }
   }
 
-  // 3. Check core (dylib first, then internal)
+  // 3. Check core (build-cache dylib first, then internal)
   core_sym = make_c_symbol('core', path)
 
   if (policy.allow_dylib) {
@@ -1541,11 +1544,8 @@ Shop.remove = function(pkg) {
     fd.rmdir(pkg_dir, 1)
   }
 
-  // Remove built dylibs
-  var lib_dir = global_shop_path + '/lib/' + safe_package_path(pkg)
-  if (fd.is_dir(lib_dir)) {
-    fd.rmdir(lib_dir, 1)
-  }
+  // Invalidate package dylib cache
+  package_dylibs[pkg] = null
 
   log.console("Removed " + pkg)
   return true
@@ -1599,14 +1599,9 @@ Shop.module_reload = function(path, package) {
   var lookup_key = package ? package + ':' + path : ':' + path
   module_info_cache[lookup_key] = null
 
-  // Close old dylib handle if any
-  var old_dylib_path = null
+  // Invalidate package dylib cache so next resolve triggers rebuild
   if (package) {
-    old_dylib_path = get_dylib_path(package, path)
-    if (open_dls[old_dylib_path]) {
-      os.dylib_close(open_dls[old_dylib_path])
-      open_dls[old_dylib_path] = null
-    }
+    package_dylibs[package] = null
   }
 
   var info = resolve_module_info(path, package)
@@ -1717,17 +1712,6 @@ Shop.lib_name_for_package = function(pkg) {
   return safe_package_path(pkg)
 }
 
-// Returns { ok: bool, results: [{pkg, ok, error}] }
-// Get the deterministic dylib path for a module (public API)
-Shop.get_dylib_path = function(pkg, stem) {
-  return get_dylib_path(pkg, stem)
-}
-
-// Get the deterministic mach path for a module (public API)
-Shop.get_mach_path = function(pkg, stem) {
-  return get_mach_path(pkg, stem)
-}
-
 // Load a module explicitly as mach bytecode, bypassing dylib resolution.
 // Returns the loaded module value. Disrupts if the module cannot be found.
 Shop.load_as_mach = function(path, pkg) {
@@ -1742,9 +1726,6 @@ Shop.load_as_mach = function(path, pkg) {
   var ast = null
   var ir = null
   var optimized = null
-  var pkg_dir = null
-  var stem = null
-  var mach_path = null
   var file_info = null
   var inject = null
   var env = null
@@ -1755,27 +1736,13 @@ Shop.load_as_mach = function(path, pkg) {
   content = text(fd.slurp(file_path))
   content_key = stone(blob(content))
 
-  // Try installed .mach in lib/
-  if (pkg) {
-    pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
-    if (starts_with(file_path, pkg_dir + '/')) {
-      stem = text(file_path, length(pkg_dir) + 1)
-      mach_path = get_mach_path(pkg, stem)
-      if (fd.is_file(mach_path)) {
-        compiled = fd.slurp(mach_path)
-      }
-    }
-  }
-
   // Try cached mach blob
-  if (!compiled) {
-    cached = pull_from_cache(content_key)
-    if (cached) compiled = cached
-  }
+  cached = pull_from_cache(content_key)
+  if (cached) compiled = cached
 
   // Try cached mcode -> compile to mach
   if (!compiled) {
-    cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
+    cached_mcode_path = hash_path(content_key, 'mcode')
     if (fd.is_file(cached_mcode_path)) {
       mcode_json = text(fd.slurp(cached_mcode_path))
       compiled = mach_compile_mcode_bin(file_path, mcode_json)
@@ -1795,7 +1762,7 @@ Shop.load_as_mach = function(path, pkg) {
     ir = _mcode_mod(ast)
     optimized = _streamline_mod(ir)
     mcode_json = shop_json.encode(optimized)
-    cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
+    cached_mcode_path = hash_path(content_key, 'mcode')
     ensure_dir(global_shop_path + '/build')
     fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
     compiled = mach_compile_mcode_bin(file_path, mcode_json)