Merge branch 'master' into fix_aot

2026-02-18 23:55:17 -06:00
parent e004b2c472 777474ab4f
commit e59bfe19f7
23 changed files with 682 additions and 649 deletions
--- a/internal/engine.cm
+++ b/internal/engine.cm
@@ -574,7 +574,7 @@ log = function(name, args) {
    caller = {file: c_stack[0].file, line: c_stack[0].line}
    if (stack_channels[name]) stack = c_stack
  } else {
-    caller = caller_info(2)
+    caller = caller_info(0)
    if (stack_channels[name]) stack = os.stack(1)
  }

--- a/internal/os.c
+++ b/internal/os.c
@@ -1,4 +1,5 @@
 #include "cell.h"
+#include "cell_internal.h"

 #include <sys/stat.h>
 #include <sys/types.h>
@@ -306,8 +307,18 @@ static JSValue js_os_rusage(JSContext *js, JSValue self, int argc, JSValue *argv

 JSC_SCALL(os_system,
  int err = system(str);
-  JS_SetPauseFlag(js, 0);
-  ret = number2js(js,err);
+  /* Reset actor turn timer after blocking system() call.
+     The scheduler's kill timer may have fired while system() blocked,
+     setting pause_flag = 2.  Bump turn_gen so stale timer events are
+     ignored, and clear the pause flag so the VM doesn't raise
+     "interrupted" on the next backward branch. */
+  cell_rt *crt = JS_GetContextOpaque(js);
+  if (crt) {
+    atomic_fetch_add_explicit(&crt->turn_gen, 1, memory_order_relaxed);
+    JS_SetPauseFlag(js, 0);
+    crt->turn_start_ns = cell_ns();
+  }
+  ret = number2js(js, err);
 )

 JSC_CCALL(os_exit,
--- a/internal/shop.cm
+++ b/internal/shop.cm
@@ -54,9 +54,10 @@ function ensure_dir(path) {
  }
 }

-function hash_path(content)
+function hash_path(content, salt)
 {
-  return global_shop_path + '/build' + '/' + content_hash(content)
+  var s = salt || 'mach'
+  return global_shop_path + '/build/' + content_hash(stone(blob(text(content) + '\n' + s)))
 }

 var Shop = {}
@@ -418,6 +419,7 @@ Shop.extract_commit_hash = function(pkg, response) {
 }

 var open_dls = {}
+var package_dylibs = {}  // pkg -> [{file, symbol, dylib}, ...]

 // Host target detection for native dylib resolution
 function detect_host_target() {
@@ -434,49 +436,30 @@ function detect_host_target() {

 var host_target = detect_host_target()

-// Must match build.cm NATIVE_CACHE_VERSION to detect stale native artifacts.
-def NATIVE_CACHE_VERSION = "native-v23"
-
-function native_sanitize_flags() {
-  if (fd.is_file('.cell/asan_aot')) {
-    return ' -fsanitize=address -fno-omit-frame-pointer'
-  }
-  return ''
-}
-
-// Check for a native .cm dylib at the deterministic lib path
+// Check for a native .cm dylib in the build cache
 // Returns a native descriptor {_native, _handle, _sym}, or null if no native dylib exists
-// Also checks staleness: if source has changed, the content-addressed build artifact
-// won't exist for the new hash, so the installed dylib is treated as stale.
 function try_native_mod_dylib(pkg, stem) {
-  var dylib_path = get_dylib_path(pkg, stem)
-  var src_path = null
-  var src = null
-  var host = null
-  var hash = null
-  var san_flags = null
-  var tc_ext = null
-  var build_path = null
-  var handle = null
-  var sym = null
+  var build_mod = use_cache['core/build']
+  if (!build_mod) return null

-  if (!fd.is_file(dylib_path)) return null
+  var src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem
+  if (!fd.is_file(src_path)) return null

-  // Staleness check: verify the content-addressed build artifact exists
-  src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem
-  if (fd.is_file(src_path)) {
-    src = text(fd.slurp(src_path))
-    host = detect_host_target()
-    san_flags = native_sanitize_flags()
-    hash = content_hash(src + '\n' + host + '\nnative\n' + NATIVE_CACHE_VERSION + '\n' + san_flags)
-    tc_ext = dylib_ext
-    build_path = global_shop_path + '/build/' + hash + '.' + host + tc_ext
-    if (!fd.is_file(build_path)) return null
-  }
+  var src = text(fd.slurp(src_path))
+  var host = detect_host_target()
+  if (!host) return null

-  handle = os.dylib_open(dylib_path)
+  var san_flags = build_mod.native_sanitize_flags ? build_mod.native_sanitize_flags() : ''
+  var native_key = build_mod.native_cache_content ?
+    build_mod.native_cache_content(src, host, san_flags) :
+    (src + '\n' + host)
+  var build_path = build_mod.cache_path(native_key, build_mod.SALT_NATIVE)
+  if (!fd.is_file(build_path)) return null
+
+  log.shop('native dylib cache hit: ' + stem)
+  var handle = os.dylib_open(build_path)
  if (!handle) return null
-  sym = Shop.c_symbol_for_file(pkg, stem)
+  var sym = Shop.c_symbol_for_file(pkg, stem)
  return {_native: true, _handle: handle, _sym: sym}
 }

@@ -687,9 +670,6 @@ function resolve_mod_fn(path, pkg) {
  var cached = null
  var ast = null
  var compiled = null
-  var mach_path = null
-  var mach_blob = null
-  var mcode_path = null
  var ir = null
  var optimized = null
  var mcode_json = null
@@ -741,9 +721,9 @@ function resolve_mod_fn(path, pkg) {
    }
  }

-  // Check for cached mcode in content-addressed store (salted hash to distinguish from mach)
+  // Check for cached mcode in content-addressed store
  if (policy.allow_compile) {
-    cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
+    cached_mcode_path = hash_path(content_key, 'mcode')
    if (fd.is_file(cached_mcode_path)) {
      mcode_json = text(fd.slurp(cached_mcode_path))
      compiled = mach_compile_mcode_bin(path, mcode_json)
@@ -867,52 +847,97 @@ function make_c_symbol(pkg, file) {
  return 'js_' + pkg_safe + '_' + file_safe + '_use'
 }

-// Get the deterministic dylib path for a module in lib/<pkg>/<stem>.dylib
-function get_dylib_path(pkg, stem) {
-  return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + dylib_ext
+// Compute the manifest path for a package (must match build.cm's manifest_path)
+function dylib_manifest_path(pkg) {
+  var hash = content_hash(stone(blob(pkg + '\n' + 'manifest')))
+  return global_shop_path + '/build/' + hash
 }

-// Get the deterministic mach path for a module in lib/<pkg>/<stem>.mach
-function get_mach_path(pkg, stem) {
-  return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + '.mach'
+// Read a pre-built dylib manifest for a package.
+// Returns the array of {file, symbol, dylib} or null.
+function read_dylib_manifest(pkg) {
+  var mpath = dylib_manifest_path(pkg)
+  if (!fd.is_file(mpath)) return null
+  var content = text(fd.slurp(mpath))
+  if (!content || length(content) == 0) return null
+  return json.decode(content)
 }

-// Open a per-module dylib and return the dlopen handle
-// Pre-loads sibling dylibs with RTLD_LAZY|RTLD_GLOBAL so cross-dylib symbols resolve
-function open_module_dylib(dylib_path) {
-  if (open_dls[dylib_path]) return open_dls[dylib_path]
-  if (!fd.is_file(dylib_path)) return null
-  var dir = fd.dirname(dylib_path)
-  var entries = fd.readdir(dir)
-  var i = 0
-  var sibling = null
-  var handle = null
-  while (i < length(entries)) {
-    if (ends_with(entries[i], dylib_ext) && entries[i] != fd.basename(dylib_path)) {
-      sibling = dir + '/' + entries[i]
-      if (!open_dls[sibling]) {
-        handle = os.dylib_preload(sibling)
-        if (handle) open_dls[sibling] = handle
-      }
-    }
-    i = i + 1
+// Ensure all C modules for a package are built and loaded.
+// Returns the array of {file, symbol, dylib} results, cached per package.
+function ensure_package_dylibs(pkg) {
+  if (package_dylibs[pkg] != null) return package_dylibs[pkg]
+  if (pkg == 'core') {
+    package_dylibs[pkg] = []
+    return []
  }
-  open_dls[dylib_path] = os.dylib_open(dylib_path)
-  return open_dls[dylib_path]
+
+  var results = null
+  var build_mod = use_cache['core/build']
+  var target = null
+  var c_files = null
+
+  if (build_mod) {
+    target = detect_host_target()
+    if (!target) return null
+
+    c_files = pkg_tools.get_c_files(pkg, target, true)
+    if (!c_files || length(c_files) == 0) {
+      package_dylibs[pkg] = []
+      return []
+    }
+
+    log.shop('ensuring C modules for ' + pkg)
+    results = build_mod.build_dynamic(pkg, target, 'release', {})
+  } else {
+    // No build module at runtime — read manifest from cell build
+    results = read_dylib_manifest(pkg)
+    if (!results) return null
+    log.shop('loaded manifest for ' + pkg + ' (' + text(length(results)) + ' modules)')
+  }
+
+  if (results == null) results = []
+  package_dylibs[pkg] = results
+
+  // Preload all sibling dylibs with RTLD_LAZY|RTLD_GLOBAL
+  arrfor(results, function(r) {
+    var handle = null
+    if (r.dylib && !open_dls[r.dylib]) {
+      handle = os.dylib_preload(r.dylib)
+      if (handle) open_dls[r.dylib] = handle
+    }
+  })
+
+  return results
 }

-// Try to resolve a C symbol from the deterministic dylib path
+// Try to resolve a C symbol by building the package on demand
 // Returns a loader function or null
 function try_dylib_symbol(sym, pkg, file_stem) {
-  var dylib_path = get_dylib_path(pkg, file_stem)
-  var handle = open_module_dylib(dylib_path)
+  var dylibs = ensure_package_dylibs(pkg)
+  if (!dylibs || length(dylibs) == 0) return null
+
+  var c_file = file_stem + '.c'
+  var cpp_file = file_stem + '.cpp'
+  var entry = find(dylibs, function(r) {
+    return r.file == c_file || r.file == cpp_file
+  })
+  if (!entry || !entry.dylib) return null
+
+  var handle = open_dls[entry.dylib]
+  if (!handle) {
+    handle = os.dylib_open(entry.dylib)
+    if (handle) open_dls[entry.dylib] = handle
+  }
  if (!handle) return null
  if (!os.dylib_has_symbol(handle, sym)) return null
+
+  log.shop('resolved ' + sym + ' from build cache')
  return function() { return os.dylib_symbol(handle, sym) }
 }

 // Resolve a C symbol by searching:
-// At each scope: check lib/ dylib first, then internal (static)
+// At each scope: check build-cache dylib first, then internal (static)
 function resolve_c_symbol(path, package_context) {
  var explicit = split_explicit_package_import(path)
  var sym = null
@@ -934,7 +959,7 @@ function resolve_c_symbol(path, package_context) {
    sym = make_c_symbol(explicit.package, explicit.path)
    file_stem = replace(explicit.path, '.c', '')

-    // Check lib/ dylib first
+    // Check build-cache dylib first
    if (policy.allow_dylib) {
      loader = try_dylib_symbol(sym, explicit.package, file_stem)
      if (loader) {
@@ -962,7 +987,7 @@ function resolve_c_symbol(path, package_context) {
  if (!package_context || package_context == 'core') {
    core_sym = make_c_symbol('core', path)

-    // Check lib/ dylib first for core
+    // Check build-cache dylib first for core
    if (policy.allow_dylib) {
      loader = try_dylib_symbol(core_sym, 'core', path)
      if (loader) {
@@ -984,7 +1009,7 @@ function resolve_c_symbol(path, package_context) {
    return null
  }

-  // 1. Check own package (dylib first, then internal)
+  // 1. Check own package (build-cache dylib first, then internal)
  sym = make_c_symbol(package_context, path)

  if (policy.allow_dylib) {
@@ -1040,7 +1065,7 @@ function resolve_c_symbol(path, package_context) {
    }
  }

-  // 3. Check core (dylib first, then internal)
+  // 3. Check core (build-cache dylib first, then internal)
  core_sym = make_c_symbol('core', path)

  if (policy.allow_dylib) {
@@ -1553,11 +1578,8 @@ Shop.remove = function(pkg) {
    fd.rmdir(pkg_dir, 1)
  }

-  // Remove built dylibs
-  var lib_dir = global_shop_path + '/lib/' + safe_package_path(pkg)
-  if (fd.is_dir(lib_dir)) {
-    fd.rmdir(lib_dir, 1)
-  }
+  // Invalidate package dylib cache
+  package_dylibs[pkg] = null

  log.console("Removed " + pkg)
  return true
@@ -1611,14 +1633,9 @@ Shop.module_reload = function(path, package) {
  var lookup_key = package ? package + ':' + path : ':' + path
  module_info_cache[lookup_key] = null

-  // Close old dylib handle if any
-  var old_dylib_path = null
+  // Invalidate package dylib cache so next resolve triggers rebuild
  if (package) {
-    old_dylib_path = get_dylib_path(package, path)
-    if (open_dls[old_dylib_path]) {
-      os.dylib_close(open_dls[old_dylib_path])
-      open_dls[old_dylib_path] = null
-    }
+    package_dylibs[package] = null
  }

  var info = resolve_module_info(path, package)
@@ -1729,17 +1746,6 @@ Shop.lib_name_for_package = function(pkg) {
  return safe_package_path(pkg)
 }

-// Returns { ok: bool, results: [{pkg, ok, error}] }
-// Get the deterministic dylib path for a module (public API)
-Shop.get_dylib_path = function(pkg, stem) {
-  return get_dylib_path(pkg, stem)
-}
-
-// Get the deterministic mach path for a module (public API)
-Shop.get_mach_path = function(pkg, stem) {
-  return get_mach_path(pkg, stem)
-}
-
 // Load a module explicitly as mach bytecode, bypassing dylib resolution.
 // Returns the loaded module value. Disrupts if the module cannot be found.
 Shop.load_as_mach = function(path, pkg) {
@@ -1754,9 +1760,6 @@ Shop.load_as_mach = function(path, pkg) {
  var ast = null
  var ir = null
  var optimized = null
-  var pkg_dir = null
-  var stem = null
-  var mach_path = null
  var file_info = null
  var inject = null
  var env = null
@@ -1767,27 +1770,13 @@ Shop.load_as_mach = function(path, pkg) {
  content = text(fd.slurp(file_path))
  content_key = stone(blob(content))

-  // Try installed .mach in lib/
-  if (pkg) {
-    pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
-    if (starts_with(file_path, pkg_dir + '/')) {
-      stem = text(file_path, length(pkg_dir) + 1)
-      mach_path = get_mach_path(pkg, stem)
-      if (fd.is_file(mach_path)) {
-        compiled = fd.slurp(mach_path)
-      }
-    }
-  }
-
  // Try cached mach blob
-  if (!compiled) {
-    cached = pull_from_cache(content_key)
-    if (cached) compiled = cached
-  }
+  cached = pull_from_cache(content_key)
+  if (cached) compiled = cached

  // Try cached mcode -> compile to mach
  if (!compiled) {
-    cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
+    cached_mcode_path = hash_path(content_key, 'mcode')
    if (fd.is_file(cached_mcode_path)) {
      mcode_json = text(fd.slurp(cached_mcode_path))
      compiled = mach_compile_mcode_bin(file_path, mcode_json)
@@ -1807,7 +1796,7 @@ Shop.load_as_mach = function(path, pkg) {
    ir = _mcode_mod(ast)
    optimized = _streamline_mod(ir)
    mcode_json = shop_json.encode(optimized)
-    cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
+    cached_mcode_path = hash_path(content_key, 'mcode')
    ensure_dir(global_shop_path + '/build')
    fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
    compiled = mach_compile_mcode_bin(file_path, mcode_json)