Merge branch 'master' into quicken_build

build.cm

@@ -236,32 +236,23 @@ function hash_all_deps(cmd_str, deps) {
 // Compilation
 // ============================================================================
 
-// Compile a single C file for a package
+// Build the compile command string and common flags for a C file.
-// Returns the object file path (content-addressed in .cell/build)
+// Returns {cmd_str, src_path, cc, common_flags, pkg_dir} or null if source missing.
-Build.compile_file = function(pkg, file, target, opts) {
+function compile_setup(pkg, file, target, opts) {
   var _opts = opts || {}
   var _buildtype = _opts.buildtype || 'release'
   var pkg_dir = shop.get_package_dir(pkg)
   var src_path = pkg_dir + '/' + file
   var core_dir = null
 
-  if (!fd.is_file(src_path)) {
+  if (!fd.is_file(src_path)) return null
-    log.error('Source file not found: ' + src_path)
-    return null
-  }
 
-  // Use pre-fetched cflags if provided, otherwise fetch them
   var cflags = _opts.cflags || replace_sigils_array(pkg_tools.get_flags(pkg, 'CFLAGS', target), pkg_dir)
   var target_cflags = toolchains[target].c_args || []
   var cc = toolchains[target].c
-
-  // Symbol name for this file
   var sym_name = shop.c_symbol_for_file(pkg, file)
-
-  // Build common flags (shared between dep scan and compilation)
   var common_flags = []
 
-  // Add buildtype-specific flags
   if (_buildtype == 'release') {
     common_flags = array(common_flags, ['-O3', '-DNDEBUG'])
   } else if (_buildtype == 'debug') {
@@ -273,18 +264,15 @@ Build.compile_file = function(pkg, file, target, opts) {
   push(common_flags, '-DCELL_USE_NAME=' + sym_name)
   push(common_flags, '-I"' + pkg_dir + '"')
 
-  // Auto-discover include/ directory
   if (fd.is_dir(pkg_dir + '/include')) {
     push(common_flags, '-I"' + pkg_dir + '/include"')
   }
 
-  // External packages need core's source dir for cell.h, quickjs.h, blob.h
   if (pkg != 'core') {
     core_dir = shop.get_package_dir('core')
     push(common_flags, '-I"' + core_dir + '/source"')
   }
 
-  // Add package CFLAGS (resolve relative -I paths)
   arrfor(cflags, function(flag) {
     var f = flag
     var ipath = null
@@ -297,27 +285,64 @@ Build.compile_file = function(pkg, file, target, opts) {
     push(common_flags, f)
   })
 
-  // Add target CFLAGS
   arrfor(target_cflags, function(flag) {
     push(common_flags, flag)
   })
 
-  // Build full compilation command
   var cmd_parts = [cc, '-c', '-fPIC']
   cmd_parts = array(cmd_parts, common_flags)
   push(cmd_parts, '"' + src_path + '"')
 
-  var cmd_str = text(cmd_parts, ' ')
+  return {
+    cmd_str: text(cmd_parts, ' '),
+    src_path: src_path,
+    cc: cc,
+    common_flags: common_flags,
+    pkg_dir: pkg_dir
+  }
+}
+
+// Probe for the full content key (source + all deps + compile flags).
+// Returns {full_content, deps, fail} or null if deps not cached yet (cold path).
+function probe_source_key(setup, file) {
+  var file_content = memo_read(setup.src_path)
+  var quick_content = setup.cmd_str + '\n' + file_content
+  var fail_path = cache_path(quick_content, SALT_FAIL)
+  var deps_path = cache_path(quick_content, SALT_DEPS)
+  var deps = null
+  var full_content = null
+
+  if (fd.is_file(fail_path)) return {fail: true}
+
+  if (fd.is_file(deps_path)) {
+    deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) {
+      return length(p) > 0
+    })
+    full_content = hash_all_deps(setup.cmd_str, deps)
+    return {full_content: full_content, deps: deps}
+  }
+
+  return null
+}
+
+// Compile a single C file for a package
+// Returns the object file path (content-addressed in .cell/build)
+Build.compile_file = function(pkg, file, target, opts) {
+  var _opts = opts || {}
+  var setup = compile_setup(pkg, file, target, _opts)
+  if (!setup) {
+    log.error('Source file not found: ' + shop.get_package_dir(pkg) + '/' + file)
+    return null
+  }
 
   if (_opts.verbose) {
-    log.build('[verbose] CFLAGS: ' + text(cflags, ' '))
+    log.build('[verbose] compile: ' + setup.cmd_str)
-    log.build('[verbose] compile: ' + cmd_str)
   }
 
   // Layer 2: stat-based manifest probe (zero file reads on warm cache)
   var mf_obj = null
   if (!_opts.force) {
-    mf_obj = bmfst_probe(cmd_str, src_path)
+    mf_obj = bmfst_probe(setup.cmd_str, setup.src_path)
     if (mf_obj) {
       if (_opts.verbose) log.build('[verbose] manifest hit: ' + file)
       log.shop('manifest hit ' + file)
@@ -325,67 +350,71 @@ Build.compile_file = function(pkg, file, target, opts) {
     }
   }
 
-  // Two-level cache: quick hash for deps file, full hash for object
-  var file_content = memo_read(src_path)
-  var quick_content = cmd_str + '\n' + file_content
-  var deps_path = cache_path(quick_content, SALT_DEPS)
-  var fail_path = cache_path(quick_content, SALT_FAIL)
-
   var build_dir = get_build_dir()
   fd.ensure_dir(build_dir)
 
-  // Check for cached failure (skip files that previously failed to compile)
+  var probe = probe_source_key(setup, file)
-  if (fd.is_file(fail_path)) {
+
+  // Check for cached failure
+  if (probe && probe.fail) {
     if (_opts.verbose) log.build('[verbose] skipping ' + file + ' (cached failure)')
     log.shop('skip ' + file + ' (cached failure)')
     return null
   }
 
-  var deps = null
   var full_content = null
+  var deps = null
   var obj_path = null
 
-  // Warm path: read cached dep list, verify by hashing all deps
+  // Warm path: deps cached, check object
-  if (fd.is_file(deps_path)) {
+  if (probe && probe.full_content) {
-    deps = filter(array(text(fd.slurp(deps_path)), '\n'), function(p) {
+    full_content = probe.full_content
-      return length(p) > 0
-    })
-    full_content = hash_all_deps(cmd_str, deps)
     obj_path = cache_path(full_content, SALT_OBJ)
     if (fd.is_file(obj_path)) {
       if (_opts.verbose) log.build('[verbose] cache hit: ' + file)
       log.shop('cache hit ' + file)
-      bmfst_save(cmd_str, src_path, deps, obj_path)
+      bmfst_save(setup.cmd_str, setup.src_path, probe.deps, obj_path)
       return obj_path
     }
     log.shop('cache stale ' + file + ' (header changed)')
+    deps = probe.deps
   }
 
+  var file_content = null
+  var quick_content = null
+  var err_path = null
+  var full_cmd = null
+  var err_text = null
+  var missing = null
+  var err_lines = null
+  var first_err = null
+  var ret = null
+
   // Cold path: run cc -MM to discover deps
+  if (!deps) {
     log.shop('dep scan ' + file)
-  deps = get_c_deps(cc, common_flags, src_path)
+    deps = get_c_deps(setup.cc, setup.common_flags, setup.src_path)
-  full_content = hash_all_deps(cmd_str, deps)
+    full_content = hash_all_deps(setup.cmd_str, deps)
     obj_path = cache_path(full_content, SALT_OBJ)
 
     // Check if object exists (might exist from previous build with same deps)
     if (fd.is_file(obj_path)) {
-    fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
+      file_content = memo_read(setup.src_path)
+      quick_content = setup.cmd_str + '\n' + file_content
+      fd.slurpwrite(cache_path(quick_content, SALT_DEPS), stone(blob(text(deps, '\n'))))
       if (_opts.verbose) log.build('[verbose] cache hit: ' + file + ' (after dep scan)')
       log.shop('cache hit ' + file + ' (after dep scan)')
-    bmfst_save(cmd_str, src_path, deps, obj_path)
+      bmfst_save(setup.cmd_str, setup.src_path, deps, obj_path)
       return obj_path
     }
+  }
 
   // Compile
   log.shop('compiling ' + file)
   log.console('Compiling ' + file)
-  var err_path = '/tmp/cell_build_err_' + content_hash(src_path) + '.log'
+  err_path = '/tmp/cell_build_err_' + content_hash(setup.src_path) + '.log'
-  var full_cmd = cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"'
+  full_cmd = setup.cmd_str + ' -o "' + obj_path + '" 2>"' + err_path + '"'
-  var err_text = null
+  ret = os.system(full_cmd)
-  var missing = null
-  var err_lines = null
-  var first_err = null
-  var ret = os.system(full_cmd)
   if (ret != 0) {
     if (fd.is_file(err_path)) {
       err_text = text(fd.slurp(err_path))
@@ -403,14 +432,17 @@ Build.compile_file = function(pkg, file, target, opts) {
       if (err_text) log.error(err_text)
       else log.error('Command: ' + full_cmd)
     }
-    // Cache the failure so we don't retry on every build
+    file_content = memo_read(setup.src_path)
-    fd.slurpwrite(fail_path, stone(blob(err_text || 'compilation failed')))
+    quick_content = setup.cmd_str + '\n' + file_content
+    fd.slurpwrite(cache_path(quick_content, SALT_FAIL), stone(blob(err_text || 'compilation failed')))
     return null
   }
 
   // Save deps for future warm-path lookups
-  fd.slurpwrite(deps_path, stone(blob(text(deps, '\n'))))
+  file_content = memo_read(setup.src_path)
-  bmfst_save(cmd_str, src_path, deps, obj_path)
+  quick_content = setup.cmd_str + '\n' + file_content
+  fd.slurpwrite(cache_path(quick_content, SALT_DEPS), stone(blob(text(deps, '\n'))))
+  bmfst_save(setup.cmd_str, setup.src_path, deps, obj_path)
   return obj_path
 }
 
@@ -438,46 +470,42 @@ Build.build_package = function(pkg, target, exclude_main, buildtype) {
 // Dynamic library building
 // ============================================================================
 
-// Compute link content string from all inputs that affect the dylib output
+// Compute dylib content key from source content key + link info
-function compute_link_content(objects, ldflags, target_ldflags, opts) {
+// link_opts: {extra_objects, ldflags, target_ldflags, target, cc}
-  // Sort objects for deterministic hash
+function compute_dylib_content(full_content, link_opts) {
-  var sorted_objects = sort(objects)
+  var parts = [full_content]
-
+  push(parts, 'target:' + text(link_opts.target))
-  // Build a string representing all link inputs
+  push(parts, 'cc:' + text(link_opts.cc))
-  var parts = []
+  arrfor(link_opts.extra_objects, function(obj) {
-  push(parts, 'target:' + text(opts.target))
+    if (obj != null) push(parts, 'extra:' + text(obj))
-  push(parts, 'cc:' + text(opts.cc))
-  arrfor(sorted_objects, function(obj) {
-    // Object paths are content-addressed, so the path itself is the hash
-    push(parts, 'obj:' + text(obj))
   })
-  arrfor(ldflags, function(flag) {
+  arrfor(link_opts.ldflags, function(flag) {
     push(parts, 'ldflag:' + text(flag))
   })
-  arrfor(target_ldflags, function(flag) {
+  arrfor(link_opts.target_ldflags, function(flag) {
     push(parts, 'target_ldflag:' + text(flag))
   })
-
   return text(parts, '\n')
 }
 
 // Build a per-module dynamic library for a single C file
-// Returns the content-addressed dylib path in .cell/build/<hash>.<target>.dylib
+// Returns the content-addressed dylib path in .cell/build/<hash>
+// Checks dylib cache first; only compiles the object if the dylib is stale.
 Build.build_module_dylib = function(pkg, file, target, opts) {
   var _opts = opts || {}
   var _target = target || Build.detect_host_target()
   var _buildtype = _opts.buildtype || 'release'
   var _extra = _opts.extra_objects || []
-  var obj = Build.compile_file(pkg, file, _target, {buildtype: _buildtype, cflags: _opts.cflags, force: _opts.force})
+
-  if (!obj) return null
+  var setup = compile_setup(pkg, file, _target, {buildtype: _buildtype, cflags: _opts.cflags})
+  if (!setup) return null
 
   var tc = toolchains[_target]
   var cc = tc.cpp || tc.c
   var local_dir = get_local_dir()
-  var pkg_dir = shop.get_package_dir(pkg)
 
   // Get link flags
-  var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target), pkg_dir)
+  var ldflags = replace_sigils_array(pkg_tools.get_flags(pkg, 'LDFLAGS', _target), setup.pkg_dir)
   var target_ldflags = tc.c_link_args || []
   var resolved_ldflags = []
   arrfor(ldflags, function(flag) {
@@ -485,29 +513,84 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
     var lpath = null
     if (starts_with(f, '-L') && !starts_with(f, '-L/')) {
       lpath = text(f, 2)
-      if (!starts_with(lpath, pkg_dir)) {
+      if (!starts_with(lpath, setup.pkg_dir)) {
-        f = '-L"' + pkg_dir + '/' + lpath + '"'
+        f = '-L"' + setup.pkg_dir + '/' + lpath + '"'
       }
     }
     push(resolved_ldflags, f)
   })
 
-  // Content-addressed output: hash of (all objects + link flags + target)
-  var all_objects = [obj]
-  all_objects = array(all_objects, _extra)
-  var link_content = compute_link_content(all_objects, resolved_ldflags, target_ldflags, {target: _target, cc: cc})
   var build_dir = get_build_dir()
   fd.ensure_dir(build_dir)
-  var dylib_path = cache_path(link_content, SALT_DYLIB)
+
+  var link_info = {extra_objects: _extra, ldflags: resolved_ldflags, target_ldflags: target_ldflags, target: _target, cc: cc}
+
+  // Probe source key — check dylib cache before compiling
+  var probe = probe_source_key(setup, file)
+  var dylib_path = null
+  var dylib_content = null
+  var obj = null
+  var obj_path = null
   var cmd_parts = null
   var cmd_str = null
   var ret = null
+  var post_probe = null
+  var fallback_probe = null
+
+  if (probe && probe.fail) {
+    log.shop('skip ' + file + ' (cached failure)')
+    return null
+  }
+
+  if (probe && probe.full_content) {
+    dylib_content = compute_dylib_content(probe.full_content, link_info)
+    dylib_path = cache_path(dylib_content, SALT_DYLIB)
+    if (!_opts.force && fd.is_file(dylib_path)) {
+      log.shop('cache hit ' + file)
+      return dylib_path
+    }
+
+    // Dylib stale but object might be cached — check before compiling
+    obj_path = cache_path(probe.full_content, SALT_OBJ)
+    if (fd.is_file(obj_path)) {
+      obj = obj_path
+    }
+  }
+
+  // Object not cached — compile it
+  if (!obj) {
+    obj = Build.compile_file(pkg, file, _target, {buildtype: _buildtype, cflags: _opts.cflags, force: _opts.force})
+    if (!obj) return null
+
+    // Recompute dylib key with the now-known source key
+    if (!dylib_path) {
+      post_probe = probe_source_key(setup, file)
+      if (post_probe && post_probe.full_content) {
+        dylib_content = compute_dylib_content(post_probe.full_content, link_info)
+        dylib_path = cache_path(dylib_content, SALT_DYLIB)
+        if (fd.is_file(dylib_path)) return dylib_path
+      }
+    }
+  }
+
+  // Need dylib_path for output
+  if (!dylib_path) {
+    // Fallback: probe should succeed now since compile_file cached deps
+    fallback_probe = probe_source_key(setup, file)
+    if (fallback_probe && fallback_probe.full_content) {
+      dylib_content = compute_dylib_content(fallback_probe.full_content, link_info)
+      dylib_path = cache_path(dylib_content, SALT_DYLIB)
+    } else {
+      log.error('Cannot compute dylib key for ' + file)
+      return null
+    }
+  }
 
   if (_opts.verbose) {
     log.build('[verbose] LDFLAGS: ' + text(resolved_ldflags, ' '))
   }
 
-  if (!fd.is_file(dylib_path)) {
+  // Link
   cmd_parts = [cc, '-shared', '-fPIC']
 
   if (tc.system == 'darwin') {
@@ -547,9 +630,6 @@ Build.build_module_dylib = function(pkg, file, target, opts) {
     log.error('Linking failed: ' + file)
     return null
   }
-  } else {
-    log.shop('link cache hit ' + file)
-  }
 
   return dylib_path
 }

link.cm

@@ -86,7 +86,7 @@ Link.add = function(canonical, target, shop) {
     }
   } else {
     // Remote package target - ensure it's installed
-    shop.get(target)
+    shop.sync(target)
   }
 
   var links = Link.load()
@@ -115,8 +115,7 @@ Link.add = function(canonical, target, shop) {
           }
           // Install the dependency if not already in shop
           var _get_dep = function() {
-            shop.get(dep_locator)
+            shop.sync(dep_locator)
-            shop.extract(dep_locator)
           } disruption {
             log.build(`  Warning: Could not install dependency ${dep_locator}`)
           }
@@ -233,8 +232,7 @@ Link.sync_all = function(shop) {
             }
             // Install the dependency if not already in shop
             var _get = function() {
-              shop.get(dep_locator)
+              shop.sync(dep_locator)
-              shop.extract(dep_locator)
             } disruption {
               // Silently continue - dependency may already be installed
             }