Merge branch 'improve_fetch' into audit_dups

internal/engine.cm

@@ -1402,27 +1402,38 @@ if (ends_with(prog, '.ce')) prog = text(prog, 0, -3)
 
 var package = use_core('package')
 
-// Find the .ce file
-var prog_path = prog + ".ce"
-var pkg_dir = null
-var core_dir = null
-if (!fd.is_file(prog_path)) {
-  pkg_dir = package.find_package_dir(".")
-  if (pkg_dir)
-    prog_path = pkg_dir + '/' + prog + '.ce'
-}
-if (!fd.is_file(prog_path)) {
-  // Check core packages
-  core_dir = core_path
-  prog_path = core_dir + '/' + prog + '.ce'
-}
-if (!fd.is_file(prog_path)) {
-  os.print(`Main program ${prog} could not be found\n`)
-  os.exit(1)
+// Find the .ce file using unified resolver
+var cwd_package = package.find_package_dir(".")
+var prog_info = shop.resolve_program ? shop.resolve_program(prog, cwd_package) : null
+var prog_path = null
+if (prog_info) {
+  prog_path = prog_info.path
+} else {
+  // Fallback: check CWD, package dir, and core
+  prog_path = prog + ".ce"
+  if (!fd.is_file(prog_path) && cwd_package)
+    prog_path = cwd_package + '/' + prog + '.ce'
+  if (!fd.is_file(prog_path))
+    prog_path = core_path + '/' + prog + '.ce'
+  if (!fd.is_file(prog_path)) {
+    os.print(`Main program ${prog} could not be found\n`)
+    os.exit(1)
+  }
 }
 
 $_.clock(_ => {
-  var file_info = shop.file_info ? shop.file_info(prog_path) : null
+  var _file_info_ok = false
+  var file_info = null
+  var _try_fi = function() {
+    file_info = shop.file_info ? shop.file_info(prog_path) : null
+    _file_info_ok = true
+  } disruption {}
+  _try_fi()
+  if (!_file_info_ok || !file_info)
+    file_info = {path: prog_path, is_module: false, is_actor: true, package: null, name: prog}
+  // If the unified resolver found the package, use that as the authoritative source
+  if (prog_info && prog_info.pkg)
+    file_info.package = prog_info.pkg
   var inject = shop.script_inject_for ? shop.script_inject_for(file_info) : []
 
   // Build env with runtime functions + capability injections
@@ -1442,18 +1453,33 @@ $_.clock(_ => {
 
   var pkg = file_info ? file_info.package : null
 
-  // Verify all transitive dependency packages are present
+  // Verify all transitive dependency packages are present, auto-install if missing
   var _deps = null
   var _di = 0
   var _dep_dir = null
+  var _auto_install = null
   if (pkg) {
     _deps = package.gather_dependencies(pkg)
     _di = 0
     while (_di < length(_deps)) {
       _dep_dir = package.get_dir(_deps[_di])
       if (!fd.is_dir(_dep_dir)) {
-        log.error('missing dependency package: ' + _deps[_di])
-        disrupt
+        log.console('installing missing dependency: ' + _deps[_di])
+        _auto_install = function() {
+          shop.update(_deps[_di])
+          shop.fetch(_deps[_di])
+          shop.extract(_deps[_di])
+          shop.build_package_scripts(_deps[_di])
+        } disruption {
+          log.error('failed to install dependency: ' + _deps[_di])
+          disrupt
+        }
+        _auto_install()
+        _dep_dir = package.get_dir(_deps[_di])
+        if (!fd.is_dir(_dep_dir)) {
+          log.error('missing dependency package: ' + _deps[_di])
+          disrupt
+        }
       }
       _di = _di + 1
     }
@@ -1461,10 +1487,22 @@ $_.clock(_ => {
 
   env.use = function(path) {
     var ck = 'core/' + path
+    var _use_core_result = null
+    var _use_core_ok = false
     if (use_cache[ck]) return use_cache[ck]
-    var core_mod = use_core(path)
-    if (core_mod) return core_mod
-    return shop.use(path, pkg)
+    var _try_core = function() {
+      _use_core_result = use_core(path)
+      _use_core_ok = true
+    } disruption {}
+    _try_core()
+    if (_use_core_ok && _use_core_result) return _use_core_result
+    var _shop_use = function() {
+      return shop.use(path, pkg)
+    } disruption {
+      log.error(`use('${path}') failed (package: ${pkg})`)
+      disrupt
+    }
+    return _shop_use()
   }
   env.args = _cell.args.arg
   env.log  = log

internal/os.c

@@ -318,7 +318,9 @@ JSC_SCALL(os_system,
 )
 
 JSC_CCALL(os_exit,
-  exit(0);
+  int code = 0;
+  if (argc > 0) JS_ToInt32(js, &code, argv[0]);
+  exit(code);
 )
 
 static JSValue js_os_dylib_open(JSContext *js, JSValue self, int argc, JSValue *argv)
@@ -714,7 +716,7 @@ static const JSCFunctionListEntry js_os_funcs[] = {
   MIST_FUNC_DEF(os, rusage, 0),
   MIST_FUNC_DEF(os, mallinfo, 0),
   MIST_FUNC_DEF(os, system, 1),
-  MIST_FUNC_DEF(os, exit, 0),
+  MIST_FUNC_DEF(os, exit, 1),
   MIST_FUNC_DEF(os, sleep, 1),
   MIST_FUNC_DEF(os, dylib_open, 1),
   MIST_FUNC_DEF(os, dylib_preload, 1),

internal/shop.cm

@@ -137,9 +137,6 @@ function split_explicit_package_import(path)
 
     if (package_in_shop(pkg_candidate))
       return {package: pkg_candidate, path: mod_path}
-
-    if (Shop.resolve_package_info(pkg_candidate))
-      return {package: pkg_candidate, path: mod_path}
   }
 
   return null
@@ -158,6 +155,8 @@ function abs_path_to_package(package_dir)
   }
 
   var packages_prefix = get_packages_dir() + '/'
+  var packages_prefix_abs = fd.realpath(get_packages_dir())
+  if (packages_prefix_abs) packages_prefix_abs = packages_prefix_abs + '/'
   var core_dir = packages_prefix + core_package
 
   // Check if this is the core package directory (or its symlink target)
@@ -176,6 +175,10 @@ function abs_path_to_package(package_dir)
   if (starts_with(package_dir, packages_prefix))
     return text(package_dir, length(packages_prefix))
 
+  // Also try absolute path comparison (package_dir may be absolute, packages_prefix relative)
+  if (packages_prefix_abs && starts_with(package_dir, packages_prefix_abs))
+    return text(package_dir, length(packages_prefix_abs))
+
   // Check if this local path is the target of a link
   // If so, return the canonical package name (link origin) instead
   var link_origin = link.get_origin(package_dir)
@@ -344,9 +347,11 @@ function get_policy() {
 
 // Get information about how to resolve a package
 // Local packages always start with /
+// Remote packages must be exactly host/owner/repo (3 components)
 Shop.resolve_package_info = function(pkg) {
   if (starts_with(pkg, '/')) return 'local'
-  if (search(pkg, 'gitea') != null) return 'gitea'
+  var parts = array(pkg, '/')
+  if (length(parts) == 3 && search(parts[0], 'gitea') != null) return 'gitea'
   return null
 }
 
@@ -797,6 +802,11 @@ function resolve_path(path, ctx)
   var ctx_path = null
   var alias = null
   var package_path = null
+  var lock = null
+  var best_pkg = null
+  var best_remainder = null
+  var shop_dir = null
+  var shop_file = null
 
   if (explicit) {
     if (is_internal_path(explicit.path) && ctx && explicit.package != ctx)
@@ -842,6 +852,25 @@ function resolve_path(path, ctx)
   if (fd.is_file(package_path))
     return {path: package_path, scope: SCOPE_PACKAGE, pkg: ctx}
 
+  // Shop package scanning: longest prefix match against lock.toml entries
+  lock = Shop.load_lock()
+  best_pkg = null
+  best_remainder = null
+  arrfor(array(lock), function(pkg_name) {
+    if (starts_with(path, pkg_name + '/')) {
+      if (!best_pkg || length(pkg_name) > length(best_pkg)) {
+        best_pkg = pkg_name
+        best_remainder = text(path, length(pkg_name) + 1)
+      }
+    }
+  })
+  if (best_pkg && best_remainder) {
+    shop_dir = get_packages_dir() + '/' + safe_package_path(best_pkg)
+    shop_file = shop_dir + '/' + best_remainder
+    if (fd.is_file(shop_file))
+      return {path: shop_file, scope: SCOPE_PACKAGE, pkg: best_pkg}
+  }
+
   core_dir = Shop.get_core_dir()
   core_file_path = core_dir + '/' + path
   if (fd.is_file(core_file_path))
@@ -1317,6 +1346,117 @@ Shop.resolve_use_path = function(path, ctx) {
   return info.path
 }
 
+// Resolve a program (.ce) path using the unified resolver.
+// Returns {path, scope, pkg} or null.
+// If the path looks like a remote package locator and is not found locally,
+// attempts to auto-fetch and install it.
+Shop.resolve_program = function(prog, package_context) {
+  var info = resolve_path(prog + '.ce', package_context)
+  if (info) return info
+
+  // Auto-install: if the path matches a recognized remote locator, try fetching
+  var lock = Shop.load_lock()
+  var best_pkg = null
+  var best_remainder = null
+  var pkg_info = null
+  var parts = array(prog, '/')
+  var i = 0
+  var candidate = null
+  var _auto = null
+  arrfor(array(lock), function(pkg_name) {
+    if (starts_with(prog, pkg_name + '/')) {
+      if (!best_pkg || length(pkg_name) > length(best_pkg)) {
+        best_pkg = pkg_name
+        best_remainder = text(prog, length(pkg_name) + 1)
+      }
+    }
+  })
+
+  // If not in lock, check if this looks like a fetchable package
+  // For gitea-style URLs, the package root is host/owner/repo (3 components)
+  if (!best_pkg && length(parts) > 3) {
+    candidate = text(array(parts, 0, 3), '/')
+    pkg_info = Shop.resolve_package_info(candidate)
+    if (pkg_info && pkg_info != 'local') {
+      best_pkg = candidate
+      best_remainder = text(array(parts, 3), '/')
+    }
+  }
+
+  if (best_pkg && best_remainder) {
+    log.console('fetching ' + best_pkg + '...')
+    _auto = function() {
+      // Install the package itself first
+      Shop.update(best_pkg)
+      Shop.fetch(best_pkg)
+      Shop.extract(best_pkg)
+      // Install dependencies iteratively (each dep must be extracted before reading its deps)
+      var all_deps = {}
+      var queue = [best_pkg]
+      var qi = 0
+      var current = null
+      var direct_deps = null
+      var dep_locator = null
+      var dep_dir = null
+      var build_mod = null
+      var target = null
+      var _build_c = null
+      var _read_deps = null
+      while (qi < length(queue)) {
+        current = queue[qi]
+        qi = qi + 1
+        _read_deps = function() {
+          direct_deps = pkg_tools.dependencies(current)
+        } disruption {
+          direct_deps = null
+        }
+        _read_deps()
+        if (direct_deps) {
+          arrfor(array(direct_deps), function(alias) {
+            dep_locator = direct_deps[alias]
+            if (!all_deps[dep_locator]) {
+              all_deps[dep_locator] = true
+              dep_dir = pkg_tools.get_dir(dep_locator)
+              if (!fd.is_dir(dep_dir)) {
+                log.console('  installing dependency: ' + dep_locator)
+                Shop.update(dep_locator)
+                Shop.fetch(dep_locator)
+                Shop.extract(dep_locator)
+              }
+              push(queue, dep_locator)
+            }
+          })
+        }
+      }
+      // Build scripts for all packages
+      Shop.build_package_scripts(best_pkg)
+      arrfor(array(all_deps), function(dep) {
+        Shop.build_package_scripts(dep)
+      })
+      // Build C modules
+      build_mod = use_cache['core/build']
+      if (build_mod) {
+        _build_c = function() {
+          target = build_mod.detect_host_target()
+          arrfor(array(all_deps), function(dep) {
+            build_mod.build_dynamic(dep, target, 'release')
+          })
+          build_mod.build_dynamic(best_pkg, target, 'release')
+        } disruption {}
+        _build_c()
+      }
+    } disruption {
+      return null
+    }
+    _auto()
+    // Retry resolution
+    info = resolve_path(prog + '.ce', package_context)
+    if (info) return info
+  }
+
+  return null
+}
+
 // Resolve a use() module path to {resolved_path, package, type} without compiling.
 // type is 'script', 'native', or null. Checks .cm files, C symbols, and aliases.
 Shop.resolve_import_info = function(path, ctx) {
@@ -1429,7 +1569,7 @@ Shop.fetch = function(pkg) {
       if (actual_hash == expected_hash) {
         return { status: 'cached' }
       }
-      log.console("Zip hash mismatch for " + pkg + ", re-fetching...")
+      log.shop("Zip hash mismatch for " + pkg + ", re-fetching...")
     } else {
       // No hash stored yet - compute and store it
       actual_hash = text(crypto.blake2(zip_blob), 'h')
@@ -1537,17 +1677,23 @@ function get_package_zip(pkg)
 // Update: Check for new version, update lock, fetch and extract
 // Returns the new lock entry if updated, null if already up to date or failed
 Shop.update = function(pkg) {
+  Shop.verify_package_name(pkg)
   var lock = Shop.load_lock()
   var lock_entry = lock[pkg]
   var info = Shop.resolve_package_info(pkg)
 
-  log.console(`checking ${pkg}`)
+  if (!info) {
+    log.error("Not a valid package locator: " + pkg)
+    return null
+  }
+
+  log.shop(`checking ${pkg}`)
 
   var new_entry = null
   if (info == 'local') {
     // Check if local path exists
     if (!fd.is_dir(pkg)) {
-      log.console(`  Local path does not exist: ${pkg}`)
+      log.shop(`  Local path does not exist: ${pkg}`)
       return null
     }
     // Local packages always get a lock entry
@@ -1563,8 +1709,8 @@ Shop.update = function(pkg) {
   var local_commit = lock_entry ? lock_entry.commit : null
   var remote_commit = fetch_remote_hash(pkg)
 
-  log.console(`local commit: ${local_commit}`)
-  log.console(`remote commit: ${remote_commit}`)
+  log.shop(`local commit: ${local_commit}`)
+  log.shop(`remote commit: ${remote_commit}`)
 
   if (!remote_commit) {
     log.error("Could not resolve commit for " + pkg)
@@ -1593,7 +1739,7 @@ function install_zip(zip_blob, target_dir) {
   if (fd.is_link(target_dir)) fd.unlink(target_dir)
   if (fd.is_dir(target_dir)) fd.rmdir(target_dir, 1)
 
-  log.console("Extracting to " + target_dir)
+  log.shop("Extracting to " + target_dir)
   ensure_dir(target_dir)
 
   var count = zip.count()