john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge branch 'improve_fetch' into audit_dups

Browse Source
This commit is contained in:
John Alanbrook
2026-02-20 14:43:26 -06:00
parent e4588e43f2 601a78b3c7
commit ed69d53573
8 changed files with 283 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
add.ce
View File

@@ -78,29 +78,6 @@ if (!fd.is_file(cwd + '/cell.toml')) {
$stop()
}
// Recursively find all cell packages in a directory
function find_packages(dir) {
var found = []
var list = fd.readdir(dir)
if (!list) return found
if (fd.is_file(dir + '/cell.toml')) {
push(found, dir)
}
arrfor(list, function(item) {
if (item == '.' || item == '..' || item == '.cell' || item == '.git') return
var full = dir + '/' + item
var st = fd.stat(full)
var sub = null
if (st && st.isDirectory) {
sub = find_packages(full)
arrfor(sub, function(p) {
push(found, p)
})
}
})
return found
}
// If -r flag, find all packages recursively and add each
if (recursive) {
if (!locator) {
@@ -111,7 +88,9 @@ if (recursive) {
log.error(`${locator} is not a directory`)
$stop()
}
locators = find_packages(resolved)
locators = filter(pkg.find_packages(resolved), function(p) {
return p != cwd
})
if (length(locators) == 0) {
log.console("No packages found in " + resolved)
$stop()

20
fetch.ce
View File

@@ -33,12 +33,26 @@ for (i = 0; i < length(args); i++) {
var all_packages = shop.list_packages()
var lock = shop.load_lock()
var packages_to_fetch = []
var _update = null
if (target_pkg) {
// Fetch specific package
// Fetch specific package - auto-update if not in lock
if (find(all_packages, target_pkg) == null) {
log.error("Package not found: " + target_pkg)
$stop()
log.console("Package not in lock, updating: " + target_pkg)
_update = function() {
shop.update(target_pkg)
} disruption {
log.error("Could not update package: " + target_pkg)
$stop()
}
_update()
// Reload after update
all_packages = shop.list_packages()
lock = shop.load_lock()
if (find(all_packages, target_pkg) == null) {
log.error("Package not found: " + target_pkg)
$stop()
}
}
push(packages_to_fetch, target_pkg)
} else {

37
install.ce
View File

@@ -79,28 +79,7 @@ if (locator && (locator == '.' || starts_with(locator, './') || starts_with(loca
}
}
// Recursively find all cell packages in a directory
function find_packages(dir) {
var found = []
var list = fd.readdir(dir)
if (!list) return found
if (fd.is_file(dir + '/cell.toml')) {
push(found, dir)
}
arrfor(list, function(item) {
if (item == '.' || item == '..' || item == '.cell' || item == '.git') return
var full = dir + '/' + item
var st = fd.stat(full)
var sub = null
if (st && st.isDirectory) {
sub = find_packages(full)
arrfor(sub, function(p) {
push(found, p)
})
}
})
return found
}
var cwd = fd.realpath('.')
// If -r flag, find all packages recursively and install each
if (recursive) {
@@ -112,7 +91,9 @@ if (recursive) {
log.error(`${locator} is not a directory`)
$stop()
}
locators = find_packages(resolved)
locators = filter(pkg.find_packages(resolved), function(p) {
return p != cwd
})
if (length(locators) == 0) {
log.console("No packages found in " + resolved)
$stop()
@@ -133,6 +114,16 @@ var visited = {}
// Recursive mode: install all found packages and exit
if (recursive) {
if (dry_run) {
log.console("Would install:")
arrfor(locators, function(loc) {
var lock = shop.load_lock()
var exists = lock[loc] != null
log.console(" " + loc + (exists ? " (already installed)" : ""))
})
$stop()
}
arrfor(locators, function(loc) {
log.console(" Installing " + loc + "...")
var _inst = function() {

86
internal/engine.cm
View File

@@ -1402,27 +1402,38 @@ if (ends_with(prog, '.ce')) prog = text(prog, 0, -3)
var package = use_core('package')
// Find the .ce file
var prog_path = prog + ".ce"
var pkg_dir = null
var core_dir = null
if (!fd.is_file(prog_path)) {
pkg_dir = package.find_package_dir(".")
if (pkg_dir)
prog_path = pkg_dir + '/' + prog + '.ce'
}
if (!fd.is_file(prog_path)) {
// Check core packages
core_dir = core_path
prog_path = core_dir + '/' + prog + '.ce'
}
if (!fd.is_file(prog_path)) {
os.print(`Main program ${prog} could not be found\n`)
os.exit(1)
// Find the .ce file using unified resolver
var cwd_package = package.find_package_dir(".")
var prog_info = shop.resolve_program ? shop.resolve_program(prog, cwd_package) : null
var prog_path = null
if (prog_info) {
prog_path = prog_info.path
} else {
// Fallback: check CWD, package dir, and core
prog_path = prog + ".ce"
if (!fd.is_file(prog_path) && cwd_package)
prog_path = cwd_package + '/' + prog + '.ce'
if (!fd.is_file(prog_path))
prog_path = core_path + '/' + prog + '.ce'
if (!fd.is_file(prog_path)) {
os.print(`Main program ${prog} could not be found\n`)
os.exit(1)
}
}
$_.clock(_ => {
var file_info = shop.file_info ? shop.file_info(prog_path) : null
var _file_info_ok = false
var file_info = null
var _try_fi = function() {
file_info = shop.file_info ? shop.file_info(prog_path) : null
_file_info_ok = true
} disruption {}
_try_fi()
if (!_file_info_ok || !file_info)
file_info = {path: prog_path, is_module: false, is_actor: true, package: null, name: prog}
// If the unified resolver found the package, use that as the authoritative source
if (prog_info && prog_info.pkg)
file_info.package = prog_info.pkg
var inject = shop.script_inject_for ? shop.script_inject_for(file_info) : []
// Build env with runtime functions + capability injections
@@ -1442,18 +1453,33 @@ $_.clock(_ => {
var pkg = file_info ? file_info.package : null
// Verify all transitive dependency packages are present
// Verify all transitive dependency packages are present, auto-install if missing
var _deps = null
var _di = 0
var _dep_dir = null
var _auto_install = null
if (pkg) {
_deps = package.gather_dependencies(pkg)
_di = 0
while (_di < length(_deps)) {
_dep_dir = package.get_dir(_deps[_di])
if (!fd.is_dir(_dep_dir)) {
log.error('missing dependency package: ' + _deps[_di])
disrupt
log.console('installing missing dependency: ' + _deps[_di])
_auto_install = function() {
shop.update(_deps[_di])
shop.fetch(_deps[_di])
shop.extract(_deps[_di])
shop.build_package_scripts(_deps[_di])
} disruption {
log.error('failed to install dependency: ' + _deps[_di])
disrupt
}
_auto_install()
_dep_dir = package.get_dir(_deps[_di])
if (!fd.is_dir(_dep_dir)) {
log.error('missing dependency package: ' + _deps[_di])
disrupt
}
}
_di = _di + 1
}
@@ -1461,10 +1487,22 @@ $_.clock(_ => {
env.use = function(path) {
var ck = 'core/' + path
var _use_core_result = null
var _use_core_ok = false
if (use_cache[ck]) return use_cache[ck]
var core_mod = use_core(path)
if (core_mod) return core_mod
return shop.use(path, pkg)
var _try_core = function() {
_use_core_result = use_core(path)
_use_core_ok = true
} disruption {}
_try_core()
if (_use_core_ok && _use_core_result) return _use_core_result
var _shop_use = function() {
return shop.use(path, pkg)
} disruption {
log.error(`use('${path}') failed (package: ${pkg})`)
disrupt
}
return _shop_use()
}
env.args = _cell.args.arg
env.log = log

6
internal/os.c
View File

@@ -318,7 +318,9 @@ JSC_SCALL(os_system,
)
JSC_CCALL(os_exit,
exit(0);
int code = 0;
if (argc > 0) JS_ToInt32(js, &code, argv[0]);
exit(code);
)
static JSValue js_os_dylib_open(JSContext *js, JSValue self, int argc, JSValue *argv)
@@ -714,7 +716,7 @@ static const JSCFunctionListEntry js_os_funcs[] = {
MIST_FUNC_DEF(os, rusage, 0),
MIST_FUNC_DEF(os, mallinfo, 0),
MIST_FUNC_DEF(os, system, 1),
MIST_FUNC_DEF(os, exit, 0),
MIST_FUNC_DEF(os, exit, 1),
MIST_FUNC_DEF(os, sleep, 1),
MIST_FUNC_DEF(os, dylib_open, 1),
MIST_FUNC_DEF(os, dylib_preload, 1),

166
internal/shop.cm
View File

@@ -137,9 +137,6 @@ function split_explicit_package_import(path)
if (package_in_shop(pkg_candidate))
return {package: pkg_candidate, path: mod_path}
if (Shop.resolve_package_info(pkg_candidate))
return {package: pkg_candidate, path: mod_path}
}
return null
@@ -158,6 +155,8 @@ function abs_path_to_package(package_dir)
}
var packages_prefix = get_packages_dir() + '/'
var packages_prefix_abs = fd.realpath(get_packages_dir())
if (packages_prefix_abs) packages_prefix_abs = packages_prefix_abs + '/'
var core_dir = packages_prefix + core_package
// Check if this is the core package directory (or its symlink target)
@@ -176,6 +175,10 @@ function abs_path_to_package(package_dir)
if (starts_with(package_dir, packages_prefix))
return text(package_dir, length(packages_prefix))
// Also try absolute path comparison (package_dir may be absolute, packages_prefix relative)
if (packages_prefix_abs && starts_with(package_dir, packages_prefix_abs))
return text(package_dir, length(packages_prefix_abs))
// Check if this local path is the target of a link
// If so, return the canonical package name (link origin) instead
var link_origin = link.get_origin(package_dir)
@@ -344,9 +347,11 @@ function get_policy() {
// Get information about how to resolve a package
// Local packages always start with /
// Remote packages must be exactly host/owner/repo (3 components)
Shop.resolve_package_info = function(pkg) {
if (starts_with(pkg, '/')) return 'local'
if (search(pkg, 'gitea') != null) return 'gitea'
var parts = array(pkg, '/')
if (length(parts) == 3 && search(parts[0], 'gitea') != null) return 'gitea'
return null
}
@@ -797,6 +802,11 @@ function resolve_path(path, ctx)
var ctx_path = null
var alias = null
var package_path = null
var lock = null
var best_pkg = null
var best_remainder = null
var shop_dir = null
var shop_file = null
if (explicit) {
if (is_internal_path(explicit.path) && ctx && explicit.package != ctx)
@@ -842,6 +852,25 @@ function resolve_path(path, ctx)
if (fd.is_file(package_path))
return {path: package_path, scope: SCOPE_PACKAGE, pkg: ctx}
// Shop package scanning: longest prefix match against lock.toml entries
lock = Shop.load_lock()
best_pkg = null
best_remainder = null
arrfor(array(lock), function(pkg_name) {
if (starts_with(path, pkg_name + '/')) {
if (!best_pkg || length(pkg_name) > length(best_pkg)) {
best_pkg = pkg_name
best_remainder = text(path, length(pkg_name) + 1)
}
}
})
if (best_pkg && best_remainder) {
shop_dir = get_packages_dir() + '/' + safe_package_path(best_pkg)
shop_file = shop_dir + '/' + best_remainder
if (fd.is_file(shop_file))
return {path: shop_file, scope: SCOPE_PACKAGE, pkg: best_pkg}
}
core_dir = Shop.get_core_dir()
core_file_path = core_dir + '/' + path
if (fd.is_file(core_file_path))
@@ -1317,6 +1346,117 @@ Shop.resolve_use_path = function(path, ctx) {
return info.path
}
// Resolve a program (.ce) path using the unified resolver.
// Returns {path, scope, pkg} or null.
// If the path looks like a remote package locator and is not found locally,
// attempts to auto-fetch and install it.
Shop.resolve_program = function(prog, package_context) {
var info = resolve_path(prog + '.ce', package_context)
if (info) return info
// Auto-install: if the path matches a recognized remote locator, try fetching
var lock = Shop.load_lock()
var best_pkg = null
var best_remainder = null
var pkg_info = null
var parts = array(prog, '/')
var i = 0
var candidate = null
var _auto = null
arrfor(array(lock), function(pkg_name) {
if (starts_with(prog, pkg_name + '/')) {
if (!best_pkg || length(pkg_name) > length(best_pkg)) {
best_pkg = pkg_name
best_remainder = text(prog, length(pkg_name) + 1)
}
}
})
// If not in lock, check if this looks like a fetchable package
// For gitea-style URLs, the package root is host/owner/repo (3 components)
if (!best_pkg && length(parts) > 3) {
candidate = text(array(parts, 0, 3), '/')
pkg_info = Shop.resolve_package_info(candidate)
if (pkg_info && pkg_info != 'local') {
best_pkg = candidate
best_remainder = text(array(parts, 3), '/')
}
}
if (best_pkg && best_remainder) {
log.console('fetching ' + best_pkg + '...')
_auto = function() {
// Install the package itself first
Shop.update(best_pkg)
Shop.fetch(best_pkg)
Shop.extract(best_pkg)
// Install dependencies iteratively (each dep must be extracted before reading its deps)
var all_deps = {}
var queue = [best_pkg]
var qi = 0
var current = null
var direct_deps = null
var dep_locator = null
var dep_dir = null
var build_mod = null
var target = null
var _build_c = null
var _read_deps = null
while (qi < length(queue)) {
current = queue[qi]
qi = qi + 1
_read_deps = function() {
direct_deps = pkg_tools.dependencies(current)
} disruption {
direct_deps = null
}
_read_deps()
if (direct_deps) {
arrfor(array(direct_deps), function(alias) {
dep_locator = direct_deps[alias]
if (!all_deps[dep_locator]) {
all_deps[dep_locator] = true
dep_dir = pkg_tools.get_dir(dep_locator)
if (!fd.is_dir(dep_dir)) {
log.console(' installing dependency: ' + dep_locator)
Shop.update(dep_locator)
Shop.fetch(dep_locator)
Shop.extract(dep_locator)
}
push(queue, dep_locator)
}
})
}
}
// Build scripts for all packages
Shop.build_package_scripts(best_pkg)
arrfor(array(all_deps), function(dep) {
Shop.build_package_scripts(dep)
})
// Build C modules
build_mod = use_cache['core/build']
if (build_mod) {
_build_c = function() {
target = build_mod.detect_host_target()
arrfor(array(all_deps), function(dep) {
build_mod.build_dynamic(dep, target, 'release')
})
build_mod.build_dynamic(best_pkg, target, 'release')
} disruption {}
_build_c()
}
} disruption {
return null
}
_auto()
// Retry resolution
info = resolve_path(prog + '.ce', package_context)
if (info) return info
}
return null
}
// Resolve a use() module path to {resolved_path, package, type} without compiling.
// type is 'script', 'native', or null. Checks .cm files, C symbols, and aliases.
Shop.resolve_import_info = function(path, ctx) {
@@ -1429,7 +1569,7 @@ Shop.fetch = function(pkg) {
if (actual_hash == expected_hash) {
return { status: 'cached' }
}
log.console("Zip hash mismatch for " + pkg + ", re-fetching...")
log.shop("Zip hash mismatch for " + pkg + ", re-fetching...")
} else {
// No hash stored yet - compute and store it
actual_hash = text(crypto.blake2(zip_blob), 'h')
@@ -1537,17 +1677,23 @@ function get_package_zip(pkg)
// Update: Check for new version, update lock, fetch and extract
// Returns the new lock entry if updated, null if already up to date or failed
Shop.update = function(pkg) {
Shop.verify_package_name(pkg)
var lock = Shop.load_lock()
var lock_entry = lock[pkg]
var info = Shop.resolve_package_info(pkg)
log.console(`checking ${pkg}`)
if (!info) {
log.error("Not a valid package locator: " + pkg)
return null
}
log.shop(`checking ${pkg}`)
var new_entry = null
if (info == 'local') {
// Check if local path exists
if (!fd.is_dir(pkg)) {
log.console(` Local path does not exist: ${pkg}`)
log.shop(` Local path does not exist: ${pkg}`)
return null
}
// Local packages always get a lock entry
@@ -1563,8 +1709,8 @@ Shop.update = function(pkg) {
var local_commit = lock_entry ? lock_entry.commit : null
var remote_commit = fetch_remote_hash(pkg)
log.console(`local commit: ${local_commit}`)
log.console(`remote commit: ${remote_commit}`)
log.shop(`local commit: ${local_commit}`)
log.shop(`remote commit: ${remote_commit}`)
if (!remote_commit) {
log.error("Could not resolve commit for " + pkg)
@@ -1593,7 +1739,7 @@ function install_zip(zip_blob, target_dir) {
if (fd.is_link(target_dir)) fd.unlink(target_dir)
if (fd.is_dir(target_dir)) fd.rmdir(target_dir, 1)
log.console("Extracting to " + target_dir)
log.shop("Extracting to " + target_dir)
ensure_dir(target_dir)
var count = zip.count()

22
package.cm
View File

@@ -202,6 +202,28 @@ package.gather_dependencies = function(name)
return array(all_deps)
}
// Recursively find all cell packages (dirs with cell.toml) under a directory
package.find_packages = function(dir) {
var found = []
var list = fd.readdir(dir)
if (!list) return found
if (fd.is_file(dir + '/cell.toml'))
push(found, dir)
arrfor(list, function(item) {
if (item == '.' || item == '..' || item == '.cell' || item == '.git') return
var full = dir + '/' + item
var st = fd.stat(full)
var sub = null
if (st && st.isDirectory) {
sub = package.find_packages(full)
arrfor(sub, function(p) {
push(found, p)
})
}
})
return found
}
package.list_files = function(pkg) {
var dir = get_path(pkg)
if (!fd.is_dir(dir)) return []

5
source/runtime.c
View File

@@ -8055,6 +8055,11 @@ static JSValue js_cell_text_format (JSContext *ctx, JSValue this_val, int argc,
}
}
if (!made_substitution && JS_IsNull (cv_ref.val)) {
substitution = JS_NewString (ctx, "null");
made_substitution = 1;
}
if (!made_substitution && !JS_IsNull (cv_ref.val)) {
JSValue conv_text_val = JS_ToString (ctx, cv_ref.val);
if (JS_IsText (conv_text_val)) {