john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

centralized ensure dir

Browse Source
This commit is contained in:
John Alanbrook
2026-02-20 15:02:16 -06:00
parent ed69d53573 4ac92c8a87
commit 882a3ae8cb
24 changed files with 341 additions and 949 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
internal/engine.cm
View File

@@ -1466,20 +1466,12 @@ $_.clock(_ => {
if (!fd.is_dir(_dep_dir)) {
log.console('installing missing dependency: ' + _deps[_di])
_auto_install = function() {
shop.update(_deps[_di])
shop.fetch(_deps[_di])
shop.extract(_deps[_di])
shop.build_package_scripts(_deps[_di])
shop.sync(_deps[_di])
} disruption {
log.error('failed to install dependency: ' + _deps[_di])
disrupt
}
_auto_install()
_dep_dir = package.get_dir(_deps[_di])
if (!fd.is_dir(_dep_dir)) {
log.error('missing dependency package: ' + _deps[_di])
disrupt
}
}
_di = _di + 1
}

283
internal/shop.cm
View File

@@ -40,20 +40,6 @@ function put_into_cache(content, obj)
fd.slurpwrite(path, obj)
}
function ensure_dir(path) {
if (fd.stat(path).isDirectory) return
var parts = array(path, '/')
var current = starts_with(path, '/') ? '/' : ''
var i = 0
for (i = 0; i < length(parts); i++) {
if (parts[i] == '') continue
current = current + parts[i] + '/'
if (!fd.stat(current).isDirectory) {
fd.mkdir(current)
}
}
}
function hash_path(content, salt)
{
var s = salt || 'mach'
@@ -88,11 +74,6 @@ Shop.get_core_dir = function() {
return get_packages_dir() + '/' + core_package
}
// Get the links file path (in the global shop)
function get_links_path() {
return global_shop_path + '/link.toml'
}
// Get the reports directory (in the global shop)
Shop.get_reports_dir = function() {
return global_shop_path + '/reports'
@@ -131,7 +112,7 @@ function split_explicit_package_import(path)
mod_path = text(array(parts, i), '/')
if (!mod_path || length(mod_path) == 0) continue
candidate_dir = get_packages_dir() + '/' + safe_package_path(pkg_candidate)
candidate_dir = get_packages_dir() + '/' + fd.safe_package_path(pkg_candidate)
if (fd.is_file(candidate_dir + '/cell.toml'))
return {package: pkg_candidate, path: mod_path}
@@ -259,15 +240,15 @@ function get_canonical_package(alias, package_context) {
return null
}
// return the safe path for the package
// guaranteed to be validated
function safe_package_path(pkg)
{
// For absolute paths, replace / with _ to create a valid directory name
// Also replace @ with _
if (pkg && starts_with(pkg, '/'))
return replace(replace(pkg, '/', '_'), '@', '_')
return replace(pkg, '@', '_')
// Resolve a locator string to its canonical form
// Handles '.', './', '../', and existing directory paths
Shop.resolve_locator = function(locator) {
var resolved = null
if (locator == '.' || starts_with(locator, './') || starts_with(locator, '../') || fd.is_dir(locator)) {
resolved = fd.realpath(locator)
if (resolved) return resolved
}
return locator
}
function package_cache_path(pkg)
@@ -297,7 +278,8 @@ Shop.load_lock = function() {
// Save lock.toml configuration (to global shop)
Shop.save_lock = function(lock) {
var path = global_shop_path + '/lock.toml'
fd.slurpwrite(path, stone(blob(toml.encode(lock))));
fd.slurpwrite(path, stone(blob(toml.encode(lock))))
_lock = lock
}
@@ -447,7 +429,7 @@ function try_native_mod_dylib(pkg, stem) {
var build_mod = use_cache['core/build']
if (!build_mod) return null
var src_path = get_packages_dir() + '/' + safe_package_path(pkg) + '/' + stem
var src_path = get_packages_dir() + '/' + fd.safe_package_path(pkg) + '/' + stem
if (!fd.is_file(src_path)) return null
var src = text(fd.slurp(src_path))
@@ -665,7 +647,7 @@ Shop.all_script_paths = function() {
packages = array(packages, ['core'])
}
for (i = 0; i < length(packages); i++) {
pkg_dir = starts_with(packages[i], '/') ? packages[i] : get_packages_dir() + '/' + safe_package_path(packages[i])
pkg_dir = starts_with(packages[i], '/') ? packages[i] : get_packages_dir() + '/' + fd.safe_package_path(packages[i])
scripts = get_package_scripts(packages[i])
for (j = 0; j < length(scripts); j++) {
result[] = {
@@ -710,7 +692,7 @@ function resolve_mod_fn(path, pkg) {
// Compute _pkg_dir and _stem early so all paths can use them
if (pkg) {
_pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
_pkg_dir = get_packages_dir() + '/' + fd.safe_package_path(pkg)
if (starts_with(path, _pkg_dir + '/')) {
_stem = text(path, length(_pkg_dir) + 1)
}
@@ -772,7 +754,7 @@ function resolve_mod_fn(path, pkg) {
mcode_json = shop_json.encode(optimized)
// Cache mcode (architecture-independent) in content-addressed store
ensure_dir(global_shop_path + '/build')
fd.ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
// Cache mach blob
@@ -813,7 +795,7 @@ function resolve_path(path, ctx)
explicit = null
}
if (explicit) {
explicit_path = get_packages_dir() + '/' + safe_package_path(explicit.package) + '/' + explicit.path
explicit_path = get_packages_dir() + '/' + fd.safe_package_path(explicit.package) + '/' + explicit.path
if (fd.is_file(explicit_path))
return {path: explicit_path, scope: SCOPE_PACKAGE, pkg: explicit.package}
}
@@ -829,7 +811,7 @@ function resolve_path(path, ctx)
if (starts_with(ctx, '/'))
ctx_dir = ctx
else
ctx_dir = get_packages_dir() + '/' + safe_package_path(ctx)
ctx_dir = get_packages_dir() + '/' + fd.safe_package_path(ctx)
ctx_path = ctx_dir + '/' + path
if (fd.is_file(ctx_path)) {
@@ -843,12 +825,12 @@ function resolve_path(path, ctx)
alias = pkg_tools.split_alias(ctx, path)
if (alias) {
alias_path = get_packages_dir() + '/' + safe_package_path(alias.package) + '/' + alias.path
alias_path = get_packages_dir() + '/' + fd.safe_package_path(alias.package) + '/' + alias.path
if (fd.is_file(alias_path))
return {path: alias_path, scope: SCOPE_PACKAGE, pkg: ctx}
}
package_path = get_packages_dir() + '/' + safe_package_path(path)
package_path = get_packages_dir() + '/' + fd.safe_package_path(path)
if (fd.is_file(package_path))
return {path: package_path, scope: SCOPE_PACKAGE, pkg: ctx}
@@ -865,7 +847,7 @@ function resolve_path(path, ctx)
}
})
if (best_pkg && best_remainder) {
shop_dir = get_packages_dir() + '/' + safe_package_path(best_pkg)
shop_dir = get_packages_dir() + '/' + fd.safe_package_path(best_pkg)
shop_file = shop_dir + '/' + best_remainder
if (fd.is_file(shop_file))
return {path: shop_file, scope: SCOPE_PACKAGE, pkg: best_pkg}
@@ -1283,7 +1265,7 @@ function get_module(path, package_context) {
if (!info) {
log.shop(`Module '${path}' could not be found in package '${package_context}'`)
_ctx_dir = package_context ? (starts_with(package_context, '/') ? package_context : get_packages_dir() + '/' + safe_package_path(package_context)) : null
_ctx_dir = package_context ? (starts_with(package_context, '/') ? package_context : get_packages_dir() + '/' + fd.safe_package_path(package_context)) : null
if (_ctx_dir) {
if (fd.is_file(_ctx_dir + '/' + path + '.c') || fd.is_file(_ctx_dir + '/' + path + '.cpp'))
log.shop(`C source exists at ${_ctx_dir}/${path}.c but was not compiled - run 'cell build'`)
@@ -1319,7 +1301,7 @@ Shop.use = function use(path, package_context) {
var _alias2 = null
if (!info) {
log.shop(`Module '${path}' could not be found in package '${package_context}'`)
_ctx_dir2 = package_context ? (starts_with(package_context, '/') ? package_context : get_packages_dir() + '/' + safe_package_path(package_context)) : null
_ctx_dir2 = package_context ? (starts_with(package_context, '/') ? package_context : get_packages_dir() + '/' + fd.safe_package_path(package_context)) : null
if (_ctx_dir2) {
if (fd.is_file(_ctx_dir2 + '/' + path + '.c') || fd.is_file(_ctx_dir2 + '/' + path + '.cpp'))
log.shop(`C source exists at ${_ctx_dir2}/${path}.c but was not compiled - run 'cell build'`)
@@ -1336,8 +1318,6 @@ Shop.use = function use(path, package_context) {
return use_cache[info.cache_key]
}
Shop.resolve_locator = resolve_locator
// Resolve a use() module path to a filesystem path without compiling.
// Returns the absolute path string, or null if not found.
Shop.resolve_use_path = function(path, ctx) {
@@ -1354,14 +1334,13 @@ Shop.resolve_program = function(prog, package_context) {
var info = resolve_path(prog + '.ce', package_context)
if (info) return info
// Auto-install: if the path matches a recognized remote locator, try fetching
// Find best matching package from lock or infer from path
var lock = Shop.load_lock()
var best_pkg = null
var best_remainder = null
var pkg_info = null
var parts = array(prog, '/')
var i = 0
var candidate = null
var pkg_info = null
var _auto = null
arrfor(array(lock), function(pkg_name) {
if (starts_with(prog, pkg_name + '/')) {
@@ -1372,8 +1351,7 @@ Shop.resolve_program = function(prog, package_context) {
}
})
// If not in lock, check if this looks like a fetchable package
// For gitea-style URLs, the package root is host/owner/repo (3 components)
// If not in lock, try gitea-style 3-component package (host/owner/repo)
if (!best_pkg && length(parts) > 3) {
candidate = text(array(parts, 0, 3), '/')
pkg_info = Shop.resolve_package_info(candidate)
@@ -1383,78 +1361,19 @@ Shop.resolve_program = function(prog, package_context) {
}
}
if (best_pkg && best_remainder) {
log.console('fetching ' + best_pkg + '...')
_auto = function() {
// Install the package itself first
Shop.update(best_pkg)
Shop.fetch(best_pkg)
Shop.extract(best_pkg)
// Install dependencies iteratively (each dep must be extracted before reading its deps)
var all_deps = {}
var queue = [best_pkg]
var qi = 0
var current = null
var direct_deps = null
var dep_locator = null
var dep_dir = null
var build_mod = null
var target = null
var _build_c = null
var _read_deps = null
while (qi < length(queue)) {
current = queue[qi]
qi = qi + 1
_read_deps = function() {
direct_deps = pkg_tools.dependencies(current)
} disruption {
direct_deps = null
}
_read_deps()
if (direct_deps) {
arrfor(array(direct_deps), function(alias) {
dep_locator = direct_deps[alias]
if (!all_deps[dep_locator]) {
all_deps[dep_locator] = true
dep_dir = pkg_tools.get_dir(dep_locator)
if (!fd.is_dir(dep_dir)) {
log.console(' installing dependency: ' + dep_locator)
Shop.update(dep_locator)
Shop.fetch(dep_locator)
Shop.extract(dep_locator)
}
push(queue, dep_locator)
}
})
}
}
// Build scripts for all packages
Shop.build_package_scripts(best_pkg)
arrfor(array(all_deps), function(dep) {
Shop.build_package_scripts(dep)
})
// Build C modules
build_mod = use_cache['core/build']
if (build_mod) {
_build_c = function() {
target = build_mod.detect_host_target()
arrfor(array(all_deps), function(dep) {
build_mod.build_dynamic(dep, target, 'release')
})
build_mod.build_dynamic(best_pkg, target, 'release')
} disruption {}
_build_c()
}
} disruption {
return null
}
_auto()
// Retry resolution
info = resolve_path(prog + '.ce', package_context)
if (info) return info
}
if (!best_pkg || !best_remainder) return null
return null
// Auto-install the package and all its dependencies
log.console('fetching ' + best_pkg + '...')
_auto = function() {
Shop.sync_with_deps(best_pkg)
} disruption {
return null
}
_auto()
info = resolve_path(prog + '.ce', package_context)
return info
}
// Resolve a use() module path to {resolved_path, package, type} without compiling.
@@ -1486,7 +1405,7 @@ function get_cache_path(pkg, commit) {
function get_package_abs_dir(package)
{
return get_packages_dir() + '/' + safe_package_path(package)
return get_packages_dir() + '/' + fd.safe_package_path(package)
}
// Fetch the latest commit hash from remote for a package
@@ -1510,7 +1429,7 @@ function fetch_remote_hash(pkg) {
// Returns the zip blob or null on failure
function download_zip(pkg, commit_hash) {
var cache_path = get_cache_path(pkg, commit_hash)
ensure_dir(global_shop_path + '/cache')
fd.ensure_dir(global_shop_path + '/cache')
var download_url = Shop.get_download_url(pkg, commit_hash)
if (!download_url) {
@@ -1732,6 +1651,82 @@ Shop.update = function(pkg) {
return new_entry
}
// Sync a package: ensure it's in lock, fetched, extracted, and compiled
// opts.refresh - check remote for updates even if lock entry exists
// opts.no_build - skip C module build step
// opts.target - explicit build target (auto-detected if not provided)
// opts.buildtype - 'release'|'debug'|'minsize' (default 'release')
Shop.sync = function(pkg, opts) {
var lock = Shop.load_lock()
var info = Shop.resolve_package_info(pkg)
var build_mod = null
var target = null
var _build_c = null
// Step 1: Ensure lock entry (update if refresh or not in lock)
if ((opts && opts.refresh) || !lock[pkg])
Shop.update(pkg)
// Step 2: Fetch zip (no-op for local packages)
if (info && info != 'local')
Shop.fetch(pkg)
// Step 3: Extract to packages dir
Shop.extract(pkg)
// Step 4: Compile scripts
Shop.build_package_scripts(pkg)
// Step 5: Build C modules
if (!opts || !opts.no_build) {
build_mod = use_cache['core/build']
if (build_mod) {
target = (opts && opts.target) ? opts.target : build_mod.detect_host_target()
_build_c = function() {
build_mod.build_dynamic(pkg, target, (opts && opts.buildtype) ? opts.buildtype : 'release')
} disruption {
// Not all packages have C code
}
_build_c()
}
}
}
// Sync a package and all its dependencies (BFS)
Shop.sync_with_deps = function(pkg, opts) {
var visited = {}
var queue = [pkg]
var qi = 0
var current = null
var deps = null
var dep_locator = null
var _read_deps = null
while (qi < length(queue)) {
current = queue[qi]
qi = qi + 1
if (visited[current]) continue
visited[current] = true
Shop.sync(current, opts)
_read_deps = function() {
deps = pkg_tools.dependencies(current)
} disruption {
deps = null
}
_read_deps()
if (deps) {
arrfor(array(deps), function(alias) {
dep_locator = deps[alias]
if (!visited[dep_locator])
push(queue, dep_locator)
})
}
}
}
function install_zip(zip_blob, target_dir) {
var zip = miniz.read(zip_blob)
if (!zip) { print("Failed to read zip archive"); disrupt }
@@ -1740,7 +1735,7 @@ function install_zip(zip_blob, target_dir) {
if (fd.is_dir(target_dir)) fd.rmdir(target_dir, 1)
log.shop("Extracting to " + target_dir)
ensure_dir(target_dir)
fd.ensure_dir(target_dir)
var count = zip.count()
var created_dirs = {}
@@ -1763,7 +1758,7 @@ function install_zip(zip_blob, target_dir) {
dir_path = fd.dirname(full_path)
if (!created_dirs[dir_path]) {
ensure_dir(dir_path)
fd.ensure_dir(dir_path)
created_dirs[dir_path] = true
}
file_data = zip.slurp(filename)
@@ -1784,7 +1779,7 @@ Shop.remove = function(pkg) {
}
// Remove package symlink/directory
var pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
var pkg_dir = get_packages_dir() + '/' + fd.safe_package_path(pkg)
if (fd.is_link(pkg_dir)) {
fd.unlink(pkg_dir)
} else if (fd.is_dir(pkg_dir)) {
@@ -1798,34 +1793,6 @@ Shop.remove = function(pkg) {
return true
}
Shop.get = function(pkg) {
var lock = Shop.load_lock()
var info = null
var commit = null
if (!lock[pkg]) {
info = Shop.resolve_package_info(pkg)
if (!info) {
print("Invalid package: " + pkg); disrupt
}
commit = null
if (info != 'local') {
commit = fetch_remote_hash(pkg)
if (!commit) {
print("Could not resolve commit for " + pkg); disrupt
}
}
lock[pkg] = {
type: info,
commit: commit,
updated: time.number()
}
Shop.save_lock(lock)
}
}
// Compile a module
// List all files in a package
@@ -1902,6 +1869,7 @@ Shop.build_package_scripts = function(package)
ok = ok + 1
} disruption {
push(errors, script)
log.console(" compile error: " + package + '/' + script)
}
_try()
})
@@ -1922,7 +1890,8 @@ Shop.get_lib_dir = function() {
return global_shop_path + '/lib'
}
Shop.ensure_dir = ensure_dir
Shop.ensure_dir = fd.ensure_dir
Shop.install_zip = install_zip
Shop.ensure_package_dylibs = ensure_package_dylibs
Shop.get_local_dir = function() {
@@ -1936,7 +1905,7 @@ Shop.get_build_dir = function() {
// Get the absolute path for a package
Shop.get_package_dir = function(pkg) {
return get_packages_dir() + '/' + safe_package_path(pkg)
return get_packages_dir() + '/' + fd.safe_package_path(pkg)
}
// Generate C symbol name for a file within a package
@@ -1957,7 +1926,7 @@ Shop.c_symbol_prefix = function(pkg) {
// Get the library directory name for a package
Shop.lib_name_for_package = function(pkg) {
return safe_package_path(pkg)
return fd.safe_package_path(pkg)
}
// Load a module explicitly as mach bytecode, bypassing dylib resolution.
@@ -2011,7 +1980,7 @@ Shop.load_as_mach = function(path, pkg) {
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
cached_mcode_path = hash_path(content_key, 'mcode')
ensure_dir(global_shop_path + '/build')
fd.ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
put_into_cache(content_key, compiled)
@@ -2048,7 +2017,7 @@ Shop.load_as_dylib = function(path, pkg) {
}
if (!real_pkg) return null
pkg_dir = get_packages_dir() + '/' + safe_package_path(real_pkg)
pkg_dir = get_packages_dir() + '/' + fd.safe_package_path(real_pkg)
if (!starts_with(file_path, pkg_dir + '/')) return null
stem = text(file_path, length(pkg_dir) + 1)
result = try_native_mod_dylib(real_pkg, stem)

54
internal/testlib.cm
View File

@@ -33,26 +33,54 @@ function get_pkg_dir(package_name) {
return shop.get_package_dir(package_name)
}
// Ensure directory exists
function ensure_dir(path) {
if (fd.is_dir(path)) return true
var parts = array(path, '/')
var current = starts_with(path, '/') ? '/' : ''
// Deep comparison of two values (handles arrays and objects)
function values_equal(a, b) {
var i = 0
for (i = 0; i < length(parts); i++) {
if (parts[i] == '') continue
current = current + parts[i] + '/'
if (!fd.is_dir(current)) {
fd.mkdir(current)
var ka = null
var kb = null
if (a == b) return true
if (is_null(a) && is_null(b)) return true
if (is_null(a) || is_null(b)) return false
if (is_array(a) && is_array(b)) {
if (length(a) != length(b)) return false
i = 0
while (i < length(a)) {
if (!values_equal(a[i], b[i])) return false
i = i + 1
}
return true
}
return true
if (is_object(a) && is_object(b)) {
ka = array(a)
kb = array(b)
if (length(ka) != length(kb)) return false
i = 0
while (i < length(ka)) {
if (!values_equal(a[ka[i]], b[ka[i]])) return false
i = i + 1
}
return true
}
return false
}
// Describe a value for error messages
function describe(val) {
if (is_null(val)) return "null"
if (is_text(val)) return `"${val}"`
if (is_number(val)) return text(val)
if (is_logical(val)) return text(val)
if (is_function(val)) return "<function>"
if (is_array(val)) return `[array length=${text(length(val))}]`
if (is_object(val)) return `{record keys=${text(length(array(val)))}}`
return "<unknown>"
}
return {
is_valid_package: is_valid_package,
get_current_package_name: get_current_package_name,
get_pkg_dir: get_pkg_dir,
ensure_dir: ensure_dir
ensure_dir: fd.ensure_dir,
values_equal: values_equal,
describe: describe
}