From 46c345d34e73165e9bf81abc470e4ec81f26c007 Mon Sep 17 00:00:00 2001 From: John Alanbrook Date: Mon, 16 Feb 2026 00:04:30 -0600 Subject: [PATCH] cache invalidation --- build.cm | 4 +- compile.ce | 82 ++---------- docs/shop.md | 60 +++++++-- internal/shop.cm | 332 +++++++++++++++++++++++++++++++++++++---------- 4 files changed, 328 insertions(+), 150 deletions(-) diff --git a/build.cm b/build.cm index af46b504..1164f90c 100644 --- a/build.cm +++ b/build.cm @@ -288,7 +288,7 @@ Build.build_module_dylib = function(pkg, file, target, buildtype) { } // Install to deterministic lib//.dylib - var file_stem = fd.stem(file) + var file_stem = file var install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) var stem_dir = fd.dirname(file_stem) if (stem_dir && stem_dir != '.') { @@ -539,7 +539,7 @@ Build.compile_native = function(src_path, target, buildtype, pkg) { // Install to deterministic lib//.dylib if (pkg) { - native_stem = fd.stem(fd.basename(src_path)) + native_stem = fd.basename(src_path) native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg) ensure_dir(native_install_dir) native_install_path = native_install_dir + '/' + native_stem + dylib_ext diff --git a/compile.ce b/compile.ce index 8fd46ca5..e2aad0ac 100644 --- a/compile.ce +++ b/compile.ce @@ -1,85 +1,27 @@ -// compile.ce — compile a .cm module to native .dylib via QBE +// compile.ce — compile a .cm or .ce file to native .dylib via QBE // // Usage: -// cell --dev compile.ce +// cell compile // -// Produces .dylib in the current directory. +// Installs the dylib to .cell/lib//.dylib +var shop = use('internal/shop') +var build = use('build') var fd = use('fd') -var os = use('os') if (length(args) < 1) { - print('usage: cell --dev compile.ce ') + print('usage: cell compile ') return } var file = args[0] -var base = file -if (ends_with(base, '.cm')) { - base = text(base, 0, length(base) - 3) -} - -var safe = replace(replace(base, '/', '_'), '-', '_') -var symbol = 'js_' + safe + '_use' -var tmp = '/tmp/qbe_' + safe -var ssa_path = tmp + '.ssa' -var s_path = tmp + '.s' -var o_path = tmp + '.o' -var rt_o_path = '/tmp/qbe_rt.o' -var dylib_path = file + '.dylib' -var cwd = fd.getcwd() -var rc = 0 - -// Step 1: emit QBE IL -print('emit qbe...') -rc = os.system('cd ' + cwd + ' && ./cell --dev qbe.ce ' + file + ' > ' + ssa_path) -if (rc != 0) { - print('failed to emit qbe il') +if (!fd.is_file(file)) { + print('file not found: ' + file) return } -// Step 2: append wrapper function — called as symbol(ctx) by os.dylib_symbol. -// Delegates to cell_rt_module_entry which heap-allocates a frame -// (so closures survive) and calls cell_main. -var wrapper_cmd = `printf '\nexport function l $` + symbol + `(l %%ctx) {\n@entry\n %%result =l call $cell_rt_module_entry(l %%ctx)\n ret %%result\n}\n' >> ` + ssa_path -rc = os.system(wrapper_cmd) -if (rc != 0) { - print('wrapper append failed') - return -} +var abs = fd.realpath(file) +var file_info = shop.file_info(abs) +var pkg = file_info.package -// Step 3: compile QBE IL to assembly -print('qbe compile...') -rc = os.system('qbe -o ' + s_path + ' ' + ssa_path) -if (rc != 0) { - print('qbe compilation failed') - return -} - -// Step 4: assemble -print('assemble...') -rc = os.system('cc -c ' + s_path + ' -o ' + o_path) -if (rc != 0) { - print('assembly failed') - return -} - -// Step 5: compile runtime stubs (cached — skip if already built) -if (!fd.is_file(rt_o_path)) { - print('compile runtime stubs...') - rc = os.system('cc -c ' + cwd + '/qbe_rt.c -o ' + rt_o_path + ' -fPIC') - if (rc != 0) { - print('runtime stubs compilation failed') - return - } -} - -// Step 6: link dylib -print('link...') -rc = os.system('cc -shared -fPIC -undefined dynamic_lookup ' + o_path + ' ' + rt_o_path + ' -o ' + cwd + '/' + dylib_path) -if (rc != 0) { - print('linking failed') - return -} - -print('built: ' + dylib_path) +build.compile_native(abs, null, null, pkg) diff --git a/docs/shop.md b/docs/shop.md index e0f49a32..7c20c6f0 100644 --- a/docs/shop.md +++ b/docs/shop.md @@ -63,16 +63,17 @@ When loading a module, the shop checks (in order): 1. **In-memory cache** — `use_cache[key]`, checked first on every `use()` call 2. **Installed dylib** — per-file `.dylib` in `~/.pit/lib//.dylib` -3. **Internal symbols** — statically linked into the `pit` binary (fat builds) -4. **Installed mach** — pre-compiled bytecode in `~/.pit/lib//.mach` -5. **Cached bytecode** — content-addressed in `~/.pit/build/` (no extension) -6. **Cached .mcode IR** — JSON IR in `~/.pit/build/.mcode` -7. **Adjacent .mach/.mcode** — files alongside the source (e.g., `sprite.mach`) -8. **Source compilation** — full pipeline: analyze, mcode, streamline, serialize +3. **Installed mach** — pre-compiled bytecode in `~/.pit/lib//.mach` +4. **Cached bytecode** — content-addressed in `~/.pit/build/` (no extension) +5. **Cached .mcode IR** — JSON IR in `~/.pit/build/.mcode` +6. **Internal symbols** — statically linked into the `pit` binary (fat builds) +7. **Source compilation** — full pipeline: analyze, mcode, streamline, serialize When both a `.dylib` and `.mach` exist for the same module in `lib/`, the dylib is selected. Dylib resolution also wins over internal symbols, so a dylib in `lib/` can hot-patch a fat binary. Delete the dylib to fall back to mach or static. -Results from steps 6-8 are cached back to the content-addressed store for future loads. +Results from steps 5-7 are cached back to the content-addressed store for future loads. + +Each loading method (except the in-memory cache) can be individually enabled or disabled via `shop.toml` policy flags — see [Shop Configuration](#shop-configuration) below. ### Content-Addressed Store @@ -124,6 +125,48 @@ When a module is loaded, the shop builds an `env` object that becomes the module The set of injected capabilities is controlled by `script_inject_for()`, which can be tuned per package or file. +## Shop Configuration + +The shop reads an optional `shop.toml` file from the shop root (`~/.pit/shop.toml`). This file controls which loading methods are permitted through policy flags. + +### Policy Flags + +All flags default to `true`. Set a flag to `false` to disable that loading method. + +```toml +[policy] +allow_dylib = true # per-file .dylib loading (requires dlopen) +allow_static = true # statically linked C symbols (fat builds) +allow_mach = true # pre-compiled .mach bytecode (lib/ and build cache) +allow_compile = true # on-the-fly source compilation +``` + +### Example Configurations + +**Production lockdown** — only use pre-compiled artifacts, never compile from source: + +```toml +[policy] +allow_compile = false +``` + +**Pure-script mode** — bytecode only, no native code: + +```toml +[policy] +allow_dylib = false +allow_static = false +``` + +**No dlopen platforms** — static linking and bytecode only: + +```toml +[policy] +allow_dylib = false +``` + +If `shop.toml` is missing or has no `[policy]` section, all methods are enabled (default behavior). + ## Shop Directory Layout ``` @@ -145,7 +188,8 @@ The set of injected capabilities is controlled by `script_inject_for()`, which c │ └── .mcode # cached JSON IR ├── cache/ # downloaded package zip archives ├── lock.toml # installed package versions and commit hashes -└── link.toml # local development link overrides +├── link.toml # local development link overrides +└── shop.toml # optional shop configuration and policy flags ``` ## Key Files diff --git a/internal/shop.cm b/internal/shop.cm index 98bb2c36..6ccd21a4 100644 --- a/internal/shop.cm +++ b/internal/shop.cm @@ -339,6 +339,48 @@ Shop.save_lock = function(lock) { } +// Shop configuration (shop.toml) with policy flags +var _shop_config = null +var _default_policy = { + allow_dylib: true, + allow_static: true, + allow_mach: true, + allow_compile: true +} + +Shop.load_config = function() { + if (_shop_config) return _shop_config + if (!global_shop_path) { + _shop_config = {policy: object(_default_policy)} + return _shop_config + } + var path = global_shop_path + '/shop.toml' + if (!fd.is_file(path)) { + _shop_config = {policy: object(_default_policy)} + fd.slurpwrite(path, stone(blob(toml.encode(_shop_config)))) + return _shop_config + } + var content = text(fd.slurp(path)) + if (!length(content)) { + _shop_config = {policy: object(_default_policy)} + return _shop_config + } + _shop_config = toml.decode(content) + if (!_shop_config.policy) _shop_config.policy = {} + var keys = array(_default_policy) + var i = 0 + for (i = 0; i < length(keys); i++) { + if (_shop_config.policy[keys[i]] == null) + _shop_config.policy[keys[i]] = _default_policy[keys[i]] + } + return _shop_config +} + +function get_policy() { + var config = Shop.load_config() + return config.policy +} + // Get information about how to resolve a package // Local packages always start with / Shop.resolve_package_info = function(pkg) { @@ -508,57 +550,73 @@ function resolve_mod_fn(path, pkg) { var cached_mcode_path = null var _pkg_dir = null var _stem = null + var policy = null - // Check for native .cm dylib at deterministic path first + policy = get_policy() + + // Compute _pkg_dir and _stem early so all paths can use them if (pkg) { _pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg) if (starts_with(path, _pkg_dir + '/')) { - _stem = fd.stem(text(path, length(_pkg_dir) + 1)) - native_result = try_native_mod_dylib(pkg, _stem) - if (native_result != null) { - return {_native: true, value: native_result} - } + _stem = text(path, length(_pkg_dir) + 1) + } + } + + // Check for native .cm dylib at deterministic path first + if (policy.allow_dylib && pkg && _stem) { + native_result = try_native_mod_dylib(pkg, _stem) + if (native_result != null) { + return {_native: true, value: native_result} } } // Check cache for pre-compiled .mach blob - cached = pull_from_cache(content_key) - if (cached) { - return cached + if (policy.allow_mach) { + cached = pull_from_cache(content_key) + if (cached) { + return cached + } } // Check for cached mcode in content-addressed store (salted hash to distinguish from mach) - cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) - if (fd.is_file(cached_mcode_path)) { - mcode_json = text(fd.slurp(cached_mcode_path)) - compiled = mach_compile_mcode_bin(path, mcode_json) - put_into_cache(content_key, compiled) - return compiled + if (policy.allow_compile) { + cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) + if (fd.is_file(cached_mcode_path)) { + mcode_json = text(fd.slurp(cached_mcode_path)) + compiled = mach_compile_mcode_bin(path, mcode_json) + put_into_cache(content_key, compiled) + return compiled + } } // Compile via full pipeline: analyze → mcode → streamline → serialize // Load compiler modules from use_cache directly (NOT via Shop.use, which // would re-enter resolve_locator → resolve_mod_fn → infinite recursion) - if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode'] - if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline'] - if (!_mcode_mod || !_streamline_mod) { - print(`error: compiler modules not loaded (mcode=${_mcode_mod != null}, streamline=${_streamline_mod != null})`) - disrupt + if (policy.allow_compile) { + if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode'] + if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline'] + if (!_mcode_mod || !_streamline_mod) { + print(`error: compiler modules not loaded (mcode=${_mcode_mod != null}, streamline=${_streamline_mod != null})`) + disrupt + } + ast = analyze(content, path) + ir = _mcode_mod(ast) + optimized = _streamline_mod(ir) + mcode_json = shop_json.encode(optimized) + + // Cache mcode (architecture-independent) in content-addressed store + ensure_dir(global_shop_path + '/build') + fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json))) + + // Cache mach blob + compiled = mach_compile_mcode_bin(path, mcode_json) + put_into_cache(content_key, compiled) + + return compiled } - ast = analyze(content, path) - ir = _mcode_mod(ast) - optimized = _streamline_mod(ir) - mcode_json = shop_json.encode(optimized) - // Cache mcode (architecture-independent) in content-addressed store - ensure_dir(global_shop_path + '/build') - fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json))) - - // Cache mach blob - compiled = mach_compile_mcode_bin(path, mcode_json) - put_into_cache(content_key, compiled) - - return compiled + print(`Module ${path} could not be loaded: no artifact found or all methods blocked by policy`) + disrupt } // given a path and a package context @@ -659,6 +717,11 @@ function get_dylib_path(pkg, stem) { return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + dylib_ext } +// Get the deterministic mach path for a module in lib//.mach +function get_mach_path(pkg, stem) { + return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + '.mach' +} + // Open a per-module dylib and return the dlopen handle function open_module_dylib(dylib_path) { if (open_dls[dylib_path]) return open_dls[dylib_path] @@ -688,6 +751,9 @@ function resolve_c_symbol(path, package_context) { var canon_pkg = null var mod_name = null var file_stem = null + var policy = null + + policy = get_policy() if (explicit) { if (is_internal_path(explicit.path) && package_context && explicit.package != package_context) @@ -698,18 +764,20 @@ function resolve_c_symbol(path, package_context) { file_stem = replace(explicit.path, '.c', '') // Check lib/ dylib first - loader = try_dylib_symbol(sym, explicit.package, file_stem) - if (loader) { - return { - symbol: loader, - scope: SCOPE_PACKAGE, - package: explicit.package, - path: sym + if (policy.allow_dylib) { + loader = try_dylib_symbol(sym, explicit.package, file_stem) + if (loader) { + return { + symbol: loader, + scope: SCOPE_PACKAGE, + package: explicit.package, + path: sym + } } } // Then check internal/static - if (os.internal_exists(sym)) { + if (policy.allow_static && os.internal_exists(sym)) { return { symbol: function() { return os.load_internal(sym) }, scope: SCOPE_PACKAGE, @@ -724,16 +792,18 @@ function resolve_c_symbol(path, package_context) { core_sym = make_c_symbol('core', path) // Check lib/ dylib first for core - loader = try_dylib_symbol(core_sym, 'core', path) - if (loader) { - return { - symbol: loader, - scope: SCOPE_CORE, - path: core_sym + if (policy.allow_dylib) { + loader = try_dylib_symbol(core_sym, 'core', path) + if (loader) { + return { + symbol: loader, + scope: SCOPE_CORE, + path: core_sym + } } } - if (os.internal_exists(core_sym)) { + if (policy.allow_static && os.internal_exists(core_sym)) { return { symbol: function() { return os.load_internal(core_sym) }, scope: SCOPE_CORE, @@ -746,16 +816,18 @@ function resolve_c_symbol(path, package_context) { // 1. Check own package (dylib first, then internal) sym = make_c_symbol(package_context, path) - loader = try_dylib_symbol(sym, package_context, path) - if (loader) { - return { - symbol: loader, - scope: SCOPE_LOCAL, - path: sym + if (policy.allow_dylib) { + loader = try_dylib_symbol(sym, package_context, path) + if (loader) { + return { + symbol: loader, + scope: SCOPE_LOCAL, + path: sym + } } } - if (os.internal_exists(sym)) { + if (policy.allow_static && os.internal_exists(sym)) { return { symbol: function() { return os.load_internal(sym) }, scope: SCOPE_LOCAL, @@ -774,17 +846,19 @@ function resolve_c_symbol(path, package_context) { mod_name = get_import_name(path) sym = make_c_symbol(canon_pkg, mod_name) - loader = try_dylib_symbol(sym, canon_pkg, mod_name) - if (loader) { - return { - symbol: loader, - scope: SCOPE_PACKAGE, - package: canon_pkg, - path: sym + if (policy.allow_dylib) { + loader = try_dylib_symbol(sym, canon_pkg, mod_name) + if (loader) { + return { + symbol: loader, + scope: SCOPE_PACKAGE, + package: canon_pkg, + path: sym + } } } - if (os.internal_exists(sym)) { + if (policy.allow_static && os.internal_exists(sym)) { return { symbol: function() { return os.load_internal(sym) }, scope: SCOPE_PACKAGE, @@ -798,16 +872,18 @@ function resolve_c_symbol(path, package_context) { // 3. Check core (dylib first, then internal) core_sym = make_c_symbol('core', path) - loader = try_dylib_symbol(core_sym, 'core', path) - if (loader) { - return { - symbol: loader, - scope: SCOPE_CORE, - path: core_sym + if (policy.allow_dylib) { + loader = try_dylib_symbol(core_sym, 'core', path) + if (loader) { + return { + symbol: loader, + scope: SCOPE_CORE, + path: core_sym + } } } - if (os.internal_exists(core_sym)) { + if (policy.allow_static && os.internal_exists(core_sym)) { return { symbol: function() { return os.load_internal(core_sym) }, scope: SCOPE_CORE, @@ -1398,6 +1474,8 @@ Shop.get_lib_dir = function() { return global_shop_path + '/lib' } +Shop.ensure_dir = ensure_dir + Shop.get_local_dir = function() { return global_shop_path + "/local" } @@ -1445,6 +1523,120 @@ Shop.get_dylib_path = function(pkg, stem) { return get_dylib_path(pkg, stem) } +// Get the deterministic mach path for a module (public API) +Shop.get_mach_path = function(pkg, stem) { + return get_mach_path(pkg, stem) +} + +// Load a module explicitly as mach bytecode, bypassing dylib resolution. +// Returns the loaded module value. Disrupts if the module cannot be found. +Shop.load_as_mach = function(path, pkg) { + var locator = resolve_locator(path + '.cm', pkg) + var file_path = null + var content = null + var content_key = null + var cached = null + var cached_mcode_path = null + var mcode_json = null + var compiled = null + var ast = null + var ir = null + var optimized = null + var pkg_dir = null + var stem = null + var mach_path = null + var file_info = null + var inject = null + var env = null + + if (!locator) { print('Module ' + path + ' not found'); disrupt } + + file_path = locator.path + content = text(fd.slurp(file_path)) + content_key = stone(blob(content)) + + // Try installed .mach in lib/ + if (pkg) { + pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg) + if (starts_with(file_path, pkg_dir + '/')) { + stem = text(file_path, length(pkg_dir) + 1) + mach_path = get_mach_path(pkg, stem) + if (fd.is_file(mach_path)) { + compiled = fd.slurp(mach_path) + } + } + } + + // Try cached mach blob + if (!compiled) { + cached = pull_from_cache(content_key) + if (cached) compiled = cached + } + + // Try cached mcode -> compile to mach + if (!compiled) { + cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) + if (fd.is_file(cached_mcode_path)) { + mcode_json = text(fd.slurp(cached_mcode_path)) + compiled = mach_compile_mcode_bin(file_path, mcode_json) + put_into_cache(content_key, compiled) + } + } + + // Full compile from source + if (!compiled) { + if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode'] + if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline'] + if (!_mcode_mod || !_streamline_mod) { + print('error: compiler modules not loaded') + disrupt + } + ast = analyze(content, file_path) + ir = _mcode_mod(ast) + optimized = _streamline_mod(ir) + mcode_json = shop_json.encode(optimized) + cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode"))) + ensure_dir(global_shop_path + '/build') + fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json))) + compiled = mach_compile_mcode_bin(file_path, mcode_json) + put_into_cache(content_key, compiled) + } + + // Load the mach blob with proper env + file_info = Shop.file_info(file_path) + inject = Shop.script_inject_for(file_info) + env = inject_env(inject) + env.use = make_use_fn(file_info.package) + return mach_load(compiled, env) +} + +// Load a module explicitly as a native dylib, bypassing mach resolution. +// Returns the loaded module value, or null if no dylib exists. +Shop.load_as_dylib = function(path, pkg) { + var locator = resolve_locator(path + '.cm', pkg) + var file_path = null + var file_info = null + var pkg_dir = null + var stem = null + var result = null + var real_pkg = pkg + + if (!locator) { print('Module ' + path + ' not found'); disrupt } + + file_path = locator.path + if (!real_pkg) { + file_info = Shop.file_info(file_path) + real_pkg = file_info.package + } + if (!real_pkg) return null + + pkg_dir = get_packages_dir() + '/' + safe_package_path(real_pkg) + if (!starts_with(file_path, pkg_dir + '/')) return null + stem = text(file_path, length(pkg_dir) + 1) + result = try_native_mod_dylib(real_pkg, stem) + return result +} + Shop.audit_packages = function() { var packages = Shop.list_packages()