john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

cache invalidation

Browse Source
This commit is contained in:
John Alanbrook
2026-02-16 00:04:30 -06:00
parent 8f92870141
commit 46c345d34e
4 changed files with 328 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
build.cm
View File

@@ -288,7 +288,7 @@ Build.build_module_dylib = function(pkg, file, target, buildtype) {
}
// Install to deterministic lib/<pkg>/<stem>.dylib
var file_stem = fd.stem(file)
var file_stem = file
var install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
var stem_dir = fd.dirname(file_stem)
if (stem_dir && stem_dir != '.') {
@@ -539,7 +539,7 @@ Build.compile_native = function(src_path, target, buildtype, pkg) {
// Install to deterministic lib/<pkg>/<stem>.dylib
if (pkg) {
native_stem = fd.stem(fd.basename(src_path))
native_stem = fd.basename(src_path)
native_install_dir = shop.get_lib_dir() + '/' + shop.lib_name_for_package(pkg)
ensure_dir(native_install_dir)
native_install_path = native_install_dir + '/' + native_stem + dylib_ext

82
compile.ce
View File

@@ -1,85 +1,27 @@
// compile.ce — compile a .cm module to native .dylib via QBE
// compile.ce — compile a .cm or .ce file to native .dylib via QBE
//
// Usage:
// cell --dev compile.ce <file.cm>
// cell compile <file.cm|file.ce>
//
// Produces <file>.dylib in the current directory.
// Installs the dylib to .cell/lib/<pkg>/<stem>.dylib
var shop = use('internal/shop')
var build = use('build')
var fd = use('fd')
var os = use('os')
if (length(args) < 1) {
print('usage: cell --dev compile.ce <file.cm>')
print('usage: cell compile <file.cm|file.ce>')
return
}
var file = args[0]
var base = file
if (ends_with(base, '.cm')) {
base = text(base, 0, length(base) - 3)
}
var safe = replace(replace(base, '/', '_'), '-', '_')
var symbol = 'js_' + safe + '_use'
var tmp = '/tmp/qbe_' + safe
var ssa_path = tmp + '.ssa'
var s_path = tmp + '.s'
var o_path = tmp + '.o'
var rt_o_path = '/tmp/qbe_rt.o'
var dylib_path = file + '.dylib'
var cwd = fd.getcwd()
var rc = 0
// Step 1: emit QBE IL
print('emit qbe...')
rc = os.system('cd ' + cwd + ' && ./cell --dev qbe.ce ' + file + ' > ' + ssa_path)
if (rc != 0) {
print('failed to emit qbe il')
if (!fd.is_file(file)) {
print('file not found: ' + file)
return
}
// Step 2: append wrapper function — called as symbol(ctx) by os.dylib_symbol.
// Delegates to cell_rt_module_entry which heap-allocates a frame
// (so closures survive) and calls cell_main.
var wrapper_cmd = `printf '\nexport function l $` + symbol + `(l %%ctx) {\n@entry\n %%result =l call $cell_rt_module_entry(l %%ctx)\n ret %%result\n}\n' >> ` + ssa_path
rc = os.system(wrapper_cmd)
if (rc != 0) {
print('wrapper append failed')
return
}
var abs = fd.realpath(file)
var file_info = shop.file_info(abs)
var pkg = file_info.package
// Step 3: compile QBE IL to assembly
print('qbe compile...')
rc = os.system('qbe -o ' + s_path + ' ' + ssa_path)
if (rc != 0) {
print('qbe compilation failed')
return
}
// Step 4: assemble
print('assemble...')
rc = os.system('cc -c ' + s_path + ' -o ' + o_path)
if (rc != 0) {
print('assembly failed')
return
}
// Step 5: compile runtime stubs (cached — skip if already built)
if (!fd.is_file(rt_o_path)) {
print('compile runtime stubs...')
rc = os.system('cc -c ' + cwd + '/qbe_rt.c -o ' + rt_o_path + ' -fPIC')
if (rc != 0) {
print('runtime stubs compilation failed')
return
}
}
// Step 6: link dylib
print('link...')
rc = os.system('cc -shared -fPIC -undefined dynamic_lookup ' + o_path + ' ' + rt_o_path + ' -o ' + cwd + '/' + dylib_path)
if (rc != 0) {
print('linking failed')
return
}
print('built: ' + dylib_path)
build.compile_native(abs, null, null, pkg)

60
docs/shop.md
View File

@@ -63,16 +63,17 @@ When loading a module, the shop checks (in order):
1. **In-memory cache** — `use_cache[key]`, checked first on every `use()` call
2. **Installed dylib** — per-file `.dylib` in `~/.pit/lib/<pkg>/<stem>.dylib`
3. **Internal symbols** — statically linked into the `pit` binary (fat builds)
4. **Installed mach** — pre-compiled bytecode in `~/.pit/lib/<pkg>/<stem>.mach`
5. **Cached bytecode** — content-addressed in `~/.pit/build/<hash>` (no extension)
6. **Cached .mcode IR** — JSON IR in `~/.pit/build/<hash>.mcode`
7. **Adjacent .mach/.mcode** — files alongside the source (e.g., `sprite.mach`)
8. **Source compilation** — full pipeline: analyze, mcode, streamline, serialize
3. **Installed mach** — pre-compiled bytecode in `~/.pit/lib/<pkg>/<stem>.mach`
4. **Cached bytecode** — content-addressed in `~/.pit/build/<hash>` (no extension)
5. **Cached .mcode IR** — JSON IR in `~/.pit/build/<hash>.mcode`
6. **Internal symbols** — statically linked into the `pit` binary (fat builds)
7. **Source compilation** — full pipeline: analyze, mcode, streamline, serialize
When both a `.dylib` and `.mach` exist for the same module in `lib/`, the dylib is selected. Dylib resolution also wins over internal symbols, so a dylib in `lib/` can hot-patch a fat binary. Delete the dylib to fall back to mach or static.
Results from steps 6-8 are cached back to the content-addressed store for future loads.
Results from steps 5-7 are cached back to the content-addressed store for future loads.
Each loading method (except the in-memory cache) can be individually enabled or disabled via `shop.toml` policy flags — see [Shop Configuration](#shop-configuration) below.
### Content-Addressed Store
@@ -124,6 +125,48 @@ When a module is loaded, the shop builds an `env` object that becomes the module
The set of injected capabilities is controlled by `script_inject_for()`, which can be tuned per package or file.
## Shop Configuration
The shop reads an optional `shop.toml` file from the shop root (`~/.pit/shop.toml`). This file controls which loading methods are permitted through policy flags.
### Policy Flags
All flags default to `true`. Set a flag to `false` to disable that loading method.
```toml
[policy]
allow_dylib = true # per-file .dylib loading (requires dlopen)
allow_static = true # statically linked C symbols (fat builds)
allow_mach = true # pre-compiled .mach bytecode (lib/ and build cache)
allow_compile = true # on-the-fly source compilation
```
### Example Configurations
**Production lockdown** — only use pre-compiled artifacts, never compile from source:
```toml
[policy]
allow_compile = false
```
**Pure-script mode** — bytecode only, no native code:
```toml
[policy]
allow_dylib = false
allow_static = false
```
**No dlopen platforms** — static linking and bytecode only:
```toml
[policy]
allow_dylib = false
```
If `shop.toml` is missing or has no `[policy]` section, all methods are enabled (default behavior).
## Shop Directory Layout
```
@@ -145,7 +188,8 @@ The set of injected capabilities is controlled by `script_inject_for()`, which c
│ └── <hash>.mcode # cached JSON IR
├── cache/ # downloaded package zip archives
├── lock.toml # installed package versions and commit hashes
└── link.toml # local development link overrides
├── link.toml # local development link overrides
└── shop.toml # optional shop configuration and policy flags
```
## Key Files

332
internal/shop.cm
View File

@@ -339,6 +339,48 @@ Shop.save_lock = function(lock) {
}
// Shop configuration (shop.toml) with policy flags
var _shop_config = null
var _default_policy = {
allow_dylib: true,
allow_static: true,
allow_mach: true,
allow_compile: true
}
Shop.load_config = function() {
if (_shop_config) return _shop_config
if (!global_shop_path) {
_shop_config = {policy: object(_default_policy)}
return _shop_config
}
var path = global_shop_path + '/shop.toml'
if (!fd.is_file(path)) {
_shop_config = {policy: object(_default_policy)}
fd.slurpwrite(path, stone(blob(toml.encode(_shop_config))))
return _shop_config
}
var content = text(fd.slurp(path))
if (!length(content)) {
_shop_config = {policy: object(_default_policy)}
return _shop_config
}
_shop_config = toml.decode(content)
if (!_shop_config.policy) _shop_config.policy = {}
var keys = array(_default_policy)
var i = 0
for (i = 0; i < length(keys); i++) {
if (_shop_config.policy[keys[i]] == null)
_shop_config.policy[keys[i]] = _default_policy[keys[i]]
}
return _shop_config
}
function get_policy() {
var config = Shop.load_config()
return config.policy
}
// Get information about how to resolve a package
// Local packages always start with /
Shop.resolve_package_info = function(pkg) {
@@ -508,57 +550,73 @@ function resolve_mod_fn(path, pkg) {
var cached_mcode_path = null
var _pkg_dir = null
var _stem = null
var policy = null
// Check for native .cm dylib at deterministic path first
policy = get_policy()
// Compute _pkg_dir and _stem early so all paths can use them
if (pkg) {
_pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
if (starts_with(path, _pkg_dir + '/')) {
_stem = fd.stem(text(path, length(_pkg_dir) + 1))
native_result = try_native_mod_dylib(pkg, _stem)
if (native_result != null) {
return {_native: true, value: native_result}
}
_stem = text(path, length(_pkg_dir) + 1)
}
}
// Check for native .cm dylib at deterministic path first
if (policy.allow_dylib && pkg && _stem) {
native_result = try_native_mod_dylib(pkg, _stem)
if (native_result != null) {
return {_native: true, value: native_result}
}
}
// Check cache for pre-compiled .mach blob
cached = pull_from_cache(content_key)
if (cached) {
return cached
if (policy.allow_mach) {
cached = pull_from_cache(content_key)
if (cached) {
return cached
}
}
// Check for cached mcode in content-addressed store (salted hash to distinguish from mach)
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
if (policy.allow_compile) {
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
}
}
// Compile via full pipeline: analyze → mcode → streamline → serialize
// Load compiler modules from use_cache directly (NOT via Shop.use, which
// would re-enter resolve_locator → resolve_mod_fn → infinite recursion)
if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode']
if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline']
if (!_mcode_mod || !_streamline_mod) {
print(`error: compiler modules not loaded (mcode=${_mcode_mod != null}, streamline=${_streamline_mod != null})`)
disrupt
if (policy.allow_compile) {
if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode']
if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline']
if (!_mcode_mod || !_streamline_mod) {
print(`error: compiler modules not loaded (mcode=${_mcode_mod != null}, streamline=${_streamline_mod != null})`)
disrupt
}
ast = analyze(content, path)
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
// Cache mcode (architecture-independent) in content-addressed store
ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
// Cache mach blob
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
}
ast = analyze(content, path)
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
// Cache mcode (architecture-independent) in content-addressed store
ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
// Cache mach blob
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
print(`Module ${path} could not be loaded: no artifact found or all methods blocked by policy`)
disrupt
}
// given a path and a package context
@@ -659,6 +717,11 @@ function get_dylib_path(pkg, stem) {
return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + dylib_ext
}
// Get the deterministic mach path for a module in lib/<pkg>/<stem>.mach
function get_mach_path(pkg, stem) {
return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + '.mach'
}
// Open a per-module dylib and return the dlopen handle
function open_module_dylib(dylib_path) {
if (open_dls[dylib_path]) return open_dls[dylib_path]
@@ -688,6 +751,9 @@ function resolve_c_symbol(path, package_context) {
var canon_pkg = null
var mod_name = null
var file_stem = null
var policy = null
policy = get_policy()
if (explicit) {
if (is_internal_path(explicit.path) && package_context && explicit.package != package_context)
@@ -698,18 +764,20 @@ function resolve_c_symbol(path, package_context) {
file_stem = replace(explicit.path, '.c', '')
// Check lib/ dylib first
loader = try_dylib_symbol(sym, explicit.package, file_stem)
if (loader) {
return {
symbol: loader,
scope: SCOPE_PACKAGE,
package: explicit.package,
path: sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(sym, explicit.package, file_stem)
if (loader) {
return {
symbol: loader,
scope: SCOPE_PACKAGE,
package: explicit.package,
path: sym
}
}
}
// Then check internal/static
if (os.internal_exists(sym)) {
if (policy.allow_static && os.internal_exists(sym)) {
return {
symbol: function() { return os.load_internal(sym) },
scope: SCOPE_PACKAGE,
@@ -724,16 +792,18 @@ function resolve_c_symbol(path, package_context) {
core_sym = make_c_symbol('core', path)
// Check lib/ dylib first for core
loader = try_dylib_symbol(core_sym, 'core', path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_CORE,
path: core_sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(core_sym, 'core', path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_CORE,
path: core_sym
}
}
}
if (os.internal_exists(core_sym)) {
if (policy.allow_static && os.internal_exists(core_sym)) {
return {
symbol: function() { return os.load_internal(core_sym) },
scope: SCOPE_CORE,
@@ -746,16 +816,18 @@ function resolve_c_symbol(path, package_context) {
// 1. Check own package (dylib first, then internal)
sym = make_c_symbol(package_context, path)
loader = try_dylib_symbol(sym, package_context, path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_LOCAL,
path: sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(sym, package_context, path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_LOCAL,
path: sym
}
}
}
if (os.internal_exists(sym)) {
if (policy.allow_static && os.internal_exists(sym)) {
return {
symbol: function() { return os.load_internal(sym) },
scope: SCOPE_LOCAL,
@@ -774,17 +846,19 @@ function resolve_c_symbol(path, package_context) {
mod_name = get_import_name(path)
sym = make_c_symbol(canon_pkg, mod_name)
loader = try_dylib_symbol(sym, canon_pkg, mod_name)
if (loader) {
return {
symbol: loader,
scope: SCOPE_PACKAGE,
package: canon_pkg,
path: sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(sym, canon_pkg, mod_name)
if (loader) {
return {
symbol: loader,
scope: SCOPE_PACKAGE,
package: canon_pkg,
path: sym
}
}
}
if (os.internal_exists(sym)) {
if (policy.allow_static && os.internal_exists(sym)) {
return {
symbol: function() { return os.load_internal(sym) },
scope: SCOPE_PACKAGE,
@@ -798,16 +872,18 @@ function resolve_c_symbol(path, package_context) {
// 3. Check core (dylib first, then internal)
core_sym = make_c_symbol('core', path)
loader = try_dylib_symbol(core_sym, 'core', path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_CORE,
path: core_sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(core_sym, 'core', path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_CORE,
path: core_sym
}
}
}
if (os.internal_exists(core_sym)) {
if (policy.allow_static && os.internal_exists(core_sym)) {
return {
symbol: function() { return os.load_internal(core_sym) },
scope: SCOPE_CORE,
@@ -1398,6 +1474,8 @@ Shop.get_lib_dir = function() {
return global_shop_path + '/lib'
}
Shop.ensure_dir = ensure_dir
Shop.get_local_dir = function() {
return global_shop_path + "/local"
}
@@ -1445,6 +1523,120 @@ Shop.get_dylib_path = function(pkg, stem) {
return get_dylib_path(pkg, stem)
}
// Get the deterministic mach path for a module (public API)
Shop.get_mach_path = function(pkg, stem) {
return get_mach_path(pkg, stem)
}
// Load a module explicitly as mach bytecode, bypassing dylib resolution.
// Returns the loaded module value. Disrupts if the module cannot be found.
Shop.load_as_mach = function(path, pkg) {
var locator = resolve_locator(path + '.cm', pkg)
var file_path = null
var content = null
var content_key = null
var cached = null
var cached_mcode_path = null
var mcode_json = null
var compiled = null
var ast = null
var ir = null
var optimized = null
var pkg_dir = null
var stem = null
var mach_path = null
var file_info = null
var inject = null
var env = null
if (!locator) { print('Module ' + path + ' not found'); disrupt }
file_path = locator.path
content = text(fd.slurp(file_path))
content_key = stone(blob(content))
// Try installed .mach in lib/
if (pkg) {
pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
if (starts_with(file_path, pkg_dir + '/')) {
stem = text(file_path, length(pkg_dir) + 1)
mach_path = get_mach_path(pkg, stem)
if (fd.is_file(mach_path)) {
compiled = fd.slurp(mach_path)
}
}
}
// Try cached mach blob
if (!compiled) {
cached = pull_from_cache(content_key)
if (cached) compiled = cached
}
// Try cached mcode -> compile to mach
if (!compiled) {
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
put_into_cache(content_key, compiled)
}
}
// Full compile from source
if (!compiled) {
if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode']
if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline']
if (!_mcode_mod || !_streamline_mod) {
print('error: compiler modules not loaded')
disrupt
}
ast = analyze(content, file_path)
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
put_into_cache(content_key, compiled)
}
// Load the mach blob with proper env
file_info = Shop.file_info(file_path)
inject = Shop.script_inject_for(file_info)
env = inject_env(inject)
env.use = make_use_fn(file_info.package)
return mach_load(compiled, env)
}
// Load a module explicitly as a native dylib, bypassing mach resolution.
// Returns the loaded module value, or null if no dylib exists.
Shop.load_as_dylib = function(path, pkg) {
var locator = resolve_locator(path + '.cm', pkg)
var file_path = null
var file_info = null
var pkg_dir = null
var stem = null
var result = null
var real_pkg = pkg
if (!locator) { print('Module ' + path + ' not found'); disrupt }
file_path = locator.path
if (!real_pkg) {
file_info = Shop.file_info(file_path)
real_pkg = file_info.package
}
if (!real_pkg) return null
pkg_dir = get_packages_dir() + '/' + safe_package_path(real_pkg)
if (!starts_with(file_path, pkg_dir + '/')) return null
stem = text(file_path, length(pkg_dir) + 1)
result = try_native_mod_dylib(real_pkg, stem)
return result
}
Shop.audit_packages = function() {
var packages = Shop.list_packages()