john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

cache invalidation

Browse Source
This commit is contained in:
John Alanbrook
2026-02-16 00:04:30 -06:00
parent 8f92870141
commit 46c345d34e
4 changed files with 328 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

332
internal/shop.cm
View File

@@ -339,6 +339,48 @@ Shop.save_lock = function(lock) {
}
// Shop configuration (shop.toml) with policy flags
var _shop_config = null
var _default_policy = {
allow_dylib: true,
allow_static: true,
allow_mach: true,
allow_compile: true
}
Shop.load_config = function() {
if (_shop_config) return _shop_config
if (!global_shop_path) {
_shop_config = {policy: object(_default_policy)}
return _shop_config
}
var path = global_shop_path + '/shop.toml'
if (!fd.is_file(path)) {
_shop_config = {policy: object(_default_policy)}
fd.slurpwrite(path, stone(blob(toml.encode(_shop_config))))
return _shop_config
}
var content = text(fd.slurp(path))
if (!length(content)) {
_shop_config = {policy: object(_default_policy)}
return _shop_config
}
_shop_config = toml.decode(content)
if (!_shop_config.policy) _shop_config.policy = {}
var keys = array(_default_policy)
var i = 0
for (i = 0; i < length(keys); i++) {
if (_shop_config.policy[keys[i]] == null)
_shop_config.policy[keys[i]] = _default_policy[keys[i]]
}
return _shop_config
}
function get_policy() {
var config = Shop.load_config()
return config.policy
}
// Get information about how to resolve a package
// Local packages always start with /
Shop.resolve_package_info = function(pkg) {
@@ -508,57 +550,73 @@ function resolve_mod_fn(path, pkg) {
var cached_mcode_path = null
var _pkg_dir = null
var _stem = null
var policy = null
// Check for native .cm dylib at deterministic path first
policy = get_policy()
// Compute _pkg_dir and _stem early so all paths can use them
if (pkg) {
_pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
if (starts_with(path, _pkg_dir + '/')) {
_stem = fd.stem(text(path, length(_pkg_dir) + 1))
native_result = try_native_mod_dylib(pkg, _stem)
if (native_result != null) {
return {_native: true, value: native_result}
}
_stem = text(path, length(_pkg_dir) + 1)
}
}
// Check for native .cm dylib at deterministic path first
if (policy.allow_dylib && pkg && _stem) {
native_result = try_native_mod_dylib(pkg, _stem)
if (native_result != null) {
return {_native: true, value: native_result}
}
}
// Check cache for pre-compiled .mach blob
cached = pull_from_cache(content_key)
if (cached) {
return cached
if (policy.allow_mach) {
cached = pull_from_cache(content_key)
if (cached) {
return cached
}
}
// Check for cached mcode in content-addressed store (salted hash to distinguish from mach)
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
if (policy.allow_compile) {
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
}
}
// Compile via full pipeline: analyze → mcode → streamline → serialize
// Load compiler modules from use_cache directly (NOT via Shop.use, which
// would re-enter resolve_locator → resolve_mod_fn → infinite recursion)
if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode']
if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline']
if (!_mcode_mod || !_streamline_mod) {
print(`error: compiler modules not loaded (mcode=${_mcode_mod != null}, streamline=${_streamline_mod != null})`)
disrupt
if (policy.allow_compile) {
if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode']
if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline']
if (!_mcode_mod || !_streamline_mod) {
print(`error: compiler modules not loaded (mcode=${_mcode_mod != null}, streamline=${_streamline_mod != null})`)
disrupt
}
ast = analyze(content, path)
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
// Cache mcode (architecture-independent) in content-addressed store
ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
// Cache mach blob
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
}
ast = analyze(content, path)
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
// Cache mcode (architecture-independent) in content-addressed store
ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
// Cache mach blob
compiled = mach_compile_mcode_bin(path, mcode_json)
put_into_cache(content_key, compiled)
return compiled
print(`Module ${path} could not be loaded: no artifact found or all methods blocked by policy`)
disrupt
}
// given a path and a package context
@@ -659,6 +717,11 @@ function get_dylib_path(pkg, stem) {
return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + dylib_ext
}
// Get the deterministic mach path for a module in lib/<pkg>/<stem>.mach
function get_mach_path(pkg, stem) {
return global_shop_path + '/lib/' + safe_package_path(pkg) + '/' + stem + '.mach'
}
// Open a per-module dylib and return the dlopen handle
function open_module_dylib(dylib_path) {
if (open_dls[dylib_path]) return open_dls[dylib_path]
@@ -688,6 +751,9 @@ function resolve_c_symbol(path, package_context) {
var canon_pkg = null
var mod_name = null
var file_stem = null
var policy = null
policy = get_policy()
if (explicit) {
if (is_internal_path(explicit.path) && package_context && explicit.package != package_context)
@@ -698,18 +764,20 @@ function resolve_c_symbol(path, package_context) {
file_stem = replace(explicit.path, '.c', '')
// Check lib/ dylib first
loader = try_dylib_symbol(sym, explicit.package, file_stem)
if (loader) {
return {
symbol: loader,
scope: SCOPE_PACKAGE,
package: explicit.package,
path: sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(sym, explicit.package, file_stem)
if (loader) {
return {
symbol: loader,
scope: SCOPE_PACKAGE,
package: explicit.package,
path: sym
}
}
}
// Then check internal/static
if (os.internal_exists(sym)) {
if (policy.allow_static && os.internal_exists(sym)) {
return {
symbol: function() { return os.load_internal(sym) },
scope: SCOPE_PACKAGE,
@@ -724,16 +792,18 @@ function resolve_c_symbol(path, package_context) {
core_sym = make_c_symbol('core', path)
// Check lib/ dylib first for core
loader = try_dylib_symbol(core_sym, 'core', path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_CORE,
path: core_sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(core_sym, 'core', path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_CORE,
path: core_sym
}
}
}
if (os.internal_exists(core_sym)) {
if (policy.allow_static && os.internal_exists(core_sym)) {
return {
symbol: function() { return os.load_internal(core_sym) },
scope: SCOPE_CORE,
@@ -746,16 +816,18 @@ function resolve_c_symbol(path, package_context) {
// 1. Check own package (dylib first, then internal)
sym = make_c_symbol(package_context, path)
loader = try_dylib_symbol(sym, package_context, path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_LOCAL,
path: sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(sym, package_context, path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_LOCAL,
path: sym
}
}
}
if (os.internal_exists(sym)) {
if (policy.allow_static && os.internal_exists(sym)) {
return {
symbol: function() { return os.load_internal(sym) },
scope: SCOPE_LOCAL,
@@ -774,17 +846,19 @@ function resolve_c_symbol(path, package_context) {
mod_name = get_import_name(path)
sym = make_c_symbol(canon_pkg, mod_name)
loader = try_dylib_symbol(sym, canon_pkg, mod_name)
if (loader) {
return {
symbol: loader,
scope: SCOPE_PACKAGE,
package: canon_pkg,
path: sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(sym, canon_pkg, mod_name)
if (loader) {
return {
symbol: loader,
scope: SCOPE_PACKAGE,
package: canon_pkg,
path: sym
}
}
}
if (os.internal_exists(sym)) {
if (policy.allow_static && os.internal_exists(sym)) {
return {
symbol: function() { return os.load_internal(sym) },
scope: SCOPE_PACKAGE,
@@ -798,16 +872,18 @@ function resolve_c_symbol(path, package_context) {
// 3. Check core (dylib first, then internal)
core_sym = make_c_symbol('core', path)
loader = try_dylib_symbol(core_sym, 'core', path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_CORE,
path: core_sym
if (policy.allow_dylib) {
loader = try_dylib_symbol(core_sym, 'core', path)
if (loader) {
return {
symbol: loader,
scope: SCOPE_CORE,
path: core_sym
}
}
}
if (os.internal_exists(core_sym)) {
if (policy.allow_static && os.internal_exists(core_sym)) {
return {
symbol: function() { return os.load_internal(core_sym) },
scope: SCOPE_CORE,
@@ -1398,6 +1474,8 @@ Shop.get_lib_dir = function() {
return global_shop_path + '/lib'
}
Shop.ensure_dir = ensure_dir
Shop.get_local_dir = function() {
return global_shop_path + "/local"
}
@@ -1445,6 +1523,120 @@ Shop.get_dylib_path = function(pkg, stem) {
return get_dylib_path(pkg, stem)
}
// Get the deterministic mach path for a module (public API)
Shop.get_mach_path = function(pkg, stem) {
return get_mach_path(pkg, stem)
}
// Load a module explicitly as mach bytecode, bypassing dylib resolution.
// Returns the loaded module value. Disrupts if the module cannot be found.
Shop.load_as_mach = function(path, pkg) {
var locator = resolve_locator(path + '.cm', pkg)
var file_path = null
var content = null
var content_key = null
var cached = null
var cached_mcode_path = null
var mcode_json = null
var compiled = null
var ast = null
var ir = null
var optimized = null
var pkg_dir = null
var stem = null
var mach_path = null
var file_info = null
var inject = null
var env = null
if (!locator) { print('Module ' + path + ' not found'); disrupt }
file_path = locator.path
content = text(fd.slurp(file_path))
content_key = stone(blob(content))
// Try installed .mach in lib/
if (pkg) {
pkg_dir = get_packages_dir() + '/' + safe_package_path(pkg)
if (starts_with(file_path, pkg_dir + '/')) {
stem = text(file_path, length(pkg_dir) + 1)
mach_path = get_mach_path(pkg, stem)
if (fd.is_file(mach_path)) {
compiled = fd.slurp(mach_path)
}
}
}
// Try cached mach blob
if (!compiled) {
cached = pull_from_cache(content_key)
if (cached) compiled = cached
}
// Try cached mcode -> compile to mach
if (!compiled) {
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
if (fd.is_file(cached_mcode_path)) {
mcode_json = text(fd.slurp(cached_mcode_path))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
put_into_cache(content_key, compiled)
}
}
// Full compile from source
if (!compiled) {
if (!_mcode_mod) _mcode_mod = use_cache['core/mcode'] || use_cache['mcode']
if (!_streamline_mod) _streamline_mod = use_cache['core/streamline'] || use_cache['streamline']
if (!_mcode_mod || !_streamline_mod) {
print('error: compiler modules not loaded')
disrupt
}
ast = analyze(content, file_path)
ir = _mcode_mod(ast)
optimized = _streamline_mod(ir)
mcode_json = shop_json.encode(optimized)
cached_mcode_path = global_shop_path + '/build/' + content_hash(stone(blob(text(content_key) + "\nmcode")))
ensure_dir(global_shop_path + '/build')
fd.slurpwrite(cached_mcode_path, stone(blob(mcode_json)))
compiled = mach_compile_mcode_bin(file_path, mcode_json)
put_into_cache(content_key, compiled)
}
// Load the mach blob with proper env
file_info = Shop.file_info(file_path)
inject = Shop.script_inject_for(file_info)
env = inject_env(inject)
env.use = make_use_fn(file_info.package)
return mach_load(compiled, env)
}
// Load a module explicitly as a native dylib, bypassing mach resolution.
// Returns the loaded module value, or null if no dylib exists.
Shop.load_as_dylib = function(path, pkg) {
var locator = resolve_locator(path + '.cm', pkg)
var file_path = null
var file_info = null
var pkg_dir = null
var stem = null
var result = null
var real_pkg = pkg
if (!locator) { print('Module ' + path + ' not found'); disrupt }
file_path = locator.path
if (!real_pkg) {
file_info = Shop.file_info(file_path)
real_pkg = file_info.package
}
if (!real_pkg) return null
pkg_dir = get_packages_dir() + '/' + safe_package_path(real_pkg)
if (!starts_with(file_path, pkg_dir + '/')) return null
stem = text(file_path, length(pkg_dir) + 1)
result = try_native_mod_dylib(real_pkg, stem)
return result
}
Shop.audit_packages = function() {
var packages = Shop.list_packages()