john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

updated docs for dylib paths

Browse Source
This commit is contained in:
John Alanbrook
2026-02-18 20:30:54 -06:00
parent e2c26737f4
commit 777474ab4f
9 changed files with 115 additions and 72 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
CLAUDE.md
View File

@@ -177,7 +177,7 @@ When running locally with `./cell --dev`, these commands manage packages:
./cell --dev list # list installed packages
```
Local paths are symlinked into `.cell/packages/`. The build step compiles C files to `.cell/lib/<pkg>/<stem>.dylib`. C files in `src/` are support files linked into module dylibs, not standalone modules.
Local paths are symlinked into `.cell/packages/`. The build step compiles C files to content-addressed dylibs in `~/.cell/build/<hash>` and writes a per-package manifest so the runtime can find them. C files in `src/` are support files linked into module dylibs, not standalone modules.
## Debugging Compiler Issues

11
build.cm
View File

@@ -97,6 +97,11 @@ function cache_path(content, salt) {
return get_build_dir() + '/' + content_hash(content + '\n' + salt)
}
// Deterministic manifest path for a package's built dylibs
function manifest_path(pkg) {
return get_build_dir() + '/' + content_hash(pkg + '\n' + 'manifest')
}
function get_build_dir() {
return shop.get_build_dir()
}
@@ -499,6 +504,11 @@ Build.build_dynamic = function(pkg, target, buildtype, opts) {
}
})
// Write manifest so runtime can find dylibs without the build module
var json = use('json')
var mpath = manifest_path(pkg)
fd.slurpwrite(mpath, stone(blob(json.encode(results))))
return results
}
@@ -957,5 +967,6 @@ Build.SALT_MCODE = SALT_MCODE
Build.SALT_DEPS = SALT_DEPS
Build.SALT_FAIL = SALT_FAIL
Build.cache_path = cache_path
Build.manifest_path = manifest_path
return Build

2
docs/_index.md
View File

@@ -81,7 +81,7 @@ cd cell
make bootstrap
```
The ƿit shop is stored at `~/.pit/`.
The ƿit shop is stored at `~/.cell/`.
## Development

2
docs/c-modules.md
View File

@@ -231,7 +231,7 @@ C files are automatically compiled when you run:
cell --dev build
```
Each C file is compiled into a per-file dynamic library at `.cell/lib/<pkg>/<stem>.dylib`.
Each C file is compiled into a per-file dynamic library at a content-addressed path in `~/.cell/build/<hash>`. A manifest is written for each package so the runtime can find dylibs without rerunning the build pipeline — see [Dylib Manifests](/docs/shop/#dylib-manifests).
## Compilation Flags (cell.toml)

15
docs/cli.md
View File

@@ -53,7 +53,7 @@ For local paths, the package is symlinked into the shop rather than copied. Chan
### pit build
Build C modules for a package. Compiles each C file into a per-file dynamic library and installs them to `~/.pit/lib/<pkg>/<stem>.dylib`. C files in `src/` directories are compiled as support objects and linked into the module dylibs.
Build C modules for a package. Compiles each C file into a per-file dynamic library stored in the content-addressed build cache at `~/.cell/build/<hash>`. A per-package manifest is written so the runtime can find dylibs by package name. C files in `src/` directories are compiled as support objects and linked into the module dylibs. Files that previously failed to compile are skipped automatically (cached failure markers); they are retried when the source or compiler flags change.
```bash
pit build # build all packages
@@ -174,7 +174,7 @@ Output includes median, mean, standard deviation, and percentiles for each bench
## Shop Commands
These commands operate on the global shop (`~/.pit/`) or system-level state.
These commands operate on the global shop (`~/.cell/`) or system-level state.
### pit install
@@ -430,7 +430,7 @@ pit qbe <file.cm>
Compile a source file to a native dynamic library.
```bash
pit compile <file.cm> # outputs .dylib to .cell/lib/
pit compile <file.cm> # outputs .dylib to ~/.cell/build/
pit compile <file.ce>
```
@@ -555,15 +555,12 @@ pit install /Users/john/work/mylib
## Configuration
ƿit stores its data in `~/.pit/`:
ƿit stores its data in `~/.cell/`:
```
~/.pit/
~/.cell/
├── packages/ # installed package sources
├── lib/ # installed per-file dylibs and mach (persistent)
│ ├── core/ # core package: .dylib and .mach files
│ └── <pkg>/ # per-package subdirectories
├── build/ # ephemeral build cache (safe to delete)
├── build/ # content-addressed cache (bytecode, dylibs, manifests)
├── cache/ # downloaded archives
├── lock.toml # installed package versions
└── link.toml # local development links

20
docs/packages.md
View File

@@ -91,10 +91,10 @@ Local packages are symlinked into the shop, making development seamless.
## The Shop
ƿit stores all packages in the **shop** at `~/.pit/`:
ƿit stores all packages in the **shop** at `~/.cell/`:
```
~/.pit/
~/.cell/
├── packages/
│ ├── core -> gitea.pockle.world/john/cell
│ ├── gitea.pockle.world/
@@ -105,14 +105,8 @@ Local packages are symlinked into the shop, making development seamless.
│ └── john/
│ └── work/
│ └── mylib -> /Users/john/work/mylib
├── lib/
│ ├── core/
│ │ ├── fd.dylib
│ │ └── time.mach
│ └── gitea_pockle_world_john_prosperon/
│ └── sprite.dylib
├── build/
│ └── <content-addressed cache>
│ └── <content-addressed cache (bytecode, dylibs, manifests)>
├── cache/
│ └── <downloaded zips>
├── lock.toml
@@ -175,16 +169,16 @@ pit link delete gitea.pockle.world/john/prosperon
## C Extensions
C files in a package are compiled into per-file dynamic libraries:
C files in a package are compiled into per-file dynamic libraries stored in the content-addressed build cache:
```
mypackage/
├── cell.toml
├── render.c # compiled to lib/mypackage/render.dylib
└── physics.c # compiled to lib/mypackage/physics.dylib
├── render.c # compiled to ~/.cell/build/<hash>
└── physics.c # compiled to ~/.cell/build/<hash>
```
Each `.c` file gets its own `.dylib` in `~/.pit/lib/<pkg>/`. A `.c` file and `.cm` file with the same stem at the same scope is a build error — use distinct names.
Each `.c` file gets its own `.dylib` at a content-addressed path in `~/.cell/build/`. A per-package manifest maps module names to their dylib paths so the runtime can find them — see [Dylib Manifests](/docs/shop/#dylib-manifests). A `.c` file and `.cm` file with the same stem at the same scope is a build error — use distinct names.
See [Writing C Modules](/docs/c-modules/) for details.

80
docs/shop.md
View File

@@ -17,7 +17,7 @@ When `pit` runs a program, startup takes one of two paths:
C runtime → engine.cm (from cache) → shop.cm → user program
```
The C runtime hashes the source of `internal/engine.cm` with BLAKE2 and looks up the hash in the content-addressed cache (`~/.pit/build/<hash>`). On a cache hit, engine.cm loads directly — no bootstrap involved.
The C runtime hashes the source of `internal/engine.cm` with BLAKE2 and looks up the hash in the content-addressed cache (`~/.cell/build/<hash>`). On a cache hit, engine.cm loads directly — no bootstrap involved.
### Cold path (first run or cache cleared)
@@ -37,7 +37,7 @@ On a cache miss, the C runtime loads `boot/bootstrap.cm.mcode` (a pre-compiled s
### Cache invalidation
All caching is content-addressed by BLAKE2 hash of the source. When any source file changes, its hash changes and the old cache entry is simply never looked up again. No manual invalidation is needed. To force a full rebuild, delete `~/.pit/build/`.
All caching is content-addressed by BLAKE2 hash of the source. When any source file changes, its hash changes and the old cache entry is simply never looked up again. No manual invalidation is needed. To force a full rebuild, delete `~/.cell/build/`.
## Module Resolution
@@ -47,7 +47,7 @@ When `use('path')` is called from a package context, the shop resolves the modul
For a call like `use('sprite')` from package `myapp`:
1. **Own package** — `~/.pit/packages/myapp/sprite.cm` and C symbol `js_myapp_sprite_use`
1. **Own package** — `~/.cell/packages/myapp/sprite.cm` and C symbol `js_myapp_sprite_use`
2. **Aliased dependencies** — if `myapp/cell.toml` has `renderer = "gitea.pockle.world/john/renderer"`, checks `renderer/sprite.cm` and its C symbols
3. **Core** — built-in core modules and internal C symbols
@@ -80,31 +80,49 @@ Every module goes through a content-addressed caching pipeline. The cache key is
When loading a module, the shop checks (in order):
1. **In-memory cache** — `use_cache[key]`, checked first on every `use()` call
2. **Installed dylib** — per-file `.dylib` in `~/.pit/lib/<pkg>/<stem>.dylib`
3. **Installed mach** — pre-compiled bytecode in `~/.pit/lib/<pkg>/<stem>.mach`
4. **Cached bytecode** — content-addressed in `~/.pit/build/<hash>` (no extension)
5. **Cached .mcode IR** — JSON IR in `~/.pit/build/<hash>.mcode`
6. **Internal symbols** — statically linked into the `pit` binary (fat builds)
7. **Source compilation** — full pipeline: analyze, mcode, streamline, serialize
2. **Build-cache dylib** — content-addressed `.dylib` in `~/.cell/build/<hash>`, found via manifest (see [Dylib Manifests](#dylib-manifests))
3. **Cached bytecode** — content-addressed in `~/.cell/build/<hash>` (no extension)
4. **Internal symbols** — statically linked into the `cell` binary (fat builds)
5. **Source compilation** — full pipeline: analyze, mcode, streamline, serialize
When both a `.dylib` and `.mach` exist for the same module in `lib/`, the dylib is selected. Dylib resolution also wins over internal symbols, so a dylib in `lib/` can hot-patch a fat binary. Delete the dylib to fall back to mach or static.
Results from steps 5-7 are cached back to the content-addressed store for future loads.
Dylib resolution wins over internal symbols, so a built dylib can hot-patch a fat binary. Results from compilation are cached back to the content-addressed store for future loads.
Each loading method (except the in-memory cache) can be individually enabled or disabled via `shop.toml` policy flags — see [Shop Configuration](#shop-configuration) below.
### Content-Addressed Store
The build cache at `~/.pit/build/` stores ephemeral artifacts named by the BLAKE2 hash of their inputs:
The build cache at `~/.cell/build/` stores all compiled artifacts named by the BLAKE2 hash of their inputs:
```
~/.pit/build/
├── a1b2c3d4... # cached bytecode blob (no extension)
├── c9d0e1f2...mcode # cached JSON IR
└── f3a4b5c6... # compiled dylib (checked before copying to lib/)
~/.cell/build/
├── a1b2c3d4... # cached bytecode blob, object file, dylib, or manifest (no extension)
└── ...
```
This scheme provides automatic cache invalidation: when source changes, its hash changes, and the old cache entry is simply never looked up again. When building a dylib, the build cache is checked first — if a matching hash exists, it is copied to `lib/` without recompiling.
Every artifact type uses a unique salt appended to the content before hashing, so collisions between different artifact types are impossible:
| Salt | Artifact |
|------|----------|
| `obj` | compiled C object file |
| `dylib` | linked dynamic library |
| `native` | native-compiled .cm dylib |
| `mach` | mach bytecode blob |
| `mcode` | mcode IR (JSON) |
| `deps` | cached `cc -MM` dependency list |
| `fail` | cached compilation failure marker |
| `manifest` | package dylib manifest (JSON) |
This scheme provides automatic cache invalidation: when source changes, its hash changes, and the old cache entry is simply never looked up again.
### Failure Caching
When a C file fails to compile (missing SDK headers, syntax errors, etc.), the build system writes a failure marker to the cache using the `fail` salt. On subsequent builds, the failure marker is found and the file is skipped immediately — no time wasted retrying files that can't compile. The failure marker is keyed on the same content as the compilation (command string + source content), so if the source changes or compiler flags change, the failure is automatically invalidated and compilation is retried.
### Dylib Manifests
Dylibs live at content-addressed paths (`~/.cell/build/<hash>`) that can only be computed by running the full build pipeline. To allow the runtime to find pre-built dylibs without invoking the build module, `cell build` writes a **manifest** for each package. The manifest is a JSON file mapping each C module to its `{file, symbol, dylib}` entry. The manifest path is itself content-addressed (BLAKE2 hash of the package name + `manifest` salt), so the runtime can compute it from the package name alone.
At runtime, when `use()` needs a C module from another package, the shop reads the manifest to find the dylib path. This means `cell build` must be run before C modules from packages can be loaded.
### Core Module Caching
@@ -124,10 +142,10 @@ symbol: js_gitea_pockle_world_john_prosperon_sprite_use
### C Resolution Sources
1. **Installed dylibs** — per-file dylibs in `~/.pit/lib/<pkg>/<stem>.dylib` (deterministic paths, no manifests)
2. **Internal symbols** — statically linked into the `pit` binary (fat builds)
1. **Build-cache dylibs** — content-addressed dylibs in `~/.cell/build/<hash>`, found via per-package manifests written by `cell build`
2. **Internal symbols** — statically linked into the `cell` binary (fat builds)
Dylibs are checked first at each resolution scope, so an installed dylib always wins over a statically linked symbol. This enables hot-patching fat binaries by placing a dylib in `lib/`.
Dylibs are checked first at each resolution scope, so a built dylib always wins over a statically linked symbol. This enables hot-patching fat binaries.
### Name Collisions
@@ -145,7 +163,7 @@ The set of injected capabilities is controlled by `script_inject_for()`, which c
## Shop Configuration
The shop reads an optional `shop.toml` file from the shop root (`~/.pit/shop.toml`). This file controls which loading methods are permitted through policy flags.
The shop reads an optional `shop.toml` file from the shop root (`~/.cell/shop.toml`). This file controls which loading methods are permitted through policy flags.
### Policy Flags
@@ -188,22 +206,12 @@ If `shop.toml` is missing or has no `[policy]` section, all methods are enabled
## Shop Directory Layout
```
~/.pit/
~/.cell/
├── packages/ # installed packages (directories and symlinks)
│ └── core -> ... # symlink to the ƿit core
├── lib/ # INSTALLED per-file artifacts (persistent, human-readable)
│ ├── core/
│ │ ├── fd.dylib
│ │ ├── time.mach
│ │ ├── time.dylib
│ │ └── internal/
│ │ └── os.dylib
│ └── gitea_pockle_world_john_prosperon/
│ ├── sprite.dylib
│ └── render.dylib
├── build/ # EPHEMERAL cache (safe to delete anytime)
│ ├── <hash> # cached bytecode or dylib blobs (no extension)
│ └── <hash>.mcode # cached JSON IR
├── build/ # content-addressed cache (safe to delete anytime)
│ ├── <hash> # cached bytecode, object file, dylib, or manifest
│ └── ...
├── cache/ # downloaded package zip archives
├── lock.toml # installed package versions and commit hashes
├── link.toml # local development link overrides

45
internal/shop.cm
View File

@@ -843,25 +843,51 @@ function make_c_symbol(pkg, file) {
return 'js_' + pkg_safe + '_' + file_safe + '_use'
}
// Compute the manifest path for a package (must match build.cm's manifest_path)
function dylib_manifest_path(pkg) {
var hash = content_hash(stone(blob(pkg + '\n' + 'manifest')))
return global_shop_path + '/build/' + hash
}
// Read a pre-built dylib manifest for a package.
// Returns the array of {file, symbol, dylib} or null.
function read_dylib_manifest(pkg) {
var mpath = dylib_manifest_path(pkg)
if (!fd.is_file(mpath)) return null
var content = text(fd.slurp(mpath))
if (!content || length(content) == 0) return null
return json.decode(content)
}
// Ensure all C modules for a package are built and loaded.
// Returns the array of {file, symbol, dylib} results, cached per package.
function ensure_package_dylibs(pkg) {
if (package_dylibs[pkg]) return package_dylibs[pkg]
var results = null
var build_mod = use_cache['core/build']
if (!build_mod) return null
var target = null
var c_files = null
var target = detect_host_target()
if (!target) return null
if (build_mod) {
target = detect_host_target()
if (!target) return null
var c_files = pkg_tools.get_c_files(pkg, target, true)
if (!c_files || length(c_files) == 0) {
package_dylibs[pkg] = []
return []
c_files = pkg_tools.get_c_files(pkg, target, true)
if (!c_files || length(c_files) == 0) {
package_dylibs[pkg] = []
return []
}
log.shop('ensuring C modules for ' + pkg)
results = build_mod.build_dynamic(pkg, target, 'release', {})
} else {
// No build module at runtime — read manifest from cell build
results = read_dylib_manifest(pkg)
if (!results) return null
log.shop('loaded manifest for ' + pkg + ' (' + text(length(results)) + ' modules)')
}
log.shop('ensuring C modules for ' + pkg)
var results = build_mod.build_dynamic(pkg, target, 'release', {})
package_dylibs[pkg] = results
// Preload all sibling dylibs with RTLD_LAZY|RTLD_GLOBAL
@@ -873,7 +899,6 @@ function ensure_package_dylibs(pkg) {
}
})
log.shop('built ' + text(length(results)) + ' C module(s) for ' + pkg)
return results
}

10
source/runtime.c
View File

@@ -6889,7 +6889,15 @@ static JSValue js_cell_text (JSContext *ctx, JSValue this_val, int argc, JSValue
if (bd->length % 8 != 0)
return JS_RaiseDisrupt (ctx,
"text: blob not byte-aligned for UTF-8");
return JS_NewStringLen (ctx, (const char *)data, byte_len);
/* Copy blob data to a temp buffer before JS_NewStringLen, because
JS_NewStringLen allocates internally (js_alloc_string) which can
trigger GC, moving the blob and invalidating data. */
char *tmp = pjs_malloc (byte_len);
if (!tmp) return JS_ThrowMemoryError (ctx);
memcpy (tmp, data, byte_len);
JSValue result = JS_NewStringLen (ctx, tmp, byte_len);
pjs_free (tmp);
return result;
}
}