john/cell
1
0
Fork 0
Code Issues 31 Pull Requests Actions Packages Projects Releases 2 Wiki Activity

Merge branch 'native_boot'

Browse Source
This commit is contained in:
John Alanbrook
2026-02-22 20:48:19 -06:00
parent 62440d3ed6 4edc4b7cc5
commit b03edb0d90
19 changed files with 59808 additions and 22461 deletions
Download Patch File Download Diff File Expand all files Collapse all files

105
source/cell.c
View File

@@ -20,6 +20,7 @@
#include <signal.h>
#include <unistd.h>
#include <sys/stat.h>
#include <dlfcn.h>
#include "monocypher.h"
/* Test suite declarations */
@@ -249,6 +250,54 @@ static char *try_engine_cache(size_t *out_size) {
return NULL;
}
static const char *detect_host_target(void) {
#if defined(__APPLE__) && defined(__aarch64__)
return "macos_arm64";
#elif defined(__APPLE__) && defined(__x86_64__)
return "macos_x86_64";
#elif defined(__linux__) && defined(__x86_64__)
return "linux";
#elif defined(__linux__) && defined(__aarch64__)
return "linux_arm64";
#elif defined(_WIN32)
return "windows";
#else
return NULL;
#endif
}
static char *try_engine_native_cache(void) {
const char *target = detect_host_target();
if (!target) return NULL;
size_t src_size;
char *src = load_core_file(ENGINE_SRC, &src_size);
if (!src) return NULL;
size_t target_len = strlen(target);
size_t key_len = src_size + 1 + target_len + 8 + 7;
char *key = malloc(key_len + 1);
if (!key) { free(src); return NULL; }
memcpy(key, src, src_size);
size_t pos = src_size;
key[pos++] = '\n';
memcpy(key + pos, target, target_len);
pos += target_len;
memcpy(key + pos, "\nnative\n", 8);
pos += 8;
memcpy(key + pos, "\nnative", 7);
pos += 7;
key[pos] = '\0';
free(src);
char *hex = compute_blake2_hex(key, pos);
free(key);
if (!hex) return NULL;
char *cpath = build_cache_path(hex);
free(hex);
if (!cpath) return NULL;
struct stat st;
if (stat(cpath, &st) != 0) { free(cpath); return NULL; }
return cpath;
}
// Get the core path for use by scripts
const char* cell_get_core_path(void) {
return core_path;
@@ -327,6 +376,8 @@ void script_startup(cell_rt *prt)
}
btmp = shop_path ? JS_NewString(js, shop_path) : JS_NULL;
JS_SetPropertyStr(js, boot_env_ref.val, "shop_path", btmp);
if (native_mode)
JS_SetPropertyStr(js, boot_env_ref.val, "native_mode", JS_NewBool(js, 1));
JSValue boot_env = JS_Stone(js, boot_env_ref.val);
JS_DeleteGCRef(js, &boot_env_ref);
@@ -465,6 +516,8 @@ static void print_usage(const char *prog)
printf("Run the 'help' script like 'cell help' to see available scripts\n");
}
JSValue cell_rt_native_module_load_named(JSContext *ctx, void *dl_handle, const char *sym_name, JSValue env);
int cell_init(int argc, char **argv)
{
/* Check for --help flag */
@@ -604,12 +657,21 @@ int cell_init(int argc, char **argv)
JS_FreeValue(ctx, js_core_blob_use(ctx));
int exit_code = 0;
int use_native_engine = 0;
char *native_dylib_path = NULL;
void *native_handle = NULL;
// Native mode: try native engine cache first
if (native_mode)
native_dylib_path = try_engine_native_cache();
// Try engine fast-path: load engine.cm from source-hash cache
size_t bin_size;
char *bin_data = try_engine_cache(&bin_size);
char *bin_data = NULL;
if (!native_dylib_path)
bin_data = try_engine_cache(&bin_size);
if (!bin_data) {
if (!native_dylib_path && !bin_data) {
// Cold path: run bootstrap to seed cache, then retry
size_t boot_size;
char *boot_data = load_core_file(BOOTSTRAP_MCODE, &boot_size);
@@ -640,6 +702,8 @@ int cell_init(int argc, char **argv)
JS_SetPropertyStr(ctx, boot_env_ref.val, "actorsym", JS_DupValue(ctx, cli_rt->actor_sym_ref.val));
btmp = js_core_json_use(ctx);
JS_SetPropertyStr(ctx, boot_env_ref.val, "json", btmp);
if (native_mode)
JS_SetPropertyStr(ctx, boot_env_ref.val, "native_mode", JS_NewBool(ctx, 1));
JS_SetPropertyStr(ctx, boot_env_ref.val, "init", JS_NULL);
JSGCRef boot_args_ref;
JS_AddGCRef(ctx, &boot_args_ref);
@@ -661,14 +725,32 @@ int cell_init(int argc, char **argv)
return 1;
}
// Retry engine from cache (new-style bootstrap seeds it)
bin_data = try_engine_cache(&bin_size);
if (!bin_data) {
// Old-style bootstrap already ran the program — skip engine load
goto check_actors;
// After bootstrap, retry cache
if (native_mode)
native_dylib_path = try_engine_native_cache();
if (!native_dylib_path) {
bin_data = try_engine_cache(&bin_size);
if (!bin_data) {
// Old-style bootstrap already ran the program — skip engine load
goto check_actors;
}
}
}
// Open native dylib if we have a path
if (native_dylib_path) {
native_handle = dlopen(native_dylib_path, RTLD_NOW | RTLD_GLOBAL);
if (native_handle) {
use_native_engine = 1;
} else {
// Fall back to bytecode
if (!bin_data)
bin_data = try_engine_cache(&bin_size);
}
free(native_dylib_path);
native_dylib_path = NULL;
}
{
// Build engine environment
JSGCRef env_ref;
@@ -713,10 +795,15 @@ int cell_init(int argc, char **argv)
JSValue hidden_env = JS_Stone(ctx, env_ref.val);
g_crash_ctx = ctx;
JSValue result = JS_RunMachBin(ctx, (const uint8_t *)bin_data, bin_size, hidden_env);
JSValue result;
if (use_native_engine) {
result = cell_rt_native_module_load_named(ctx, native_handle, "cell_main", hidden_env);
} else {
result = JS_RunMachBin(ctx, (const uint8_t *)bin_data, bin_size, hidden_env);
free(bin_data);
}
g_crash_ctx = NULL;
JS_DeleteGCRef(ctx, &env_ref);
free(bin_data);
if (JS_IsException(result)) {
JS_GetException(ctx);

8
source/mach.c
View File

@@ -965,7 +965,7 @@ JSValue js_new_register_function(JSContext *ctx, JSCodeRegister *code, JSValue e
return out;
}
JSValue js_new_native_function_with_code(JSContext *ctx, JSValue code_obj, int arity, JSValue outer_frame) {
JSValue js_new_native_function_with_code(JSContext *ctx, JSValue code_obj, int arity, JSValue outer_frame, JSValue env_record) {
JSGCRef frame_ref;
JSGCRef fn_ref;
JSFunction *fn;
@@ -988,7 +988,7 @@ JSValue js_new_native_function_with_code(JSContext *ctx, JSValue code_obj, int a
fn->name = JS_NULL;
fn = JS_VALUE_GET_FUNCTION(fn_ref.val);
fn->u.cell.code = code_obj;
fn->u.cell.env_record = JS_NULL;
fn->u.cell.env_record = env_record;
fn->u.cell.outer_frame = frame_ref.val;
JSValue out = fn_ref.val;
@@ -999,11 +999,11 @@ JSValue js_new_native_function_with_code(JSContext *ctx, JSValue code_obj, int a
/* Create a native (QBE-compiled) function */
JSValue js_new_native_function(JSContext *ctx, void *fn_ptr, void *dl_handle,
uint16_t nr_slots, int arity, JSValue outer_frame) {
uint16_t nr_slots, int arity, JSValue outer_frame, JSValue env) {
JSValue code_obj = js_new_native_code(ctx, fn_ptr, dl_handle, nr_slots, arity);
if (JS_IsException(code_obj))
return JS_EXCEPTION;
return js_new_native_function_with_code(ctx, code_obj, arity, outer_frame);
return js_new_native_function_with_code(ctx, code_obj, arity, outer_frame, env);
}
/* Binary operations helper */

115
source/qbe_helpers.c
View File

@@ -212,9 +212,6 @@ typedef struct {
AOTCodeCacheEntry *code_cache;
int code_cache_count;
int code_cache_cap;
JSGCRef native_env_ref;
int has_native_env;
int native_env_ref_inited;
AOTGCRefChunk **gc_ref_chunks;
int gc_ref_chunk_count;
int aot_depth;
@@ -392,51 +389,22 @@ int cell_rt_store_dynamic(JSContext *ctx, JSValue val, JSValue obj,
}
}
/* --- Intrinsic/global lookup --- */
/* --- Env-based variable lookup (env at frame slot 0) --- */
void cell_rt_set_native_env(JSContext *ctx, JSValue env) {
NativeRTState *st = native_state(ctx);
if (!st) return;
if (!JS_IsNull(env) && !JS_IsStone(env)) {
fprintf(stderr, "cell_rt_set_native_env: ERROR env not stone\n");
abort();
JSValue cell_rt_access_env(JSContext *ctx, JSValue env, int64_t lit_idx) {
JSValue key = aot_lit_from_index(ctx, lit_idx);
if (JS_IsException(key))
return JS_EXCEPTION;
/* Try env first (linear scan for stoned records) */
if (!JS_IsNull(env) && JS_IsRecord(env)) {
JSRecord *rec = (JSRecord *)chase(env);
uint64_t mask = objhdr_cap56(rec->mist_hdr);
for (uint64_t i = 1; i <= mask; i++) {
if (js_key_equal(rec->slots[i].key, key))
return rec->slots[i].val;
}
}
/* Drop module literal pool roots before switching native env/module. */
aot_clear_lit_pool(ctx, st);
/* Native module boundary: clear per-actor key cache so stale keys
cannot survive across context/module lifetimes. */
free(st->key_cache);
st->key_cache = NULL;
st->key_cache_count = 0;
st->key_cache_cap = 0;
if (st->has_native_env && st->native_env_ref_inited) {
JS_DeleteGCRef(ctx, &st->native_env_ref);
st->native_env_ref_inited = 0;
}
if (!JS_IsNull(env)) {
JS_AddGCRef(ctx, &st->native_env_ref);
st->native_env_ref_inited = 1;
st->native_env_ref.val = env;
st->has_native_env = 1;
} else {
st->has_native_env = 0;
st->native_env_ref.val = JS_NULL;
}
}
static JSValue cell_rt_get_intrinsic_key(JSContext *ctx, JSValue key) {
NativeRTState *st = native_state(ctx);
if (!st) return JS_EXCEPTION;
/* Check native env first (runtime-provided functions like log) */
if (st->has_native_env) {
JSValue v = JS_GetProperty(ctx, st->native_env_ref.val, key);
if (!JS_IsNull(v))
return v;
}
/* Linear scan of global object — avoids hash mismatch issues with
stoned records whose keys may be in cold storage */
/* Fallback to global object */
JSValue gobj = ctx->global_obj;
if (JS_IsRecord(gobj)) {
JSRecord *rec = (JSRecord *)chase(gobj);
@@ -446,24 +414,12 @@ static JSValue cell_rt_get_intrinsic_key(JSContext *ctx, JSValue key) {
return rec->slots[i].val;
}
}
JS_RaiseDisrupt(ctx, "name is not defined");
const char *kstr = JS_ToCString(ctx, key);
JS_RaiseDisrupt(ctx, "'%s' is not defined", kstr ? kstr : "?");
if (kstr) JS_FreeCString(ctx, kstr);
return JS_EXCEPTION;
}
JSValue cell_rt_get_intrinsic(JSContext *ctx, const char *name) {
JSValue key = aot_key_from_cstr(ctx, name);
if (JS_IsException(key))
return JS_EXCEPTION;
return cell_rt_get_intrinsic_key(ctx, key);
}
JSValue cell_rt_get_intrinsic_lit(JSContext *ctx, int64_t lit_idx) {
JSValue key = aot_lit_from_index(ctx, lit_idx);
if (JS_IsException(key))
return JS_EXCEPTION;
return cell_rt_get_intrinsic_key(ctx, key);
}
/* --- GC-managed AOT frame stack ---
Each native dispatch loop pushes a GC ref so the GC can find and
update the current frame pointer when it moves objects.
@@ -1046,7 +1002,6 @@ static JSValue aot_get_or_create_native_code(JSContext *ctx, NativeRTState *st,
Called from QBE-generated code during function creation. */
JSValue cell_rt_make_function(JSContext *ctx, int64_t fn_idx, void *outer_fp,
int64_t nr_args, int64_t nr_slots) {
(void)outer_fp;
NativeRTState *st = native_state(ctx);
if (!st) return JS_EXCEPTION;
if (!st->current_dl_handle)
@@ -1062,7 +1017,18 @@ JSValue cell_rt_make_function(JSContext *ctx, int64_t fn_idx, void *outer_fp,
if (st->aot_depth > 0)
outer_frame = aot_gc_ref_at(st, st->aot_depth - 1)->val;
return js_new_native_function_with_code(ctx, code_obj, (int)nr_args, outer_frame);
/* Read env from the parent frame's function object */
JSValue env = JS_NULL;
if (outer_fp) {
JSFrameRegister *parent = (JSFrameRegister *)((char *)outer_fp - offsetof(JSFrameRegister, slots));
if (JS_IsFunction(parent->function)) {
JSFunction *parent_fn = JS_VALUE_GET_FUNCTION(parent->function);
if (parent_fn->kind == JS_FUNC_KIND_NATIVE)
env = parent_fn->u.cell.env_record;
}
}
return js_new_native_function_with_code(ctx, code_obj, (int)nr_args, outer_frame, env);
}
/* --- Frame-based function calling ---
@@ -1209,15 +1175,16 @@ JSValue cell_rt_regexp(JSContext *ctx, const char *pattern, const char *flags) {
Creates a temporary JS_FUNC_KIND_NATIVE function so that the full
dispatch loop (tail calls, closures, etc.) works for module-level code. */
static JSValue native_module_run(JSContext *ctx, void *dl_handle,
cell_compiled_fn entry, int nr_slots) {
cell_compiled_fn entry, int nr_slots,
JSValue env) {
NativeRTState *st = native_state(ctx);
if (!st) return JS_EXCEPTION;
void *prev_handle = st->current_dl_handle;
st->current_dl_handle = dl_handle;
/* Create a native function object for the entry point */
/* Create a native function object for the entry point, with env on the function */
JSValue func_obj = js_new_native_function(ctx, (void *)entry, dl_handle,
(uint16_t)nr_slots, 0, JS_NULL);
(uint16_t)nr_slots, 0, JS_NULL, env);
if (JS_IsException(func_obj)) {
st->current_dl_handle = prev_handle;
return JS_EXCEPTION;
@@ -1237,14 +1204,11 @@ JSValue cell_rt_native_module_load(JSContext *ctx, void *dl_handle, JSValue env)
if (!fn)
return JS_RaiseDisrupt(ctx, "cell_main not found in native module dylib");
/* Make env available for cell_rt_get_intrinsic lookups */
cell_rt_set_native_env(ctx, env);
/* Try to read nr_slots from the module (exported by emitter) */
int *slots_ptr = (int *)dlsym(dl_handle, "cell_main_nr_slots");
int nr_slots = slots_ptr ? *slots_ptr : 512;
return native_module_run(ctx, dl_handle, fn, nr_slots);
return native_module_run(ctx, dl_handle, fn, nr_slots, env);
}
/* Load a native module from a dylib handle, trying a named symbol first.
@@ -1263,16 +1227,13 @@ JSValue cell_rt_native_module_load_named(JSContext *ctx, void *dl_handle, const
if (!fn)
return JS_RaiseDisrupt(ctx, "symbol not found in native module dylib");
/* Make env available for cell_rt_get_intrinsic lookups */
cell_rt_set_native_env(ctx, env);
/* Try to read nr_slots from the module */
char slots_sym[128];
snprintf(slots_sym, sizeof(slots_sym), "%s_nr_slots", used_name);
int *slots_ptr = (int *)dlsym(dl_handle, slots_sym);
int nr_slots = slots_ptr ? *slots_ptr : 512;
return native_module_run(ctx, dl_handle, fn, nr_slots);
return native_module_run(ctx, dl_handle, fn, nr_slots, env);
}
void cell_rt_free_native_state(JSContext *ctx) {
@@ -1281,12 +1242,6 @@ void cell_rt_free_native_state(JSContext *ctx) {
aot_clear_lit_pool(ctx, st);
if (st->has_native_env && st->native_env_ref_inited) {
JS_DeleteGCRef(ctx, &st->native_env_ref);
st->native_env_ref_inited = 0;
st->native_env_ref.val = JS_NULL;
}
for (int ci = 0; ci < st->gc_ref_chunk_count; ci++) {
AOTGCRefChunk *chunk = st->gc_ref_chunks[ci];
if (!chunk) continue;

4
source/quickjs-internal.h
View File

@@ -1287,8 +1287,8 @@ JSValue js_key_from_string (JSContext *ctx, JSValue val);
/* mach.c exports */
JSValue JS_CallRegisterVM(JSContext *ctx, JSCodeRegister *code, JSValue this_obj, int argc, JSValue *argv, JSValue env, JSValue outer_frame);
JSValue js_new_native_function(JSContext *ctx, void *fn_ptr, void *dl_handle, uint16_t nr_slots, int arity, JSValue outer_frame);
JSValue js_new_native_function_with_code(JSContext *ctx, JSValue code_obj, int arity, JSValue outer_frame);
JSValue js_new_native_function(JSContext *ctx, void *fn_ptr, void *dl_handle, uint16_t nr_slots, int arity, JSValue outer_frame, JSValue env);
JSValue js_new_native_function_with_code(JSContext *ctx, JSValue code_obj, int arity, JSValue outer_frame, JSValue env_record);
JSFrameRegister *alloc_frame_register(JSContext *ctx, int slot_count);
void cell_rt_free_native_state(JSContext *ctx);